From d208d875667e2a29beeec5d475f4b6b164b632fa Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Fri, 20 Oct 2023 10:19:45 +0000 Subject: drm: introduce CLOSEFB IOCTL This new IOCTL allows callers to close a framebuffer without disabling planes or CRTCs. This takes inspiration from Rob Clark's unref_fb IOCTL [1] and DRM_MODE_FB_PERSIST [2]. User-space patch for wlroots available at [3]. IGT test available at [4]. v2: add an extra pad field just in case we want to extend this IOCTL in the future (Pekka, Sima). [1]: https://lore.kernel.org/dri-devel/20170509153654.23464-1-robdclark@gmail.com/ [2]: https://lore.kernel.org/dri-devel/20211006151921.312714-1-contact@emersion.fr/ [3]: https://gitlab.freedesktop.org/wlroots/wlroots/-/merge_requests/4394 [4]: https://lists.freedesktop.org/archives/igt-dev/2023-October/063294.html Signed-off-by: Simon Ser Reviewed-by: Daniel Stone Acked-by: Pekka Paalanen Cc: Hans de Goede Cc: Dennis Filder Cc: Daniel Vetter Cc: Rob Clark Cc: Sean Paul Link: https://patchwork.freedesktop.org/patch/msgid/20231020101926.145327-2-contact@emersion.fr --- include/uapi/drm/drm.h | 20 ++++++++++++++++++++ include/uapi/drm/drm_mode.h | 10 ++++++++++ 2 files changed, 30 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/drm.h b/include/uapi/drm/drm.h index de723566c5ae..8662b5aeea0c 100644 --- a/include/uapi/drm/drm.h +++ b/include/uapi/drm/drm.h @@ -1218,6 +1218,26 @@ extern "C" { #define DRM_IOCTL_SYNCOBJ_EVENTFD DRM_IOWR(0xCF, struct drm_syncobj_eventfd) +/** + * DRM_IOCTL_MODE_CLOSEFB - Close a framebuffer. + * + * This closes a framebuffer previously added via ADDFB/ADDFB2. The IOCTL + * argument is a framebuffer object ID. + * + * This IOCTL is similar to &DRM_IOCTL_MODE_RMFB, except it doesn't disable + * planes and CRTCs. As long as the framebuffer is used by a plane, it's kept + * alive. When the plane no longer uses the framebuffer (because the + * framebuffer is replaced with another one, or the plane is disabled), the + * framebuffer is cleaned up. + * + * This is useful to implement flicker-free transitions between two processes. + * + * Depending on the threat model, user-space may want to ensure that the + * framebuffer doesn't expose any sensitive user information: closed + * framebuffers attached to a plane can be read back by the next DRM master. + */ +#define DRM_IOCTL_MODE_CLOSEFB DRM_IOWR(0xD0, struct drm_mode_closefb) + /* * Device specific ioctls should only be in their respective headers * The device specific ioctl range is from 0x40 to 0x9f. diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index 128d09138ceb..09e7a471ee30 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -1323,6 +1323,16 @@ struct drm_mode_rect { __s32 y2; }; +/** + * struct drm_mode_closefb + * @fb_id: Framebuffer ID. + * @pad: Must be zero. + */ +struct drm_mode_closefb { + __u32 fb_id; + __u32 pad; +}; + #if defined(__cplusplus) } #endif -- cgit v1.2.3 From 1118d10f5e5ab544c489fad4da373f9988416ece Mon Sep 17 00:00:00 2001 From: Iago Toral Quiroga Date: Tue, 31 Oct 2023 08:38:56 +0100 Subject: drm/v3d: update UAPI to match user-space for V3D 7.x MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit V3D 7.x takes a new parameter to configure TFU jobs that needs to be provided by user space. Signed-off-by: Iago Toral Quiroga Reviewed-by: Maíra Canal Signed-off-by: Maíra Canal Link: https://patchwork.freedesktop.org/patch/msgid/20231031073859.25298-2-itoral@igalia.com --- include/uapi/drm/v3d_drm.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h index 3dfc0af8756a..1a7d7a689de3 100644 --- a/include/uapi/drm/v3d_drm.h +++ b/include/uapi/drm/v3d_drm.h @@ -319,6 +319,11 @@ struct drm_v3d_submit_tfu { /* Pointer to an array of ioctl extensions*/ __u64 extensions; + + struct { + __u32 ioc; + __u32 pad; + } v71; }; /* Submits a compute shader for dispatch. This job will block on any -- cgit v1.2.3 From 3b511278b6ef514b3ae3d99ff62947cddd434479 Mon Sep 17 00:00:00 2001 From: Pranjal Ramajor Asha Kanojiya Date: Fri, 27 Oct 2023 10:43:30 -0600 Subject: accel/qaic: Support for 0 resize slice execution in BO Add support to partially execute a slice which is resized to zero. Executing a zero size slice in a BO should mean that there is no DMA transfers involved but you should still configure doorbell and semaphores. For example consider a BO of size 18K and it is sliced into 3 6K slices and user calls partial execute ioctl with resize as 10K. slice 0 - size is 6k and offset is 0, so resize of 10K will not cut short this slice hence we send the entire slice for execution. slice 1 - size is 6k and offset is 6k, so resize of 10K will cut short this slice and only the first 4k should be DMA along with configuring doorbell and semaphores. slice 2 - size is 6k and offset is 12k, so resize of 10k will cut short this slice and no DMA transfer would be involved but we should would configure doorbell and semaphores. This change begs to change the behavior of 0 resize. Currently, 0 resize partial execute ioctl behaves exactly like execute ioctl i.e. no resize. After this patch all the slice in BO should behave exactly like slice 2 in above example. Refactor copy_partial_exec_reqs() to make it more readable and less complex. Signed-off-by: Pranjal Ramajor Asha Kanojiya Reviewed-by: Jeffrey Hugo Signed-off-by: Jeffrey Hugo Reviewed-by: Stanislaw Gruszka Link: https://patchwork.freedesktop.org/patch/msgid/20231027164330.11978-1-quic_jhugo@quicinc.com --- drivers/accel/qaic/qaic_data.c | 104 +++++++++++++++++------------------------ include/uapi/drm/qaic_accel.h | 5 +- 2 files changed, 46 insertions(+), 63 deletions(-) (limited to 'include/uapi') diff --git a/drivers/accel/qaic/qaic_data.c b/drivers/accel/qaic/qaic_data.c index ebc3cca1b094..8da81768f2ab 100644 --- a/drivers/accel/qaic/qaic_data.c +++ b/drivers/accel/qaic/qaic_data.c @@ -51,6 +51,7 @@ }) #define NUM_EVENTS 128 #define NUM_DELAYS 10 +#define fifo_at(base, offset) ((base) + (offset) * get_dbc_req_elem_size()) static unsigned int wait_exec_default_timeout_ms = 5000; /* 5 sec default */ module_param(wait_exec_default_timeout_ms, uint, 0600); @@ -1058,6 +1059,16 @@ unlock_usr_srcu: return ret; } +static inline u32 fifo_space_avail(u32 head, u32 tail, u32 q_size) +{ + u32 avail = head - tail - 1; + + if (head <= tail) + avail += q_size; + + return avail; +} + static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, u32 dbc_id, u32 head, u32 *ptail) { @@ -1066,27 +1077,20 @@ static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slic u32 tail = *ptail; u32 avail; - avail = head - tail; - if (head <= tail) - avail += dbc->nelem; - - --avail; - + avail = fifo_space_avail(head, tail, dbc->nelem); if (avail < slice->nents) return -EAGAIN; if (tail + slice->nents > dbc->nelem) { avail = dbc->nelem - tail; avail = min_t(u32, avail, slice->nents); - memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, - sizeof(*reqs) * avail); + memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * avail); reqs += avail; avail = slice->nents - avail; if (avail) memcpy(dbc->req_q_base, reqs, sizeof(*reqs) * avail); } else { - memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, - sizeof(*reqs) * slice->nents); + memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * slice->nents); } *ptail = (tail + slice->nents) % dbc->nelem; @@ -1094,46 +1098,31 @@ static inline int copy_exec_reqs(struct qaic_device *qdev, struct bo_slice *slic return 0; } -/* - * Based on the value of resize we may only need to transmit first_n - * entries and the last entry, with last_bytes to send from the last entry. - * Note that first_n could be 0. - */ static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_slice *slice, - u64 resize, u32 dbc_id, u32 head, u32 *ptail) + u64 resize, struct dma_bridge_chan *dbc, u32 head, + u32 *ptail) { - struct dma_bridge_chan *dbc = &qdev->dbc[dbc_id]; struct dbc_req *reqs = slice->reqs; struct dbc_req *last_req; u32 tail = *ptail; - u64 total_bytes; u64 last_bytes; u32 first_n; u32 avail; - int ret; - int i; - - avail = head - tail; - if (head <= tail) - avail += dbc->nelem; - --avail; + avail = fifo_space_avail(head, tail, dbc->nelem); - total_bytes = 0; - for (i = 0; i < slice->nents; i++) { - total_bytes += le32_to_cpu(reqs[i].len); - if (total_bytes >= resize) + /* + * After this for loop is complete, first_n represents the index + * of the last DMA request of this slice that needs to be + * transferred after resizing and last_bytes represents DMA size + * of that request. + */ + last_bytes = resize; + for (first_n = 0; first_n < slice->nents; first_n++) + if (last_bytes > le32_to_cpu(reqs[first_n].len)) + last_bytes -= le32_to_cpu(reqs[first_n].len); + else break; - } - - if (total_bytes < resize) { - /* User space should have used the full buffer path. */ - ret = -EINVAL; - return ret; - } - - first_n = i; - last_bytes = i ? resize + le32_to_cpu(reqs[i].len) - total_bytes : resize; if (avail < (first_n + 1)) return -EAGAIN; @@ -1142,22 +1131,21 @@ static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_sli if (tail + first_n > dbc->nelem) { avail = dbc->nelem - tail; avail = min_t(u32, avail, first_n); - memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, - sizeof(*reqs) * avail); + memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * avail); last_req = reqs + avail; avail = first_n - avail; if (avail) memcpy(dbc->req_q_base, last_req, sizeof(*reqs) * avail); } else { - memcpy(dbc->req_q_base + tail * get_dbc_req_elem_size(), reqs, - sizeof(*reqs) * first_n); + memcpy(fifo_at(dbc->req_q_base, tail), reqs, sizeof(*reqs) * first_n); } } - /* Copy over the last entry. Here we need to adjust len to the left over + /* + * Copy over the last entry. Here we need to adjust len to the left over * size, and set src and dst to the entry it is copied to. */ - last_req = dbc->req_q_base + (tail + first_n) % dbc->nelem * get_dbc_req_elem_size(); + last_req = fifo_at(dbc->req_q_base, (tail + first_n) % dbc->nelem); memcpy(last_req, reqs + slice->nents - 1, sizeof(*reqs)); /* @@ -1168,6 +1156,9 @@ static inline int copy_partial_exec_reqs(struct qaic_device *qdev, struct bo_sli last_req->len = cpu_to_le32((u32)last_bytes); last_req->src_addr = reqs[first_n].src_addr; last_req->dest_addr = reqs[first_n].dest_addr; + if (!last_bytes) + /* Disable DMA transfer */ + last_req->cmd = GENMASK(7, 2) & reqs[first_n].cmd; *ptail = (tail + first_n + 1) % dbc->nelem; @@ -1227,26 +1218,17 @@ static int send_bo_list_to_device(struct qaic_device *qdev, struct drm_file *fil bo->req_id = dbc->next_req_id++; list_for_each_entry(slice, &bo->slices, slice) { - /* - * If this slice does not fall under the given - * resize then skip this slice and continue the loop - */ - if (is_partial && pexec[i].resize && pexec[i].resize <= slice->offset) - continue; - for (j = 0; j < slice->nents; j++) slice->reqs[j].req_id = cpu_to_le16(bo->req_id); - /* - * If it is a partial execute ioctl call then check if - * resize has cut this slice short then do a partial copy - * else do complete copy - */ - if (is_partial && pexec[i].resize && - pexec[i].resize < slice->offset + slice->size) + if (is_partial && (!pexec[i].resize || pexec[i].resize <= slice->offset)) + /* Configure the slice for no DMA transfer */ + ret = copy_partial_exec_reqs(qdev, slice, 0, dbc, head, tail); + else if (is_partial && pexec[i].resize < slice->offset + slice->size) + /* Configure the slice to be partially DMA transferred */ ret = copy_partial_exec_reqs(qdev, slice, - pexec[i].resize - slice->offset, - dbc->id, head, tail); + pexec[i].resize - slice->offset, dbc, + head, tail); else ret = copy_exec_reqs(qdev, slice, dbc->id, head, tail); if (ret) { diff --git a/include/uapi/drm/qaic_accel.h b/include/uapi/drm/qaic_accel.h index 43ac5d864512..9dab32316aee 100644 --- a/include/uapi/drm/qaic_accel.h +++ b/include/uapi/drm/qaic_accel.h @@ -287,8 +287,9 @@ struct qaic_execute_entry { * struct qaic_partial_execute_entry - Defines a BO to resize and submit. * @handle: In. GEM handle of the BO to commit to the device. * @dir: In. Direction of data. 1 = to device, 2 = from device. - * @resize: In. New size of the BO. Must be <= the original BO size. 0 is - * short for no resize. + * @resize: In. New size of the BO. Must be <= the original BO size. + * @resize as 0 would be interpreted as no DMA transfer is + * involved. */ struct qaic_partial_execute_entry { __u32 handle; -- cgit v1.2.3 From 48d45fac3940347becd290b96b2fc6d5ad8171f7 Mon Sep 17 00:00:00 2001 From: Jacek Lawrynowicz Date: Tue, 31 Oct 2023 08:31:55 +0100 Subject: accel/ivpu: Remove support for uncached buffers Usages of DRM_IVPU_BO_UNCACHED should be replaced by DRM_IVPU_BO_WC. There is no functional benefit from DRM_IVPU_BO_UNCACHED if these buffers are never mapped to host VM. This allows to cut the buffer handling code in the kernel driver by half. Usage of DRM_IVPU_BO_UNCACHED buffers was removed from user-space driver and will not be part of first UMD release. Signed-off-by: Jacek Lawrynowicz Reviewed-by: Jeffrey Hugo Link: https://patchwork.freedesktop.org/patch/msgid/20231031073156.1301669-4-stanislaw.gruszka@linux.intel.com --- drivers/accel/ivpu/ivpu_fw.c | 2 +- drivers/accel/ivpu/ivpu_gem.c | 3 --- include/uapi/drm/ivpu_accel.h | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/drivers/accel/ivpu/ivpu_fw.c b/drivers/accel/ivpu/ivpu_fw.c index 3fd74cd4205f..3d0c18d68f6c 100644 --- a/drivers/accel/ivpu/ivpu_fw.c +++ b/drivers/accel/ivpu/ivpu_fw.c @@ -286,7 +286,7 @@ static int ivpu_fw_mem_init(struct ivpu_device *vdev) if (fw->shave_nn_size) { fw->mem_shave_nn = ivpu_bo_alloc_internal(vdev, vdev->hw->ranges.shave.start, - fw->shave_nn_size, DRM_IVPU_BO_UNCACHED); + fw->shave_nn_size, DRM_IVPU_BO_WC); if (!fw->mem_shave_nn) { ivpu_err(vdev, "Failed to allocate shavenn buffer\n"); ret = -ENOMEM; diff --git a/drivers/accel/ivpu/ivpu_gem.c b/drivers/accel/ivpu/ivpu_gem.c index 915c53d7bb97..2a91eb1e3627 100644 --- a/drivers/accel/ivpu/ivpu_gem.c +++ b/drivers/accel/ivpu/ivpu_gem.c @@ -89,8 +89,6 @@ static int __must_check shmem_alloc_pages_locked(struct ivpu_bo *bo) if (bo->flags & DRM_IVPU_BO_WC) set_pages_array_wc(pages, npages); - else if (bo->flags & DRM_IVPU_BO_UNCACHED) - set_pages_array_uc(pages, npages); bo->pages = pages; return 0; @@ -366,7 +364,6 @@ ivpu_bo_alloc(struct ivpu_device *vdev, u64 size, u32 flags, const struct ivpu_b switch (flags & DRM_IVPU_BO_CACHE_MASK) { case DRM_IVPU_BO_CACHED: - case DRM_IVPU_BO_UNCACHED: case DRM_IVPU_BO_WC: break; default: diff --git a/include/uapi/drm/ivpu_accel.h b/include/uapi/drm/ivpu_accel.h index 262db0c3beee..de1944e42c65 100644 --- a/include/uapi/drm/ivpu_accel.h +++ b/include/uapi/drm/ivpu_accel.h @@ -196,7 +196,7 @@ struct drm_ivpu_bo_create { * * %DRM_IVPU_BO_UNCACHED: * - * Allocated BO will not be cached on host side nor snooped on the VPU side. + * Not supported. Use DRM_IVPU_BO_WC instead. * * %DRM_IVPU_BO_WC: * -- cgit v1.2.3 From 7add80126bcedddd157ddc09988b032c93ed56c7 Mon Sep 17 00:00:00 2001 From: Gurchetan Singh Date: Wed, 18 Oct 2023 11:17:27 -0700 Subject: drm/uapi: add explicit virtgpu context debug name There are two problems with the current method of determining the virtio-gpu debug name. 1) TASK_COMM_LEN is defined to be 16 bytes only, and this is a Linux kernel idiom (see PR_SET_NAME + PR_GET_NAME). Though, Android/FreeBSD get around this via setprogname(..)/getprogname(..) in libc. On Android, names longer than 16 bytes are common. For example, one often encounters a program like "com.android.systemui". The virtio-gpu spec allows the debug name to be up to 64 bytes, so ideally userspace should be able to set debug names up to 64 bytes. 2) The current implementation determines the debug name using whatever task initiated virtgpu. This is could be a "RenderThread" of a larger program, when we actually want to propagate the debug name of the program. To fix these issues, add a new CONTEXT_INIT param that allows userspace to set the debug name when creating a context. It takes a null-terminated C-string as the param value. The length of the string (excluding the terminator) **should** be <= 64 bytes. Otherwise, the debug_name will be truncated to 64 bytes. Link to open-source userspace: https://android-review.googlesource.com/c/platform/hardware/google/gfxstream/+/2787176 Signed-off-by: Gurchetan Singh Reviewed-by: Josh Simonot Reviewed-by: Dmitry Osipenko Signed-off-by: Dmitry Osipenko Link: https://patchwork.freedesktop.org/patch/msgid/20231018181727.772-2-gurchetansingh@chromium.org --- drivers/gpu/drm/virtio/virtgpu_drv.h | 5 +++++ drivers/gpu/drm/virtio/virtgpu_ioctl.c | 36 ++++++++++++++++++++++++++++------ include/uapi/drm/virtgpu_drm.h | 2 ++ 3 files changed, 37 insertions(+), 6 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h index 96365a772f77..bb7d86a0c6a1 100644 --- a/drivers/gpu/drm/virtio/virtgpu_drv.h +++ b/drivers/gpu/drm/virtio/virtgpu_drv.h @@ -58,6 +58,9 @@ #define MAX_CAPSET_ID 63 #define MAX_RINGS 64 +/* See virtio_gpu_ctx_create. One additional character for NULL terminator. */ +#define DEBUG_NAME_MAX_LEN 65 + struct virtio_gpu_object_params { unsigned long size; bool dumb; @@ -274,6 +277,8 @@ struct virtio_gpu_fpriv { uint64_t base_fence_ctx; uint64_t ring_idx_mask; struct mutex context_lock; + char debug_name[DEBUG_NAME_MAX_LEN]; + bool explicit_debug_name; }; /* virtgpu_ioctl.c */ diff --git a/drivers/gpu/drm/virtio/virtgpu_ioctl.c b/drivers/gpu/drm/virtio/virtgpu_ioctl.c index 8d13b17c215b..1e2042419f95 100644 --- a/drivers/gpu/drm/virtio/virtgpu_ioctl.c +++ b/drivers/gpu/drm/virtio/virtgpu_ioctl.c @@ -42,12 +42,19 @@ static void virtio_gpu_create_context_locked(struct virtio_gpu_device *vgdev, struct virtio_gpu_fpriv *vfpriv) { - char dbgname[TASK_COMM_LEN]; + if (vfpriv->explicit_debug_name) { + virtio_gpu_cmd_context_create(vgdev, vfpriv->ctx_id, + vfpriv->context_init, + strlen(vfpriv->debug_name), + vfpriv->debug_name); + } else { + char dbgname[TASK_COMM_LEN]; - get_task_comm(dbgname, current); - virtio_gpu_cmd_context_create(vgdev, vfpriv->ctx_id, - vfpriv->context_init, strlen(dbgname), - dbgname); + get_task_comm(dbgname, current); + virtio_gpu_cmd_context_create(vgdev, vfpriv->ctx_id, + vfpriv->context_init, strlen(dbgname), + dbgname); + } vfpriv->context_created = true; } @@ -107,6 +114,9 @@ static int virtio_gpu_getparam_ioctl(struct drm_device *dev, void *data, case VIRTGPU_PARAM_SUPPORTED_CAPSET_IDs: value = vgdev->capset_id_mask; break; + case VIRTGPU_PARAM_EXPLICIT_DEBUG_NAME: + value = vgdev->has_context_init ? 1 : 0; + break; default: return -EINVAL; } @@ -580,7 +590,7 @@ static int virtio_gpu_context_init_ioctl(struct drm_device *dev, return -EINVAL; /* Number of unique parameters supported at this time. */ - if (num_params > 3) + if (num_params > 4) return -EINVAL; ctx_set_params = memdup_user(u64_to_user_ptr(args->ctx_set_params), @@ -642,6 +652,20 @@ static int virtio_gpu_context_init_ioctl(struct drm_device *dev, vfpriv->ring_idx_mask = value; break; + case VIRTGPU_CONTEXT_PARAM_DEBUG_NAME: + if (vfpriv->explicit_debug_name) { + ret = -EINVAL; + goto out_unlock; + } + + ret = strncpy_from_user(vfpriv->debug_name, + u64_to_user_ptr(value), + DEBUG_NAME_MAX_LEN - 1); + if (ret < 0) + goto out_unlock; + + vfpriv->explicit_debug_name = true; + break; default: ret = -EINVAL; goto out_unlock; diff --git a/include/uapi/drm/virtgpu_drm.h b/include/uapi/drm/virtgpu_drm.h index b1d0e56565bc..c2ce71987e9b 100644 --- a/include/uapi/drm/virtgpu_drm.h +++ b/include/uapi/drm/virtgpu_drm.h @@ -97,6 +97,7 @@ struct drm_virtgpu_execbuffer { #define VIRTGPU_PARAM_CROSS_DEVICE 5 /* Cross virtio-device resource sharing */ #define VIRTGPU_PARAM_CONTEXT_INIT 6 /* DRM_VIRTGPU_CONTEXT_INIT */ #define VIRTGPU_PARAM_SUPPORTED_CAPSET_IDs 7 /* Bitmask of supported capability set ids */ +#define VIRTGPU_PARAM_EXPLICIT_DEBUG_NAME 8 /* Ability to set debug name from userspace */ struct drm_virtgpu_getparam { __u64 param; @@ -198,6 +199,7 @@ struct drm_virtgpu_resource_create_blob { #define VIRTGPU_CONTEXT_PARAM_CAPSET_ID 0x0001 #define VIRTGPU_CONTEXT_PARAM_NUM_RINGS 0x0002 #define VIRTGPU_CONTEXT_PARAM_POLL_RINGS_MASK 0x0003 +#define VIRTGPU_CONTEXT_PARAM_DEBUG_NAME 0x0004 struct drm_virtgpu_context_set_param { __u64 param; __u64 value; -- cgit v1.2.3 From 9902cb999e4e913d98e8afe4b36c08e4a793e1ce Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Mon, 6 Nov 2023 10:50:26 -0800 Subject: drm/msm/gem: Add metadata The EXT_external_objects extension is a bit awkward as it doesn't pass explicit modifiers, leaving the importer to guess with incomplete information. In the case of vk (turnip) exporting and gl (freedreno) importing, the "OPTIMAL_TILING_EXT" layout depends on VkImageCreateInfo flags (among other things), which the importer does not know. Which unfortunately leaves us with the need for a metadata back-channel. The contents of the metadata are defined by userspace. The EXT_external_objects extension is only required to work between compatible versions of gl and vk drivers, as defined by device and driver UUIDs. v2: add missing metadata kfree v3: Rework to move copy_from/to_user out from under gem obj lock to avoid angering lockdep about deadlocks against fs-reclaim Signed-off-by: Rob Clark Patchwork: https://patchwork.freedesktop.org/patch/566157/ --- drivers/gpu/drm/msm/msm_drv.c | 92 ++++++++++++++++++++++++++++++++++++++++++- drivers/gpu/drm/msm/msm_gem.c | 1 + drivers/gpu/drm/msm/msm_gem.h | 4 ++ include/uapi/drm/msm_drm.h | 2 + 4 files changed, 98 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index d751d2023f84..75d60aab5780 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -37,9 +37,10 @@ * - 1.9.0 - Add MSM_SUBMIT_FENCE_SN_IN * - 1.10.0 - Add MSM_SUBMIT_BO_NO_IMPLICIT * - 1.11.0 - Add wait boost (MSM_WAIT_FENCE_BOOST, MSM_PREP_BOOST) + * - 1.12.0 - Add MSM_INFO_SET_METADATA and MSM_INFO_GET_METADATA */ #define MSM_VERSION_MAJOR 1 -#define MSM_VERSION_MINOR 11 +#define MSM_VERSION_MINOR 12 #define MSM_VERSION_PATCHLEVEL 0 static void msm_deinit_vram(struct drm_device *ddev); @@ -546,6 +547,85 @@ static int msm_ioctl_gem_info_set_iova(struct drm_device *dev, return msm_gem_set_iova(obj, ctx->aspace, iova); } +static int msm_ioctl_gem_info_set_metadata(struct drm_gem_object *obj, + __user void *metadata, + u32 metadata_size) +{ + struct msm_gem_object *msm_obj = to_msm_bo(obj); + void *buf; + int ret; + + /* Impose a moderate upper bound on metadata size: */ + if (metadata_size > 128) { + return -EOVERFLOW; + } + + /* Use a temporary buf to keep copy_from_user() outside of gem obj lock: */ + buf = memdup_user(metadata, metadata_size); + if (IS_ERR(buf)) + return PTR_ERR(buf); + + ret = msm_gem_lock_interruptible(obj); + if (ret) + goto out; + + msm_obj->metadata = + krealloc(msm_obj->metadata, metadata_size, GFP_KERNEL); + msm_obj->metadata_size = metadata_size; + memcpy(msm_obj->metadata, buf, metadata_size); + + msm_gem_unlock(obj); + +out: + kfree(buf); + + return ret; +} + +static int msm_ioctl_gem_info_get_metadata(struct drm_gem_object *obj, + __user void *metadata, + u32 *metadata_size) +{ + struct msm_gem_object *msm_obj = to_msm_bo(obj); + void *buf; + int ret, len; + + if (!metadata) { + /* + * Querying the size is inherently racey, but + * EXT_external_objects expects the app to confirm + * via device and driver UUIDs that the exporter and + * importer versions match. All we can do from the + * kernel side is check the length under obj lock + * when userspace tries to retrieve the metadata + */ + *metadata_size = msm_obj->metadata_size; + return 0; + } + + ret = msm_gem_lock_interruptible(obj); + if (ret) + return ret; + + /* Avoid copy_to_user() under gem obj lock: */ + len = msm_obj->metadata_size; + buf = kmemdup(msm_obj->metadata, len, GFP_KERNEL); + + msm_gem_unlock(obj); + + if (*metadata_size < len) { + ret = -ETOOSMALL; + } else if (copy_to_user(metadata, buf, len)) { + ret = -EFAULT; + } else { + *metadata_size = len; + } + + kfree(buf); + + return 0; +} + static int msm_ioctl_gem_info(struct drm_device *dev, void *data, struct drm_file *file) { @@ -568,6 +648,8 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data, break; case MSM_INFO_SET_NAME: case MSM_INFO_GET_NAME: + case MSM_INFO_SET_METADATA: + case MSM_INFO_GET_METADATA: break; default: return -EINVAL; @@ -630,6 +712,14 @@ static int msm_ioctl_gem_info(struct drm_device *dev, void *data, ret = -EFAULT; } break; + case MSM_INFO_SET_METADATA: + ret = msm_ioctl_gem_info_set_metadata( + obj, u64_to_user_ptr(args->value), args->len); + break; + case MSM_INFO_GET_METADATA: + ret = msm_ioctl_gem_info_get_metadata( + obj, u64_to_user_ptr(args->value), &args->len); + break; } drm_gem_object_put(obj); diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index db1e748daa75..32303cc8e646 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -1058,6 +1058,7 @@ static void msm_gem_free_object(struct drm_gem_object *obj) drm_gem_object_release(obj); + kfree(msm_obj->metadata); kfree(msm_obj); } diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h index 8ddef5443140..6352bd3cc5ed 100644 --- a/drivers/gpu/drm/msm/msm_gem.h +++ b/drivers/gpu/drm/msm/msm_gem.h @@ -108,6 +108,10 @@ struct msm_gem_object { char name[32]; /* Identifier to print for the debugfs files */ + /* userspace metadata backchannel */ + void *metadata; + u32 metadata_size; + /** * pin_count: Number of times the pages are pinned * diff --git a/include/uapi/drm/msm_drm.h b/include/uapi/drm/msm_drm.h index 6c34272a13fd..6f2a7ad04aa4 100644 --- a/include/uapi/drm/msm_drm.h +++ b/include/uapi/drm/msm_drm.h @@ -139,6 +139,8 @@ struct drm_msm_gem_new { #define MSM_INFO_GET_NAME 0x03 /* get debug name, returned by pointer */ #define MSM_INFO_SET_IOVA 0x04 /* set the iova, passed by value */ #define MSM_INFO_GET_FLAGS 0x05 /* get the MSM_BO_x flags */ +#define MSM_INFO_SET_METADATA 0x06 /* set userspace metadata */ +#define MSM_INFO_GET_METADATA 0x07 /* get userspace metadata */ struct drm_msm_gem_info { __u32 handle; /* in */ -- cgit v1.2.3 From 1088d89e551530a9f5128770d74a1516090f1e41 Mon Sep 17 00:00:00 2001 From: Sarah Walker Date: Wed, 22 Nov 2023 16:34:25 +0000 Subject: drm/imagination/uapi: Add PowerVR driver UAPI Add the UAPI implementation for the PowerVR driver. Changes from v8: - Fixed documentation for unmapping, which previously suggested the size was not used - Corrected license identifier Changes from v7: - Remove prefixes from DRM_PVR_BO_* flags - Improve struct drm_pvr_ioctl_create_hwrt_dataset_args documentation - Remove references to static area carveouts - CREATE_BO ioctl now returns an error if provided size isn't page aligned - Clarify documentation for DRM_PVR_STATIC_DATA_AREA_EOT Changes from v6: - Add padding to struct drm_pvr_dev_query_gpu_info - Improve BYPASS_CACHE flag documentation - Add SUBMIT_JOB_FRAG_CMD_DISABLE_PIXELMERGE flag Changes from v4: - Remove CREATE_ZEROED flag for BO creation (all buffers are now zeroed) Co-developed-by: Frank Binns Signed-off-by: Frank Binns Co-developed-by: Boris Brezillon Signed-off-by: Boris Brezillon Co-developed-by: Matt Coster Signed-off-by: Matt Coster Co-developed-by: Donald Robson Signed-off-by: Donald Robson Signed-off-by: Sarah Walker Reviewed-by: Faith Ekstrand Link: https://lore.kernel.org/r/8c95a3a1d685e2b44d361b95a19eae5a478fb9d1.1700668843.git.donald.robson@imgtec.com Signed-off-by: Maxime Ripard --- MAINTAINERS | 1 + include/uapi/drm/pvr_drm.h | 1297 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1298 insertions(+) create mode 100644 include/uapi/drm/pvr_drm.h (limited to 'include/uapi') diff --git a/MAINTAINERS b/MAINTAINERS index 59c60abf341e..37642011c319 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -10398,6 +10398,7 @@ M: Donald Robson M: Matt Coster S: Supported F: Documentation/devicetree/bindings/gpu/img,powervr.yaml +F: include/uapi/drm/pvr_drm.h IMON SOUNDGRAPH USB IR RECEIVER M: Sean Young diff --git a/include/uapi/drm/pvr_drm.h b/include/uapi/drm/pvr_drm.h new file mode 100644 index 000000000000..1834375390c4 --- /dev/null +++ b/include/uapi/drm/pvr_drm.h @@ -0,0 +1,1297 @@ +/* SPDX-License-Identifier: (GPL-2.0-only WITH Linux-syscall-note) OR MIT */ +/* Copyright (c) 2023 Imagination Technologies Ltd. */ + +#ifndef PVR_DRM_UAPI_H +#define PVR_DRM_UAPI_H + +#include "drm.h" + +#include +#include + +#if defined(__cplusplus) +extern "C" { +#endif + +/** + * DOC: PowerVR UAPI + * + * The PowerVR IOCTL argument structs have a few limitations in place, in + * addition to the standard kernel restrictions: + * + * - All members must be type-aligned. + * - The overall struct must be padded to 64-bit alignment. + * - Explicit padding is almost always required. This takes the form of + * ``_padding_[x]`` members of sufficient size to pad to the next power-of-two + * alignment, where [x] is the offset into the struct in hexadecimal. Arrays + * are never used for alignment. Padding fields must be zeroed; this is + * always checked. + * - Unions may only appear as the last member of a struct. + * - Individual union members may grow in the future. The space between the + * end of a union member and the end of its containing union is considered + * "implicit padding" and must be zeroed. This is always checked. + * + * In addition to the IOCTL argument structs, the PowerVR UAPI makes use of + * DEV_QUERY argument structs. These are used to fetch information about the + * device and runtime. These structs are subject to the same rules set out + * above. + */ + +/** + * struct drm_pvr_obj_array - Container used to pass arrays of objects + * + * It is not unusual to have to extend objects to pass new parameters, and the DRM + * ioctl infrastructure is supporting that by padding ioctl arguments with zeros + * when the data passed by userspace is smaller than the struct defined in the + * drm_ioctl_desc, thus keeping things backward compatible. This type is just + * applying the same concepts to indirect objects passed through arrays referenced + * from the main ioctl arguments structure: the stride basically defines the size + * of the object passed by userspace, which allows the kernel driver to pad with + * zeros when it's smaller than the size of the object it expects. + * + * Use ``DRM_PVR_OBJ_ARRAY()`` to fill object array fields, unless you + * have a very good reason not to. + */ +struct drm_pvr_obj_array { + /** @stride: Stride of object struct. Used for versioning. */ + __u32 stride; + + /** @count: Number of objects in the array. */ + __u32 count; + + /** @array: User pointer to an array of objects. */ + __u64 array; +}; + +/** + * DRM_PVR_OBJ_ARRAY() - Helper macro for filling &struct drm_pvr_obj_array. + * @cnt: Number of elements pointed to py @ptr. + * @ptr: Pointer to start of a C array. + * + * Return: Literal of type &struct drm_pvr_obj_array. + */ +#define DRM_PVR_OBJ_ARRAY(cnt, ptr) \ + { .stride = sizeof((ptr)[0]), .count = (cnt), .array = (__u64)(uintptr_t)(ptr) } + +/** + * DOC: PowerVR IOCTL interface + */ + +/** + * PVR_IOCTL() - Build a PowerVR IOCTL number + * @_ioctl: An incrementing id for this IOCTL. Added to %DRM_COMMAND_BASE. + * @_mode: Must be one of %DRM_IOR, %DRM_IOW or %DRM_IOWR. + * @_data: The type of the args struct passed by this IOCTL. + * + * The struct referred to by @_data must have a ``drm_pvr_ioctl_`` prefix and an + * ``_args suffix``. They are therefore omitted from @_data. + * + * This should only be used to build the constants described below; it should + * never be used to call an IOCTL directly. + * + * Return: An IOCTL number to be passed to ioctl() from userspace. + */ +#define PVR_IOCTL(_ioctl, _mode, _data) \ + _mode(DRM_COMMAND_BASE + (_ioctl), struct drm_pvr_ioctl_##_data##_args) + +#define DRM_IOCTL_PVR_DEV_QUERY PVR_IOCTL(0x00, DRM_IOWR, dev_query) +#define DRM_IOCTL_PVR_CREATE_BO PVR_IOCTL(0x01, DRM_IOWR, create_bo) +#define DRM_IOCTL_PVR_GET_BO_MMAP_OFFSET PVR_IOCTL(0x02, DRM_IOWR, get_bo_mmap_offset) +#define DRM_IOCTL_PVR_CREATE_VM_CONTEXT PVR_IOCTL(0x03, DRM_IOWR, create_vm_context) +#define DRM_IOCTL_PVR_DESTROY_VM_CONTEXT PVR_IOCTL(0x04, DRM_IOW, destroy_vm_context) +#define DRM_IOCTL_PVR_VM_MAP PVR_IOCTL(0x05, DRM_IOW, vm_map) +#define DRM_IOCTL_PVR_VM_UNMAP PVR_IOCTL(0x06, DRM_IOW, vm_unmap) +#define DRM_IOCTL_PVR_CREATE_CONTEXT PVR_IOCTL(0x07, DRM_IOWR, create_context) +#define DRM_IOCTL_PVR_DESTROY_CONTEXT PVR_IOCTL(0x08, DRM_IOW, destroy_context) +#define DRM_IOCTL_PVR_CREATE_FREE_LIST PVR_IOCTL(0x09, DRM_IOWR, create_free_list) +#define DRM_IOCTL_PVR_DESTROY_FREE_LIST PVR_IOCTL(0x0a, DRM_IOW, destroy_free_list) +#define DRM_IOCTL_PVR_CREATE_HWRT_DATASET PVR_IOCTL(0x0b, DRM_IOWR, create_hwrt_dataset) +#define DRM_IOCTL_PVR_DESTROY_HWRT_DATASET PVR_IOCTL(0x0c, DRM_IOW, destroy_hwrt_dataset) +#define DRM_IOCTL_PVR_SUBMIT_JOBS PVR_IOCTL(0x0d, DRM_IOW, submit_jobs) + +/** + * DOC: PowerVR IOCTL DEV_QUERY interface + */ + +/** + * struct drm_pvr_dev_query_gpu_info - Container used to fetch information about + * the graphics processor. + * + * When fetching this type &struct drm_pvr_ioctl_dev_query_args.type must be set + * to %DRM_PVR_DEV_QUERY_GPU_INFO_GET. + */ +struct drm_pvr_dev_query_gpu_info { + /** + * @gpu_id: GPU identifier. + * + * For all currently supported GPUs this is the BVNC encoded as a 64-bit + * value as follows: + * + * +--------+--------+--------+-------+ + * | 63..48 | 47..32 | 31..16 | 15..0 | + * +========+========+========+=======+ + * | B | V | N | C | + * +--------+--------+--------+-------+ + */ + __u64 gpu_id; + + /** + * @num_phantoms: Number of Phantoms present. + */ + __u32 num_phantoms; + + /** @_padding_c: Reserved. This field must be zeroed. */ + __u32 _padding_c; +}; + +/** + * struct drm_pvr_dev_query_runtime_info - Container used to fetch information + * about the graphics runtime. + * + * When fetching this type &struct drm_pvr_ioctl_dev_query_args.type must be set + * to %DRM_PVR_DEV_QUERY_RUNTIME_INFO_GET. + */ +struct drm_pvr_dev_query_runtime_info { + /** + * @free_list_min_pages: Minimum allowed free list size, + * in PM physical pages. + */ + __u64 free_list_min_pages; + + /** + * @free_list_max_pages: Maximum allowed free list size, + * in PM physical pages. + */ + __u64 free_list_max_pages; + + /** + * @common_store_alloc_region_size: Size of the Allocation + * Region within the Common Store used for coefficient and shared + * registers, in dwords. + */ + __u32 common_store_alloc_region_size; + + /** + * @common_store_partition_space_size: Size of the + * Partition Space within the Common Store for output buffers, in + * dwords. + */ + __u32 common_store_partition_space_size; + + /** + * @max_coeffs: Maximum coefficients, in dwords. + */ + __u32 max_coeffs; + + /** + * @cdm_max_local_mem_size_regs: Maximum amount of local + * memory available to a compute kernel, in dwords. + */ + __u32 cdm_max_local_mem_size_regs; +}; + +/** + * struct drm_pvr_dev_query_quirks - Container used to fetch information about + * hardware fixes for which the device may require support in the user mode + * driver. + * + * When fetching this type &struct drm_pvr_ioctl_dev_query_args.type must be set + * to %DRM_PVR_DEV_QUERY_QUIRKS_GET. + */ +struct drm_pvr_dev_query_quirks { + /** + * @quirks: A userspace address for the hardware quirks __u32 array. + * + * The first @musthave_count items in the list are quirks that the + * client must support for this device. If userspace does not support + * all these quirks then functionality is not guaranteed and client + * initialisation must fail. + * The remaining quirks in the list affect userspace and the kernel or + * firmware. They are disabled by default and require userspace to + * opt-in. The opt-in mechanism depends on the quirk. + */ + __u64 quirks; + + /** @count: Length of @quirks (number of __u32). */ + __u16 count; + + /** + * @musthave_count: The number of entries in @quirks that are + * mandatory, starting at index 0. + */ + __u16 musthave_count; + + /** @_padding_c: Reserved. This field must be zeroed. */ + __u32 _padding_c; +}; + +/** + * struct drm_pvr_dev_query_enhancements - Container used to fetch information + * about optional enhancements supported by the device that require support in + * the user mode driver. + * + * When fetching this type &struct drm_pvr_ioctl_dev_query_args.type must be set + * to %DRM_PVR_DEV_ENHANCEMENTS_GET. + */ +struct drm_pvr_dev_query_enhancements { + /** + * @enhancements: A userspace address for the hardware enhancements + * __u32 array. + * + * These enhancements affect userspace and the kernel or firmware. They + * are disabled by default and require userspace to opt-in. The opt-in + * mechanism depends on the enhancement. + */ + __u64 enhancements; + + /** @count: Length of @enhancements (number of __u32). */ + __u16 count; + + /** @_padding_a: Reserved. This field must be zeroed. */ + __u16 _padding_a; + + /** @_padding_c: Reserved. This field must be zeroed. */ + __u32 _padding_c; +}; + +/** + * enum drm_pvr_heap_id - Array index for heap info data returned by + * %DRM_PVR_DEV_QUERY_HEAP_INFO_GET. + * + * For compatibility reasons all indices will be present in the returned array, + * however some heaps may not be present. These are indicated where + * &struct drm_pvr_heap.size is set to zero. + */ +enum drm_pvr_heap_id { + /** @DRM_PVR_HEAP_GENERAL: General purpose heap. */ + DRM_PVR_HEAP_GENERAL = 0, + /** @DRM_PVR_HEAP_PDS_CODE_DATA: PDS code and data heap. */ + DRM_PVR_HEAP_PDS_CODE_DATA, + /** @DRM_PVR_HEAP_USC_CODE: USC code heap. */ + DRM_PVR_HEAP_USC_CODE, + /** @DRM_PVR_HEAP_RGNHDR: Region header heap. Only used if GPU has BRN63142. */ + DRM_PVR_HEAP_RGNHDR, + /** @DRM_PVR_HEAP_VIS_TEST: Visibility test heap. */ + DRM_PVR_HEAP_VIS_TEST, + /** @DRM_PVR_HEAP_TRANSFER_FRAG: Transfer fragment heap. */ + DRM_PVR_HEAP_TRANSFER_FRAG, + + /** + * @DRM_PVR_HEAP_COUNT: The number of heaps returned by + * %DRM_PVR_DEV_QUERY_HEAP_INFO_GET. + * + * More heaps may be added, so this also serves as the copy limit when + * sent by the caller. + */ + DRM_PVR_HEAP_COUNT + /* Please only add additional heaps above DRM_PVR_HEAP_COUNT! */ +}; + +/** + * struct drm_pvr_heap - Container holding information about a single heap. + * + * This will always be fetched as an array. + */ +struct drm_pvr_heap { + /** @base: Base address of heap. */ + __u64 base; + + /** @size: Size of heap, in bytes. Will be 0 if the heap is not present. */ + __u64 size; + + /** @flags: Flags for this heap. Currently always 0. */ + __u32 flags; + + /** @page_size_log2: Log2 of page size. */ + __u32 page_size_log2; +}; + +/** + * struct drm_pvr_dev_query_heap_info - Container used to fetch information + * about heaps supported by the device driver. + * + * Please note all driver-supported heaps will be returned up to &heaps.count. + * Some heaps will not be present in all devices, which will be indicated by + * &struct drm_pvr_heap.size being set to zero. + * + * When fetching this type &struct drm_pvr_ioctl_dev_query_args.type must be set + * to %DRM_PVR_DEV_QUERY_HEAP_INFO_GET. + */ +struct drm_pvr_dev_query_heap_info { + /** + * @heaps: Array of &struct drm_pvr_heap. If pointer is NULL, the count + * and stride will be updated with those known to the driver version, to + * facilitate allocation by the caller. + */ + struct drm_pvr_obj_array heaps; +}; + +/** + * enum drm_pvr_static_data_area_usage - Array index for static data area info + * returned by %DRM_PVR_DEV_QUERY_STATIC_DATA_AREAS_GET. + * + * For compatibility reasons all indices will be present in the returned array, + * however some areas may not be present. These are indicated where + * &struct drm_pvr_static_data_area.size is set to zero. + */ +enum drm_pvr_static_data_area_usage { + /** + * @DRM_PVR_STATIC_DATA_AREA_EOT: End of Tile PDS program code segment. + * + * The End of Tile PDS task runs at completion of a tile during a fragment job, and is + * responsible for emitting the tile to the Pixel Back End. + */ + DRM_PVR_STATIC_DATA_AREA_EOT = 0, + + /** + * @DRM_PVR_STATIC_DATA_AREA_FENCE: MCU fence area, used during cache flush and + * invalidation. + * + * This must point to valid physical memory but the contents otherwise are not used. + */ + DRM_PVR_STATIC_DATA_AREA_FENCE, + + /** + * @DRM_PVR_STATIC_DATA_AREA_VDM_SYNC: VDM sync program. + * + * The VDM sync program is used to synchronise multiple areas of the GPU hardware. + */ + DRM_PVR_STATIC_DATA_AREA_VDM_SYNC, + + /** + * @DRM_PVR_STATIC_DATA_AREA_YUV_CSC: YUV coefficients. + * + * Area contains up to 16 slots with stride of 64 bytes. Each is a 3x4 matrix of u16 fixed + * point numbers, with 1 sign bit, 2 integer bits and 13 fractional bits. + * + * The slots are : + * 0 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY_KHR + * 1 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_IDENTITY_KHR (full range) + * 2 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_IDENTITY_KHR (conformant range) + * 3 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709_KHR (full range) + * 4 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709_KHR (conformant range) + * 5 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_601_KHR (full range) + * 6 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_601_KHR (conformant range) + * 7 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_2020_KHR (full range) + * 8 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_2020_KHR (conformant range) + * 9 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_601_KHR (conformant range, 10 bit) + * 10 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_709_KHR (conformant range, 10 bit) + * 11 = VK_SAMPLER_YCBCR_MODEL_CONVERSION_YCBCR_2020_KHR (conformant range, 10 bit) + * 14 = Identity (biased) + * 15 = Identity + */ + DRM_PVR_STATIC_DATA_AREA_YUV_CSC, +}; + +/** + * struct drm_pvr_static_data_area - Container holding information about a + * single static data area. + * + * This will always be fetched as an array. + */ +struct drm_pvr_static_data_area { + /** + * @area_usage: Usage of static data area. + * See &enum drm_pvr_static_data_area_usage. + */ + __u16 area_usage; + + /** + * @location_heap_id: Array index of heap where this of static data + * area is located. This array is fetched using + * %DRM_PVR_DEV_QUERY_HEAP_INFO_GET. + */ + __u16 location_heap_id; + + /** @size: Size of static data area. Not present if set to zero. */ + __u32 size; + + /** @offset: Offset of static data area from start of heap. */ + __u64 offset; +}; + +/** + * struct drm_pvr_dev_query_static_data_areas - Container used to fetch + * information about the static data areas in heaps supported by the device + * driver. + * + * Please note all driver-supported static data areas will be returned up to + * &static_data_areas.count. Some will not be present for all devices which, + * will be indicated by &struct drm_pvr_static_data_area.size being set to zero. + * + * Further, some heaps will not be present either. See &struct + * drm_pvr_dev_query_heap_info. + * + * When fetching this type &struct drm_pvr_ioctl_dev_query_args.type must be set + * to %DRM_PVR_DEV_QUERY_STATIC_DATA_AREAS_GET. + */ +struct drm_pvr_dev_query_static_data_areas { + /** + * @static_data_areas: Array of &struct drm_pvr_static_data_area. If + * pointer is NULL, the count and stride will be updated with those + * known to the driver version, to facilitate allocation by the caller. + */ + struct drm_pvr_obj_array static_data_areas; +}; + +/** + * enum drm_pvr_dev_query - For use with &drm_pvr_ioctl_dev_query_args.type to + * indicate the type of the receiving container. + * + * Append only. Do not reorder. + */ +enum drm_pvr_dev_query { + /** + * @DRM_PVR_DEV_QUERY_GPU_INFO_GET: The dev query args contain a pointer + * to &struct drm_pvr_dev_query_gpu_info. + */ + DRM_PVR_DEV_QUERY_GPU_INFO_GET = 0, + + /** + * @DRM_PVR_DEV_QUERY_RUNTIME_INFO_GET: The dev query args contain a + * pointer to &struct drm_pvr_dev_query_runtime_info. + */ + DRM_PVR_DEV_QUERY_RUNTIME_INFO_GET, + + /** + * @DRM_PVR_DEV_QUERY_QUIRKS_GET: The dev query args contain a pointer + * to &struct drm_pvr_dev_query_quirks. + */ + DRM_PVR_DEV_QUERY_QUIRKS_GET, + + /** + * @DRM_PVR_DEV_QUERY_ENHANCEMENTS_GET: The dev query args contain a + * pointer to &struct drm_pvr_dev_query_enhancements. + */ + DRM_PVR_DEV_QUERY_ENHANCEMENTS_GET, + + /** + * @DRM_PVR_DEV_QUERY_HEAP_INFO_GET: The dev query args contain a + * pointer to &struct drm_pvr_dev_query_heap_info. + */ + DRM_PVR_DEV_QUERY_HEAP_INFO_GET, + + /** + * @DRM_PVR_DEV_QUERY_STATIC_DATA_AREAS_GET: The dev query args contain + * a pointer to &struct drm_pvr_dev_query_static_data_areas. + */ + DRM_PVR_DEV_QUERY_STATIC_DATA_AREAS_GET, +}; + +/** + * struct drm_pvr_ioctl_dev_query_args - Arguments for %DRM_IOCTL_PVR_DEV_QUERY. + */ +struct drm_pvr_ioctl_dev_query_args { + /** + * @type: Type of query and output struct. See &enum drm_pvr_dev_query. + */ + __u32 type; + + /** + * @size: Size of the receiving struct, see @type. + * + * After a successful call this will be updated to the written byte + * length. + * Can also be used to get the minimum byte length (see @pointer). + * This allows additional fields to be appended to the structs in + * future. + */ + __u32 size; + + /** + * @pointer: Pointer to struct @type. + * + * Must be large enough to contain @size bytes. + * If pointer is NULL, the expected size will be returned in the @size + * field, but no other data will be written. + */ + __u64 pointer; +}; + +/** + * DOC: PowerVR IOCTL CREATE_BO interface + */ + +/** + * DOC: Flags for CREATE_BO + * + * We use "device" to refer to the GPU here because of the ambiguity between CPU and GPU in some + * fonts. + * + * Device mapping options + * :DRM_PVR_BO_BYPASS_DEVICE_CACHE: Specify that device accesses to this memory will bypass the + * cache. This is used for buffers that will either be regularly updated by the CPU (eg free + * lists) or will be accessed only once and therefore isn't worth caching (eg partial render + * buffers). + * By default, the device flushes its memory caches after every job, so this is not normally + * required for coherency. + * :DRM_PVR_BO_PM_FW_PROTECT: Specify that only the Parameter Manager (PM) and/or firmware + * processor should be allowed to access this memory when mapped to the device. It is not + * valid to specify this flag with DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS. + * + * CPU mapping options + * :DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS: Allow userspace to map and access the contents of this + * memory. It is not valid to specify this flag with DRM_PVR_BO_PM_FW_PROTECT. + */ +#define DRM_PVR_BO_BYPASS_DEVICE_CACHE _BITULL(0) +#define DRM_PVR_BO_PM_FW_PROTECT _BITULL(1) +#define DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS _BITULL(2) +/* Bits 3..63 are reserved. */ + +#define DRM_PVR_BO_FLAGS_MASK (DRM_PVR_BO_BYPASS_DEVICE_CACHE | DRM_PVR_BO_PM_FW_PROTECT | \ + DRM_PVR_BO_ALLOW_CPU_USERSPACE_ACCESS) + +/** + * struct drm_pvr_ioctl_create_bo_args - Arguments for %DRM_IOCTL_PVR_CREATE_BO + */ +struct drm_pvr_ioctl_create_bo_args { + /** + * @size: [IN] Size of buffer object to create. This must be page size + * aligned. + */ + __u64 size; + + /** + * @handle: [OUT] GEM handle of the new buffer object for use in + * userspace. + */ + __u32 handle; + + /** @_padding_c: Reserved. This field must be zeroed. */ + __u32 _padding_c; + + /** + * @flags: [IN] Options which will affect the behaviour of this + * creation operation and future mapping operations on the created + * object. This field must be a valid combination of ``DRM_PVR_BO_*`` + * values, with all bits marked as reserved set to zero. + */ + __u64 flags; +}; + +/** + * DOC: PowerVR IOCTL GET_BO_MMAP_OFFSET interface + */ + +/** + * struct drm_pvr_ioctl_get_bo_mmap_offset_args - Arguments for + * %DRM_IOCTL_PVR_GET_BO_MMAP_OFFSET + * + * Like other DRM drivers, the "mmap" IOCTL doesn't actually map any memory. + * Instead, it allocates a fake offset which refers to the specified buffer + * object. This offset can be used with a real mmap call on the DRM device + * itself. + */ +struct drm_pvr_ioctl_get_bo_mmap_offset_args { + /** @handle: [IN] GEM handle of the buffer object to be mapped. */ + __u32 handle; + + /** @_padding_4: Reserved. This field must be zeroed. */ + __u32 _padding_4; + + /** @offset: [OUT] Fake offset to use in the real mmap call. */ + __u64 offset; +}; + +/** + * DOC: PowerVR IOCTL CREATE_VM_CONTEXT and DESTROY_VM_CONTEXT interfaces + */ + +/** + * struct drm_pvr_ioctl_create_vm_context_args - Arguments for + * %DRM_IOCTL_PVR_CREATE_VM_CONTEXT + */ +struct drm_pvr_ioctl_create_vm_context_args { + /** @handle: [OUT] Handle for new VM context. */ + __u32 handle; + + /** @_padding_4: Reserved. This field must be zeroed. */ + __u32 _padding_4; +}; + +/** + * struct drm_pvr_ioctl_destroy_vm_context_args - Arguments for + * %DRM_IOCTL_PVR_DESTROY_VM_CONTEXT + */ +struct drm_pvr_ioctl_destroy_vm_context_args { + /** + * @handle: [IN] Handle for VM context to be destroyed. + */ + __u32 handle; + + /** @_padding_4: Reserved. This field must be zeroed. */ + __u32 _padding_4; +}; + +/** + * DOC: PowerVR IOCTL VM_MAP and VM_UNMAP interfaces + * + * The VM UAPI allows userspace to create buffer object mappings in GPU virtual address space. + * + * The client is responsible for managing GPU address space. It should allocate mappings within + * the heaps returned by %DRM_PVR_DEV_QUERY_HEAP_INFO_GET. + * + * %DRM_IOCTL_PVR_VM_MAP creates a new mapping. The client provides the target virtual address for + * the mapping. Size and offset within the mapped buffer object can be specified, so the client can + * partially map a buffer. + * + * %DRM_IOCTL_PVR_VM_UNMAP removes a mapping. The entire mapping will be removed from GPU address + * space only if the size of the mapping matches that known to the driver. + */ + +/** + * struct drm_pvr_ioctl_vm_map_args - Arguments for %DRM_IOCTL_PVR_VM_MAP. + */ +struct drm_pvr_ioctl_vm_map_args { + /** + * @vm_context_handle: [IN] Handle for VM context for this mapping to + * exist in. + */ + __u32 vm_context_handle; + + /** @flags: [IN] Flags which affect this mapping. Currently always 0. */ + __u32 flags; + + /** + * @device_addr: [IN] Requested device-virtual address for the mapping. + * This must be non-zero and aligned to the device page size for the + * heap containing the requested address. It is an error to specify an + * address which is not contained within one of the heaps returned by + * %DRM_PVR_DEV_QUERY_HEAP_INFO_GET. + */ + __u64 device_addr; + + /** + * @handle: [IN] Handle of the target buffer object. This must be a + * valid handle returned by %DRM_IOCTL_PVR_CREATE_BO. + */ + __u32 handle; + + /** @_padding_14: Reserved. This field must be zeroed. */ + __u32 _padding_14; + + /** + * @offset: [IN] Offset into the target bo from which to begin the + * mapping. + */ + __u64 offset; + + /** + * @size: [IN] Size of the requested mapping. Must be aligned to + * the device page size for the heap containing the requested address, + * as well as the host page size. When added to @device_addr, the + * result must not overflow the heap which contains @device_addr (i.e. + * the range specified by @device_addr and @size must be completely + * contained within a single heap specified by + * %DRM_PVR_DEV_QUERY_HEAP_INFO_GET). + */ + __u64 size; +}; + +/** + * struct drm_pvr_ioctl_vm_unmap_args - Arguments for %DRM_IOCTL_PVR_VM_UNMAP. + */ +struct drm_pvr_ioctl_vm_unmap_args { + /** + * @vm_context_handle: [IN] Handle for VM context that this mapping + * exists in. + */ + __u32 vm_context_handle; + + /** @_padding_4: Reserved. This field must be zeroed. */ + __u32 _padding_4; + + /** + * @device_addr: [IN] Device-virtual address at the start of the target + * mapping. This must be non-zero. + */ + __u64 device_addr; + + /** + * @size: Size in bytes of the target mapping. This must be non-zero. + */ + __u64 size; +}; + +/** + * DOC: PowerVR IOCTL CREATE_CONTEXT and DESTROY_CONTEXT interfaces + */ + +/** + * enum drm_pvr_ctx_priority - Arguments for + * &drm_pvr_ioctl_create_context_args.priority + */ +enum drm_pvr_ctx_priority { + /** @DRM_PVR_CTX_PRIORITY_LOW: Priority below normal. */ + DRM_PVR_CTX_PRIORITY_LOW = -512, + + /** @DRM_PVR_CTX_PRIORITY_NORMAL: Normal priority. */ + DRM_PVR_CTX_PRIORITY_NORMAL = 0, + + /** + * @DRM_PVR_CTX_PRIORITY_HIGH: Priority above normal. + * Note this requires ``CAP_SYS_NICE`` or ``DRM_MASTER``. + */ + DRM_PVR_CTX_PRIORITY_HIGH = 512, +}; + +/** + * enum drm_pvr_ctx_type - Arguments for + * &struct drm_pvr_ioctl_create_context_args.type + */ +enum drm_pvr_ctx_type { + /** + * @DRM_PVR_CTX_TYPE_RENDER: Render context. Use &struct + * drm_pvr_ioctl_create_render_context_args for context creation arguments. + */ + DRM_PVR_CTX_TYPE_RENDER = 0, + + /** + * @DRM_PVR_CTX_TYPE_COMPUTE: Compute context. Use &struct + * drm_pvr_ioctl_create_compute_context_args for context creation arguments. + */ + DRM_PVR_CTX_TYPE_COMPUTE, + + /** + * @DRM_PVR_CTX_TYPE_TRANSFER_FRAG: Transfer context for fragment data masters. Use + * &struct drm_pvr_ioctl_create_transfer_context_args for context creation arguments. + */ + DRM_PVR_CTX_TYPE_TRANSFER_FRAG, +}; + +/** + * struct drm_pvr_ioctl_create_context_args - Arguments for + * %DRM_IOCTL_PVR_CREATE_CONTEXT + */ +struct drm_pvr_ioctl_create_context_args { + /** + * @type: [IN] Type of context to create. + * + * This must be one of the values defined by &enum drm_pvr_ctx_type. + */ + __u32 type; + + /** @flags: [IN] Flags for context. */ + __u32 flags; + + /** + * @priority: [IN] Priority of new context. + * + * This must be one of the values defined by &enum drm_pvr_ctx_priority. + */ + __s32 priority; + + /** @handle: [OUT] Handle for new context. */ + __u32 handle; + + /** + * @static_context_state: [IN] Pointer to static context state stream. + */ + __u64 static_context_state; + + /** + * @static_context_state_len: [IN] Length of static context state, in bytes. + */ + __u32 static_context_state_len; + + /** + * @vm_context_handle: [IN] Handle for VM context that this context is + * associated with. + */ + __u32 vm_context_handle; + + /** + * @callstack_addr: [IN] Address for initial call stack pointer. Only valid + * if @type is %DRM_PVR_CTX_TYPE_RENDER, otherwise must be 0. + */ + __u64 callstack_addr; +}; + +/** + * struct drm_pvr_ioctl_destroy_context_args - Arguments for + * %DRM_IOCTL_PVR_DESTROY_CONTEXT + */ +struct drm_pvr_ioctl_destroy_context_args { + /** + * @handle: [IN] Handle for context to be destroyed. + */ + __u32 handle; + + /** @_padding_4: Reserved. This field must be zeroed. */ + __u32 _padding_4; +}; + +/** + * DOC: PowerVR IOCTL CREATE_FREE_LIST and DESTROY_FREE_LIST interfaces + */ + +/** + * struct drm_pvr_ioctl_create_free_list_args - Arguments for + * %DRM_IOCTL_PVR_CREATE_FREE_LIST + * + * Free list arguments have the following constraints : + * + * - @max_num_pages must be greater than zero. + * - @grow_threshold must be between 0 and 100. + * - @grow_num_pages must be less than or equal to &max_num_pages. + * - @initial_num_pages, @max_num_pages and @grow_num_pages must be multiples + * of 4. + * - When &grow_num_pages is 0, @initial_num_pages must be equal to + * @max_num_pages. + * - When &grow_num_pages is non-zero, @initial_num_pages must be less than + * @max_num_pages. + */ +struct drm_pvr_ioctl_create_free_list_args { + /** + * @free_list_gpu_addr: [IN] Address of GPU mapping of buffer object + * containing memory to be used by free list. + * + * The mapped region of the buffer object must be at least + * @max_num_pages * ``sizeof(__u32)``. + * + * The buffer object must have been created with + * %DRM_PVR_BO_DEVICE_PM_FW_PROTECT set and + * %DRM_PVR_BO_CPU_ALLOW_USERSPACE_ACCESS not set. + */ + __u64 free_list_gpu_addr; + + /** @initial_num_pages: [IN] Pages initially allocated to free list. */ + __u32 initial_num_pages; + + /** @max_num_pages: [IN] Maximum number of pages in free list. */ + __u32 max_num_pages; + + /** @grow_num_pages: [IN] Pages to grow free list by per request. */ + __u32 grow_num_pages; + + /** + * @grow_threshold: [IN] Percentage of FL memory used that should + * trigger a new grow request. + */ + __u32 grow_threshold; + + /** + * @vm_context_handle: [IN] Handle for VM context that the free list buffer + * object is mapped in. + */ + __u32 vm_context_handle; + + /** + * @handle: [OUT] Handle for created free list. + */ + __u32 handle; +}; + +/** + * struct drm_pvr_ioctl_destroy_free_list_args - Arguments for + * %DRM_IOCTL_PVR_DESTROY_FREE_LIST + */ +struct drm_pvr_ioctl_destroy_free_list_args { + /** + * @handle: [IN] Handle for free list to be destroyed. + */ + __u32 handle; + + /** @_padding_4: Reserved. This field must be zeroed. */ + __u32 _padding_4; +}; + +/** + * DOC: PowerVR IOCTL CREATE_HWRT_DATASET and DESTROY_HWRT_DATASET interfaces + */ + +/** + * struct drm_pvr_create_hwrt_geom_data_args - Geometry data arguments used for + * &struct drm_pvr_ioctl_create_hwrt_dataset_args.geom_data_args. + */ +struct drm_pvr_create_hwrt_geom_data_args { + /** @tpc_dev_addr: [IN] Tail pointer cache GPU virtual address. */ + __u64 tpc_dev_addr; + + /** @tpc_size: [IN] Size of TPC, in bytes. */ + __u32 tpc_size; + + /** @tpc_stride: [IN] Stride between layers in TPC, in pages */ + __u32 tpc_stride; + + /** @vheap_table_dev_addr: [IN] VHEAP table GPU virtual address. */ + __u64 vheap_table_dev_addr; + + /** @rtc_dev_addr: [IN] Render Target Cache virtual address. */ + __u64 rtc_dev_addr; +}; + +/** + * struct drm_pvr_create_hwrt_rt_data_args - Render target arguments used for + * &struct drm_pvr_ioctl_create_hwrt_dataset_args.rt_data_args. + */ +struct drm_pvr_create_hwrt_rt_data_args { + /** @pm_mlist_dev_addr: [IN] PM MLIST GPU virtual address. */ + __u64 pm_mlist_dev_addr; + + /** @macrotile_array_dev_addr: [IN] Macrotile array GPU virtual address. */ + __u64 macrotile_array_dev_addr; + + /** @region_header_dev_addr: [IN] Region header array GPU virtual address. */ + __u64 region_header_dev_addr; +}; + +#define PVR_DRM_HWRT_FREE_LIST_LOCAL 0 +#define PVR_DRM_HWRT_FREE_LIST_GLOBAL 1U + +/** + * struct drm_pvr_ioctl_create_hwrt_dataset_args - Arguments for + * %DRM_IOCTL_PVR_CREATE_HWRT_DATASET + */ +struct drm_pvr_ioctl_create_hwrt_dataset_args { + /** @geom_data_args: [IN] Geometry data arguments. */ + struct drm_pvr_create_hwrt_geom_data_args geom_data_args; + + /** + * @rt_data_args: [IN] Array of render target arguments. + * + * Each entry in this array represents a render target in a double buffered + * setup. + */ + struct drm_pvr_create_hwrt_rt_data_args rt_data_args[2]; + + /** + * @free_list_handles: [IN] Array of free list handles. + * + * free_list_handles[PVR_DRM_HWRT_FREE_LIST_LOCAL] must have initial + * size of at least that reported by + * &drm_pvr_dev_query_runtime_info.free_list_min_pages. + */ + __u32 free_list_handles[2]; + + /** @width: [IN] Width in pixels. */ + __u32 width; + + /** @height: [IN] Height in pixels. */ + __u32 height; + + /** @samples: [IN] Number of samples. */ + __u32 samples; + + /** @layers: [IN] Number of layers. */ + __u32 layers; + + /** @isp_merge_lower_x: [IN] Lower X coefficient for triangle merging. */ + __u32 isp_merge_lower_x; + + /** @isp_merge_lower_y: [IN] Lower Y coefficient for triangle merging. */ + __u32 isp_merge_lower_y; + + /** @isp_merge_scale_x: [IN] Scale X coefficient for triangle merging. */ + __u32 isp_merge_scale_x; + + /** @isp_merge_scale_y: [IN] Scale Y coefficient for triangle merging. */ + __u32 isp_merge_scale_y; + + /** @isp_merge_upper_x: [IN] Upper X coefficient for triangle merging. */ + __u32 isp_merge_upper_x; + + /** @isp_merge_upper_y: [IN] Upper Y coefficient for triangle merging. */ + __u32 isp_merge_upper_y; + + /** + * @region_header_size: [IN] Size of region header array. This common field is used by + * both render targets in this data set. + * + * The units for this field differ depending on what version of the simple internal + * parameter format the device uses. If format 2 is in use then this is interpreted as the + * number of region headers. For other formats it is interpreted as the size in dwords. + */ + __u32 region_header_size; + + /** + * @handle: [OUT] Handle for created HWRT dataset. + */ + __u32 handle; +}; + +/** + * struct drm_pvr_ioctl_destroy_hwrt_dataset_args - Arguments for + * %DRM_IOCTL_PVR_DESTROY_HWRT_DATASET + */ +struct drm_pvr_ioctl_destroy_hwrt_dataset_args { + /** + * @handle: [IN] Handle for HWRT dataset to be destroyed. + */ + __u32 handle; + + /** @_padding_4: Reserved. This field must be zeroed. */ + __u32 _padding_4; +}; + +/** + * DOC: PowerVR IOCTL SUBMIT_JOBS interface + */ + +/** + * DOC: Flags for the drm_pvr_sync_op object. + * + * .. c:macro:: DRM_PVR_SYNC_OP_HANDLE_TYPE_MASK + * + * Handle type mask for the drm_pvr_sync_op::flags field. + * + * .. c:macro:: DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ + * + * Indicates the handle passed in drm_pvr_sync_op::handle is a syncobj handle. + * This is the default type. + * + * .. c:macro:: DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_TIMELINE_SYNCOBJ + * + * Indicates the handle passed in drm_pvr_sync_op::handle is a timeline syncobj handle. + * + * .. c:macro:: DRM_PVR_SYNC_OP_FLAG_SIGNAL + * + * Signal operation requested. The out-fence bound to the job will be attached to + * the syncobj whose handle is passed in drm_pvr_sync_op::handle. + * + * .. c:macro:: DRM_PVR_SYNC_OP_FLAG_WAIT + * + * Wait operation requested. The job will wait for this particular syncobj or syncobj + * point to be signaled before being started. + * This is the default operation. + */ +#define DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_MASK 0xf +#define DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_SYNCOBJ 0 +#define DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_TIMELINE_SYNCOBJ 1 +#define DRM_PVR_SYNC_OP_FLAG_SIGNAL _BITULL(31) +#define DRM_PVR_SYNC_OP_FLAG_WAIT 0 + +#define DRM_PVR_SYNC_OP_FLAGS_MASK (DRM_PVR_SYNC_OP_FLAG_HANDLE_TYPE_MASK | \ + DRM_PVR_SYNC_OP_FLAG_SIGNAL) + +/** + * struct drm_pvr_sync_op - Object describing a sync operation + */ +struct drm_pvr_sync_op { + /** @handle: Handle of sync object. */ + __u32 handle; + + /** @flags: Combination of ``DRM_PVR_SYNC_OP_FLAG_`` flags. */ + __u32 flags; + + /** @value: Timeline value for this drm_syncobj. MBZ for a binary syncobj. */ + __u64 value; +}; + +/** + * DOC: Flags for SUBMIT_JOB ioctl geometry command. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_GEOM_CMD_FIRST + * + * Indicates if this the first command to be issued for a render. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_GEOM_CMD_LAST + * + * Indicates if this the last command to be issued for a render. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_GEOM_CMD_SINGLE_CORE + * + * Forces to use single core in a multi core device. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_GEOM_CMD_FLAGS_MASK + * + * Logical OR of all the geometry cmd flags. + */ +#define DRM_PVR_SUBMIT_JOB_GEOM_CMD_FIRST _BITULL(0) +#define DRM_PVR_SUBMIT_JOB_GEOM_CMD_LAST _BITULL(1) +#define DRM_PVR_SUBMIT_JOB_GEOM_CMD_SINGLE_CORE _BITULL(2) +#define DRM_PVR_SUBMIT_JOB_GEOM_CMD_FLAGS_MASK \ + (DRM_PVR_SUBMIT_JOB_GEOM_CMD_FIRST | \ + DRM_PVR_SUBMIT_JOB_GEOM_CMD_LAST | \ + DRM_PVR_SUBMIT_JOB_GEOM_CMD_SINGLE_CORE) + +/** + * DOC: Flags for SUBMIT_JOB ioctl fragment command. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_FRAG_CMD_SINGLE_CORE + * + * Use single core in a multi core setup. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_FRAG_CMD_DEPTHBUFFER + * + * Indicates whether a depth buffer is present. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_FRAG_CMD_STENCILBUFFER + * + * Indicates whether a stencil buffer is present. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_FRAG_CMD_PREVENT_CDM_OVERLAP + * + * Disallow compute overlapped with this render. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_FRAG_CMD_GET_VIS_RESULTS + * + * Indicates whether this render produces visibility results. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_FRAG_CMD_SCRATCHBUFFER + * + * Indicates whether partial renders write to a scratch buffer instead of + * the final surface. It also forces the full screen copy expected to be + * present on the last render after all partial renders have completed. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_FRAG_CMD_DISABLE_PIXELMERGE + * + * Disable pixel merging for this render. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_FRAG_CMD_FLAGS_MASK + * + * Logical OR of all the fragment cmd flags. + */ +#define DRM_PVR_SUBMIT_JOB_FRAG_CMD_SINGLE_CORE _BITULL(0) +#define DRM_PVR_SUBMIT_JOB_FRAG_CMD_DEPTHBUFFER _BITULL(1) +#define DRM_PVR_SUBMIT_JOB_FRAG_CMD_STENCILBUFFER _BITULL(2) +#define DRM_PVR_SUBMIT_JOB_FRAG_CMD_PREVENT_CDM_OVERLAP _BITULL(3) +#define DRM_PVR_SUBMIT_JOB_FRAG_CMD_SCRATCHBUFFER _BITULL(4) +#define DRM_PVR_SUBMIT_JOB_FRAG_CMD_GET_VIS_RESULTS _BITULL(5) +#define DRM_PVR_SUBMIT_JOB_FRAG_CMD_PARTIAL_RENDER _BITULL(6) +#define DRM_PVR_SUBMIT_JOB_FRAG_CMD_DISABLE_PIXELMERGE _BITULL(7) +#define DRM_PVR_SUBMIT_JOB_FRAG_CMD_FLAGS_MASK \ + (DRM_PVR_SUBMIT_JOB_FRAG_CMD_SINGLE_CORE | \ + DRM_PVR_SUBMIT_JOB_FRAG_CMD_DEPTHBUFFER | \ + DRM_PVR_SUBMIT_JOB_FRAG_CMD_STENCILBUFFER | \ + DRM_PVR_SUBMIT_JOB_FRAG_CMD_PREVENT_CDM_OVERLAP | \ + DRM_PVR_SUBMIT_JOB_FRAG_CMD_SCRATCHBUFFER | \ + DRM_PVR_SUBMIT_JOB_FRAG_CMD_GET_VIS_RESULTS | \ + DRM_PVR_SUBMIT_JOB_FRAG_CMD_PARTIAL_RENDER | \ + DRM_PVR_SUBMIT_JOB_FRAG_CMD_DISABLE_PIXELMERGE) + +/** + * DOC: Flags for SUBMIT_JOB ioctl compute command. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_PREVENT_ALL_OVERLAP + * + * Disallow other jobs overlapped with this compute. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_SINGLE_CORE + * + * Forces to use single core in a multi core device. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_FLAGS_MASK + * + * Logical OR of all the compute cmd flags. + */ +#define DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_PREVENT_ALL_OVERLAP _BITULL(0) +#define DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_SINGLE_CORE _BITULL(1) +#define DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_FLAGS_MASK \ + (DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_PREVENT_ALL_OVERLAP | \ + DRM_PVR_SUBMIT_JOB_COMPUTE_CMD_SINGLE_CORE) + +/** + * DOC: Flags for SUBMIT_JOB ioctl transfer command. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_TRANSFER_CMD_SINGLE_CORE + * + * Forces job to use a single core in a multi core device. + * + * .. c:macro:: DRM_PVR_SUBMIT_JOB_TRANSFER_CMD_FLAGS_MASK + * + * Logical OR of all the transfer cmd flags. + */ +#define DRM_PVR_SUBMIT_JOB_TRANSFER_CMD_SINGLE_CORE _BITULL(0) + +#define DRM_PVR_SUBMIT_JOB_TRANSFER_CMD_FLAGS_MASK \ + DRM_PVR_SUBMIT_JOB_TRANSFER_CMD_SINGLE_CORE + +/** + * enum drm_pvr_job_type - Arguments for &struct drm_pvr_job.job_type + */ +enum drm_pvr_job_type { + /** @DRM_PVR_JOB_TYPE_GEOMETRY: Job type is geometry. */ + DRM_PVR_JOB_TYPE_GEOMETRY = 0, + + /** @DRM_PVR_JOB_TYPE_FRAGMENT: Job type is fragment. */ + DRM_PVR_JOB_TYPE_FRAGMENT, + + /** @DRM_PVR_JOB_TYPE_COMPUTE: Job type is compute. */ + DRM_PVR_JOB_TYPE_COMPUTE, + + /** @DRM_PVR_JOB_TYPE_TRANSFER_FRAG: Job type is a fragment transfer. */ + DRM_PVR_JOB_TYPE_TRANSFER_FRAG, +}; + +/** + * struct drm_pvr_hwrt_data_ref - Reference HWRT data + */ +struct drm_pvr_hwrt_data_ref { + /** @set_handle: HWRT data set handle. */ + __u32 set_handle; + + /** @data_index: Index of the HWRT data inside the data set. */ + __u32 data_index; +}; + +/** + * struct drm_pvr_job - Job arguments passed to the %DRM_IOCTL_PVR_SUBMIT_JOBS ioctl + */ +struct drm_pvr_job { + /** + * @type: [IN] Type of job being submitted + * + * This must be one of the values defined by &enum drm_pvr_job_type. + */ + __u32 type; + + /** + * @context_handle: [IN] Context handle. + * + * When @job_type is %DRM_PVR_JOB_TYPE_RENDER, %DRM_PVR_JOB_TYPE_COMPUTE or + * %DRM_PVR_JOB_TYPE_TRANSFER_FRAG, this must be a valid handle returned by + * %DRM_IOCTL_PVR_CREATE_CONTEXT. The type of context must be compatible + * with the type of job being submitted. + * + * When @job_type is %DRM_PVR_JOB_TYPE_NULL, this must be zero. + */ + __u32 context_handle; + + /** + * @flags: [IN] Flags for command. + * + * Those are job-dependent. See all ``DRM_PVR_SUBMIT_JOB_*``. + */ + __u32 flags; + + /** + * @cmd_stream_len: [IN] Length of command stream, in bytes. + */ + __u32 cmd_stream_len; + + /** + * @cmd_stream: [IN] Pointer to command stream for command. + * + * The command stream must be u64-aligned. + */ + __u64 cmd_stream; + + /** @sync_ops: [IN] Fragment sync operations. */ + struct drm_pvr_obj_array sync_ops; + + /** + * @hwrt: [IN] HWRT data used by render jobs (geometry or fragment). + * + * Must be zero for non-render jobs. + */ + struct drm_pvr_hwrt_data_ref hwrt; +}; + +/** + * struct drm_pvr_ioctl_submit_jobs_args - Arguments for %DRM_IOCTL_PVR_SUBMIT_JOB + * + * If the syscall returns an error it is important to check the value of + * @jobs.count. This indicates the index into @jobs.array where the + * error occurred. + */ +struct drm_pvr_ioctl_submit_jobs_args { + /** @jobs: [IN] Array of jobs to submit. */ + struct drm_pvr_obj_array jobs; +}; + +#if defined(__cplusplus) +} +#endif + +#endif /* PVR_DRM_UAPI_H */ -- cgit v1.2.3 From 4b4af74ab9719d17538a97f43137e93296ec7437 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Wed, 22 Nov 2023 13:19:39 -0300 Subject: drm: allow DRM_MODE_PAGE_FLIP_ASYNC for atomic commits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If the driver supports it, allow user-space to supply the DRM_MODE_PAGE_FLIP_ASYNC flag to request an async page-flip. Set drm_crtc_state.async_flip accordingly. Document that drivers will reject atomic commits if an async flip isn't possible. This allows user-space to fall back to something else. For instance, Xorg falls back to a blit. Another option is to wait as close to the next vblank as possible before performing the page-flip to reduce latency. Signed-off-by: Simon Ser Reviewed-by: Alex Deucher Co-developed-by: André Almeida Signed-off-by: André Almeida Link: https://patchwork.freedesktop.org/patch/msgid/20231122161941.320564-3-andrealmeid@igalia.com --- drivers/gpu/drm/drm_atomic_uapi.c | 25 ++++++++++++++++++++++--- include/uapi/drm/drm_mode.h | 9 +++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index 52494642b095..116407002269 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -1368,6 +1368,18 @@ static void complete_signaling(struct drm_device *dev, kfree(fence_state); } +static void +set_async_flip(struct drm_atomic_state *state) +{ + struct drm_crtc *crtc; + struct drm_crtc_state *crtc_state; + int i; + + for_each_new_crtc_in_state(state, crtc, crtc_state, i) { + crtc_state->async_flip = true; + } +} + int drm_mode_atomic_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv) { @@ -1409,9 +1421,13 @@ int drm_mode_atomic_ioctl(struct drm_device *dev, } if (arg->flags & DRM_MODE_PAGE_FLIP_ASYNC) { - drm_dbg_atomic(dev, - "commit failed: invalid flag DRM_MODE_PAGE_FLIP_ASYNC\n"); - return -EINVAL; + if (!dev->mode_config.async_page_flip) { + drm_dbg_atomic(dev, + "commit failed: DRM_MODE_PAGE_FLIP_ASYNC not supported\n"); + return -EINVAL; + } + + async_flip = true; } /* can't test and expect an event at the same time. */ @@ -1514,6 +1530,9 @@ retry: if (ret) goto out; + if (arg->flags & DRM_MODE_PAGE_FLIP_ASYNC) + set_async_flip(state); + if (arg->flags & DRM_MODE_ATOMIC_TEST_ONLY) { ret = drm_atomic_check_only(state); } else if (arg->flags & DRM_MODE_ATOMIC_NONBLOCK) { diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index 09e7a471ee30..95630f170110 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -957,6 +957,15 @@ struct hdr_output_metadata { * Request that the page-flip is performed as soon as possible, ie. with no * delay due to waiting for vblank. This may cause tearing to be visible on * the screen. + * + * When used with atomic uAPI, the driver will return an error if the hardware + * doesn't support performing an asynchronous page-flip for this update. + * User-space should handle this, e.g. by falling back to a regular page-flip. + * + * Note, some hardware might need to perform one last synchronous page-flip + * before being able to switch to asynchronous page-flips. As an exception, + * the driver will return success even though that first page-flip is not + * asynchronous. */ #define DRM_MODE_PAGE_FLIP_ASYNC 0x02 #define DRM_MODE_PAGE_FLIP_TARGET_ABSOLUTE 0x4 -- cgit v1.2.3 From e4d983acffff270ccee417445a69b9ed198658b1 Mon Sep 17 00:00:00 2001 From: Simon Ser Date: Wed, 22 Nov 2023 13:19:40 -0300 Subject: drm: introduce DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This new kernel capability indicates whether async page-flips are supported via the atomic uAPI. DRM clients can use it to check for support before feeding DRM_MODE_PAGE_FLIP_ASYNC to the kernel. Make it clear that DRM_CAP_ASYNC_PAGE_FLIP is for legacy uAPI only. Signed-off-by: Simon Ser Reviewed-by: André Almeida Reviewed-by: Alex Deucher Signed-off-by: André Almeida Signed-off-by: Simon Ser Link: https://patchwork.freedesktop.org/patch/msgid/20231122161941.320564-4-andrealmeid@igalia.com --- drivers/gpu/drm/drm_ioctl.c | 4 ++++ include/uapi/drm/drm.h | 10 +++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c index 44fda68c28ae..f461ed862480 100644 --- a/drivers/gpu/drm/drm_ioctl.c +++ b/drivers/gpu/drm/drm_ioctl.c @@ -301,6 +301,10 @@ static int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_ case DRM_CAP_CRTC_IN_VBLANK_EVENT: req->value = 1; break; + case DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP: + req->value = drm_core_check_feature(dev, DRIVER_ATOMIC) && + dev->mode_config.async_page_flip; + break; default: return -EINVAL; } diff --git a/include/uapi/drm/drm.h b/include/uapi/drm/drm.h index 8662b5aeea0c..796de831f4a0 100644 --- a/include/uapi/drm/drm.h +++ b/include/uapi/drm/drm.h @@ -713,7 +713,8 @@ struct drm_gem_open { /** * DRM_CAP_ASYNC_PAGE_FLIP * - * If set to 1, the driver supports &DRM_MODE_PAGE_FLIP_ASYNC. + * If set to 1, the driver supports &DRM_MODE_PAGE_FLIP_ASYNC for legacy + * page-flips. */ #define DRM_CAP_ASYNC_PAGE_FLIP 0x7 /** @@ -773,6 +774,13 @@ struct drm_gem_open { * :ref:`drm_sync_objects`. */ #define DRM_CAP_SYNCOBJ_TIMELINE 0x14 +/** + * DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP + * + * If set to 1, the driver supports &DRM_MODE_PAGE_FLIP_ASYNC for atomic + * commits. + */ +#define DRM_CAP_ATOMIC_ASYNC_PAGE_FLIP 0x15 /* DRM_IOCTL_GET_CAP ioctl argument type */ struct drm_get_cap { -- cgit v1.2.3 From 9724ed6c1b1212d138e63f5e80647dc8b6b86696 Mon Sep 17 00:00:00 2001 From: Zack Rusin Date: Mon, 23 Oct 2023 09:46:12 +0200 Subject: drm: Introduce DRM_CLIENT_CAP_CURSOR_PLANE_HOTSPOT Virtualized drivers place additional restrictions on the cursor plane which breaks the contract of universal planes. To allow atomic modesettings with virtualized drivers the clients need to advertise that they're capable of dealing with those extra restrictions. To do that introduce DRM_CLIENT_CAP_CURSOR_PLANE_HOTSPOT which lets DRM know that the client is aware of and capable of dealing with the extra restrictions on the virtual cursor plane. Setting this option to true makes DRM expose the cursor plane on virtualized drivers. The userspace is expected to set the hotspots and handle mouse events on that plane. Signed-off-by: Zack Rusin Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Thomas Zimmermann Cc: David Airlie Cc: Daniel Vetter Cc: dri-devel@lists.freedesktop.org Acked-by: Pekka Paalanen Reviewed-by: Javier Martinez Canillas Signed-off-by: Javier Martinez Canillas Link: https://patchwork.freedesktop.org/patch/msgid/20231023074613.41327-9-aesteve@redhat.com --- drivers/gpu/drm/drm_ioctl.c | 9 +++++++++ include/uapi/drm/drm.h | 25 +++++++++++++++++++++++++ 2 files changed, 34 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c index f461ed862480..ea303386e688 100644 --- a/drivers/gpu/drm/drm_ioctl.c +++ b/drivers/gpu/drm/drm_ioctl.c @@ -365,6 +365,15 @@ drm_setclientcap(struct drm_device *dev, void *data, struct drm_file *file_priv) return -EINVAL; file_priv->writeback_connectors = req->value; break; + case DRM_CLIENT_CAP_CURSOR_PLANE_HOTSPOT: + if (!drm_core_check_feature(dev, DRIVER_CURSOR_HOTSPOT)) + return -EOPNOTSUPP; + if (!file_priv->atomic) + return -EINVAL; + if (req->value > 1) + return -EINVAL; + file_priv->supports_virtualized_cursor_plane = req->value; + break; default: return -EINVAL; } diff --git a/include/uapi/drm/drm.h b/include/uapi/drm/drm.h index 796de831f4a0..91943e5ce010 100644 --- a/include/uapi/drm/drm.h +++ b/include/uapi/drm/drm.h @@ -850,6 +850,31 @@ struct drm_get_cap { */ #define DRM_CLIENT_CAP_WRITEBACK_CONNECTORS 5 +/** + * DRM_CLIENT_CAP_CURSOR_PLANE_HOTSPOT + * + * Drivers for para-virtualized hardware (e.g. vmwgfx, qxl, virtio and + * virtualbox) have additional restrictions for cursor planes (thus + * making cursor planes on those drivers not truly universal,) e.g. + * they need cursor planes to act like one would expect from a mouse + * cursor and have correctly set hotspot properties. + * If this client cap is not set the DRM core will hide cursor plane on + * those virtualized drivers because not setting it implies that the + * client is not capable of dealing with those extra restictions. + * Clients which do set cursor hotspot and treat the cursor plane + * like a mouse cursor should set this property. + * The client must enable &DRM_CLIENT_CAP_ATOMIC first. + * + * Setting this property on drivers which do not special case + * cursor planes (i.e. non-virtualized drivers) will return + * EOPNOTSUPP, which can be used by userspace to gauge + * requirements of the hardware/drivers they're running on. + * + * This capability is always supported for atomic-capable virtualized + * drivers starting from kernel version 6.6. + */ +#define DRM_CLIENT_CAP_CURSOR_PLANE_HOTSPOT 6 + /* DRM_IOCTL_SET_CLIENT_CAP ioctl argument type */ struct drm_set_client_cap { __u64 capability; -- cgit v1.2.3 From 3cc808e3239cf566b3d3b15cf2beee066b60f241 Mon Sep 17 00:00:00 2001 From: Donald Robson Date: Tue, 28 Nov 2023 17:35:07 +0000 Subject: drm/imagination: Numerous documentation fixes. Some reported by Stephen Rothwell. The rest were found by running the kernel-doc build script. Some indentation fixes. Reported-by: Stephen Rothwell Reported-by: kernel test robot Closes: https://lore.kernel.org/oe-kbuild-all/202311241526.Y2WZeUau-lkp@intel.com/ Signed-off-by: Donald Robson Signed-off-by: Maxime Ripard Link: https://patchwork.freedesktop.org/patch/msgid/20231128173507.95119-1-donald.robson@imgtec.com --- Documentation/gpu/imagination/index.rst | 2 +- Documentation/gpu/imagination/uapi.rst | 5 +---- drivers/gpu/drm/imagination/pvr_cccb.h | 1 + drivers/gpu/drm/imagination/pvr_device.h | 25 ++++++++++++++++++------- drivers/gpu/drm/imagination/pvr_fw.h | 3 ++- drivers/gpu/drm/imagination/pvr_fw_info.h | 8 ++++---- drivers/gpu/drm/imagination/pvr_hwrt.h | 1 + drivers/gpu/drm/imagination/pvr_job.c | 4 +--- drivers/gpu/drm/imagination/pvr_mmu.c | 3 ++- drivers/gpu/drm/imagination/pvr_queue.h | 4 ++-- drivers/gpu/drm/imagination/pvr_vm.c | 2 +- include/uapi/drm/pvr_drm.h | 10 ++++------ 12 files changed, 38 insertions(+), 30 deletions(-) (limited to 'include/uapi') diff --git a/Documentation/gpu/imagination/index.rst b/Documentation/gpu/imagination/index.rst index dc9579e758c3..0c1e247cea41 100644 --- a/Documentation/gpu/imagination/index.rst +++ b/Documentation/gpu/imagination/index.rst @@ -3,7 +3,7 @@ drm/imagination PowerVR Graphics Driver ======================================= .. kernel-doc:: drivers/gpu/drm/imagination/pvr_drv.c - :doc: PowerVR Graphics Driver + :doc: PowerVR (Series 6 and later) and IMG Graphics Driver Contents ======== diff --git a/Documentation/gpu/imagination/uapi.rst b/Documentation/gpu/imagination/uapi.rst index 2227ea7e6222..7502413d0a93 100644 --- a/Documentation/gpu/imagination/uapi.rst +++ b/Documentation/gpu/imagination/uapi.rst @@ -45,9 +45,6 @@ DEV_QUERY drm_pvr_heap drm_pvr_dev_query_heap_info -.. kernel-doc:: include/uapi/drm/pvr_drm.h - :doc: Flags for DRM_PVR_DEV_QUERY_HEAP_INFO_GET. - .. kernel-doc:: include/uapi/drm/pvr_drm.h :identifiers: drm_pvr_static_data_area_usage drm_pvr_static_data_area @@ -121,7 +118,7 @@ CREATE_FREE_LIST and DESTROY_FREE_LIST :identifiers: drm_pvr_ioctl_destroy_free_list_args CREATE_HWRT_DATASET and DESTROY_HWRT_DATASET --------------------------------------- +-------------------------------------------- .. kernel-doc:: include/uapi/drm/pvr_drm.h :doc: PowerVR IOCTL CREATE_HWRT_DATASET and DESTROY_HWRT_DATASET interfaces diff --git a/drivers/gpu/drm/imagination/pvr_cccb.h b/drivers/gpu/drm/imagination/pvr_cccb.h index f35b3d4c9575..943fe8f2c963 100644 --- a/drivers/gpu/drm/imagination/pvr_cccb.h +++ b/drivers/gpu/drm/imagination/pvr_cccb.h @@ -86,6 +86,7 @@ pvr_cccb_get_size_of_cmd_with_hdr(u32 cmd_size) /** * pvr_cccb_cmdseq_can_fit() - Check if a command sequence can fit in the CCCB. + * @pvr_cccb: Target Client CCB. * @size: Command sequence size. * * Returns: diff --git a/drivers/gpu/drm/imagination/pvr_device.h b/drivers/gpu/drm/imagination/pvr_device.h index e07655fc65e8..2ca7e535799f 100644 --- a/drivers/gpu/drm/imagination/pvr_device.h +++ b/drivers/gpu/drm/imagination/pvr_device.h @@ -203,17 +203,29 @@ struct pvr_device { struct mutex lock; } queues; + /** + * @watchdog: Watchdog for communications with firmware. + */ struct { /** @work: Work item for watchdog callback. */ struct delayed_work work; - /** @old_kccb_cmds_executed: KCCB command execution count at last watchdog poll. */ + /** + * @old_kccb_cmds_executed: KCCB command execution count at last + * watchdog poll. + */ u32 old_kccb_cmds_executed; - /** @kccb_stall_count: Number of watchdog polls KCCB has been stalled for. */ + /** + * @kccb_stall_count: Number of watchdog polls KCCB has been + * stalled for. + */ u32 kccb_stall_count; } watchdog; + /** + * @kccb: Circular buffer for communications with firmware. + */ struct { /** @ccb: Kernel CCB. */ struct pvr_ccb ccb; @@ -225,8 +237,8 @@ struct pvr_device { struct pvr_fw_object *rtn_obj; /** - * @rtn: Pointer to CPU mapping of KCCB return slots. Must be accessed by - * READ_ONCE()/WRITE_ONCE(). + * @rtn: Pointer to CPU mapping of KCCB return slots. Must be + * accessed by READ_ONCE()/WRITE_ONCE(). */ u32 *rtn; @@ -293,14 +305,13 @@ struct pvr_file { /** * @pvr_dev: A reference to the powervr-specific wrapper for the - * associated device. Saves on repeated calls to - * to_pvr_device(). + * associated device. Saves on repeated calls to to_pvr_device(). */ struct pvr_device *pvr_dev; /** * @ctx_handles: Array of contexts belonging to this file. Array members - * are of type "struct pvr_context *". + * are of type "struct pvr_context *". * * This array is used to allocate handles returned to userspace. */ diff --git a/drivers/gpu/drm/imagination/pvr_fw.h b/drivers/gpu/drm/imagination/pvr_fw.h index 5cd3ef08d82b..b7966bd574a9 100644 --- a/drivers/gpu/drm/imagination/pvr_fw.h +++ b/drivers/gpu/drm/imagination/pvr_fw.h @@ -481,7 +481,8 @@ pvr_fw_object_unmap_and_destroy(struct pvr_fw_object *fw_obj) } /** - * pvr_fw_get_dma_addr() - Get DMA address for given offset in firmware object + * pvr_fw_object_get_dma_addr() - Get DMA address for given offset in firmware + * object. * @fw_obj: Pointer to object to lookup address in. * @offset: Offset within object to lookup address at. * @dma_addr_out: Pointer to location to store DMA address. diff --git a/drivers/gpu/drm/imagination/pvr_fw_info.h b/drivers/gpu/drm/imagination/pvr_fw_info.h index ad5d44a3067a..c3639440610e 100644 --- a/drivers/gpu/drm/imagination/pvr_fw_info.h +++ b/drivers/gpu/drm/imagination/pvr_fw_info.h @@ -122,13 +122,13 @@ struct pvr_fw_layout_entry { * struct pvr_fw_device_info_header - Device information header. */ struct pvr_fw_device_info_header { - /* BRN Mask size (in u64s). */ + /** @brn_mask_size: BRN mask size (in u64s). */ u64 brn_mask_size; - /* ERN Mask size (in u64s). */ + /** @ern_mask_size: ERN mask size (in u64s). */ u64 ern_mask_size; - /* Feature Mask size (in u64s). */ + /** @feature_mask_size: Feature mask size (in u64s). */ u64 feature_mask_size; - /* Feature Parameter size (in u64s). */ + /** @feature_param_size: Feature parameter size (in u64s). */ u64 feature_param_size; }; diff --git a/drivers/gpu/drm/imagination/pvr_hwrt.h b/drivers/gpu/drm/imagination/pvr_hwrt.h index 76992948d047..676070b20c3b 100644 --- a/drivers/gpu/drm/imagination/pvr_hwrt.h +++ b/drivers/gpu/drm/imagination/pvr_hwrt.h @@ -64,6 +64,7 @@ struct pvr_hwrt_dataset { /** @common_fw_obj: FW object representing common FW-side structure. */ struct pvr_fw_object *common_fw_obj; + /** @common: Common HWRT data. */ struct rogue_fwif_hwrtdata_common common; /** @data: HWRT data structures belonging to this set. */ diff --git a/drivers/gpu/drm/imagination/pvr_job.c b/drivers/gpu/drm/imagination/pvr_job.c index 9d0812710295..04139da6c04d 100644 --- a/drivers/gpu/drm/imagination/pvr_job.c +++ b/drivers/gpu/drm/imagination/pvr_job.c @@ -378,7 +378,7 @@ prepare_job_syncs(struct pvr_file *pvr_file, /** * prepare_job_syncs_for_each() - Prepare all sync objects for an array of jobs. - * @file: PowerVR file. + * @pvr_file: PowerVR file. * @job_data: Array of precreated jobs and their sync_ops. * @job_count: Number of jobs. * @signal_array: xarray to receive signal sync objects. @@ -696,8 +696,6 @@ pvr_jobs_link_geom_frag(struct pvr_job_data *job_data, u32 *job_count) * @pvr_dev: Target PowerVR device. * @pvr_file: Pointer to PowerVR file structure. * @args: Ioctl args. - * @job_count: Number of jobs in @jobs_args. On error this will be updated - * with the index into @jobs_args where the error occurred. * * This initial implementation is entirely synchronous; on return the GPU will * be idle. This will not be the case for future implementations. diff --git a/drivers/gpu/drm/imagination/pvr_mmu.c b/drivers/gpu/drm/imagination/pvr_mmu.c index b71d30e5f380..c8562bfc0dcd 100644 --- a/drivers/gpu/drm/imagination/pvr_mmu.c +++ b/drivers/gpu/drm/imagination/pvr_mmu.c @@ -335,8 +335,9 @@ pvr_mmu_backing_page_fini(struct pvr_mmu_backing_page *page) /** * pvr_mmu_backing_page_sync() - Flush a MMU backing page from the CPU to the - * device. + * device. * @page: Target backing page. + * @flags: MMU flush flags. Must be one of %PVR_MMU_SYNC_LEVEL_*_FLAGS. * * .. caution:: * diff --git a/drivers/gpu/drm/imagination/pvr_queue.h b/drivers/gpu/drm/imagination/pvr_queue.h index b5ce2c742150..e06ced69302f 100644 --- a/drivers/gpu/drm/imagination/pvr_queue.h +++ b/drivers/gpu/drm/imagination/pvr_queue.h @@ -50,7 +50,7 @@ struct pvr_queue_cccb_fence_ctx { */ struct pvr_job *job; - /** @lock: Lock protecting access to the job object. */ + /** @job_lock: Lock protecting access to the job object. */ struct mutex job_lock; }; @@ -114,7 +114,7 @@ struct pvr_queue { } timeline_ufo; /** - * last_queued_job_scheduled_fence: The scheduled fence of the last + * @last_queued_job_scheduled_fence: The scheduled fence of the last * job queued to this queue. * * We use it to insert frag -> geom dependencies when issuing combined diff --git a/drivers/gpu/drm/imagination/pvr_vm.c b/drivers/gpu/drm/imagination/pvr_vm.c index e0d74d9a6190..2aab53594a77 100644 --- a/drivers/gpu/drm/imagination/pvr_vm.c +++ b/drivers/gpu/drm/imagination/pvr_vm.c @@ -42,7 +42,7 @@ struct pvr_vm_context { /** @mmu_ctx: The context for binding to physical memory. */ struct pvr_mmu_context *mmu_ctx; - /** @gpuva_mgr: GPUVA manager object associated with this context. */ + /** @gpuvm_mgr: GPUVM object associated with this context. */ struct drm_gpuvm gpuvm_mgr; /** @lock: Global lock on this VM. */ diff --git a/include/uapi/drm/pvr_drm.h b/include/uapi/drm/pvr_drm.h index 1834375390c4..ccf6c2112468 100644 --- a/include/uapi/drm/pvr_drm.h +++ b/include/uapi/drm/pvr_drm.h @@ -741,20 +741,18 @@ enum drm_pvr_ctx_priority { */ enum drm_pvr_ctx_type { /** - * @DRM_PVR_CTX_TYPE_RENDER: Render context. Use &struct - * drm_pvr_ioctl_create_render_context_args for context creation arguments. + * @DRM_PVR_CTX_TYPE_RENDER: Render context. */ DRM_PVR_CTX_TYPE_RENDER = 0, /** - * @DRM_PVR_CTX_TYPE_COMPUTE: Compute context. Use &struct - * drm_pvr_ioctl_create_compute_context_args for context creation arguments. + * @DRM_PVR_CTX_TYPE_COMPUTE: Compute context. */ DRM_PVR_CTX_TYPE_COMPUTE, /** - * @DRM_PVR_CTX_TYPE_TRANSFER_FRAG: Transfer context for fragment data masters. Use - * &struct drm_pvr_ioctl_create_transfer_context_args for context creation arguments. + * @DRM_PVR_CTX_TYPE_TRANSFER_FRAG: Transfer context for fragment data + * master. */ DRM_PVR_CTX_TYPE_TRANSFER_FRAG, }; -- cgit v1.2.3 From aafc1a2bea67460c41a289e8bb1e4dc6d016fe11 Mon Sep 17 00:00:00 2001 From: Melissa Wen Date: Thu, 30 Nov 2023 13:40:30 -0300 Subject: drm/v3d: Add a CPU job submission MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create a new type of job, a CPU job. A CPU job is a type of job that performs operations that requires CPU intervention. The overall idea is to use user extensions to enable different types of CPU job, allowing the CPU job to perform different operations according to the type of user extension. The user extension ID identify the type of CPU job that must be dealt. Having a CPU job is interesting for synchronization purposes as a CPU job has a queue like any other V3D job and can be synchoronized by the multisync extension. Signed-off-by: Melissa Wen Co-developed-by: Maíra Canal Signed-off-by: Maíra Canal Reviewed-by: Iago Toral Quiroga Link: https://patchwork.freedesktop.org/patch/msgid/20231130164420.932823-9-mcanal@igalia.com --- drivers/gpu/drm/v3d/v3d_drv.c | 4 ++ drivers/gpu/drm/v3d/v3d_drv.h | 16 +++++++- drivers/gpu/drm/v3d/v3d_sched.c | 57 ++++++++++++++++++++++++++ drivers/gpu/drm/v3d/v3d_submit.c | 86 ++++++++++++++++++++++++++++++++++++++++ include/uapi/drm/v3d_drm.h | 17 ++++++++ 5 files changed, 179 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/v3d/v3d_drv.c b/drivers/gpu/drm/v3d/v3d_drv.c index 44a1ca57d6a4..3debf37e7d9b 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.c +++ b/drivers/gpu/drm/v3d/v3d_drv.c @@ -91,6 +91,9 @@ static int v3d_get_param_ioctl(struct drm_device *dev, void *data, case DRM_V3D_PARAM_SUPPORTS_MULTISYNC_EXT: args->value = 1; return 0; + case DRM_V3D_PARAM_SUPPORTS_CPU_QUEUE: + args->value = 1; + return 0; default: DRM_DEBUG("Unknown parameter %d\n", args->param); return -EINVAL; @@ -189,6 +192,7 @@ static const struct drm_ioctl_desc v3d_drm_ioctls[] = { DRM_IOCTL_DEF_DRV(V3D_PERFMON_CREATE, v3d_perfmon_create_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(V3D_PERFMON_DESTROY, v3d_perfmon_destroy_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(V3D_PERFMON_GET_VALUES, v3d_perfmon_get_values_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(V3D_SUBMIT_CPU, v3d_submit_cpu_ioctl, DRM_RENDER_ALLOW | DRM_AUTH), }; static const struct drm_driver v3d_drm_driver = { diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 4db9ace66024..2246a0e29955 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -19,7 +19,7 @@ struct reset_control; #define GMP_GRANULARITY (128 * 1024) -#define V3D_MAX_QUEUES (V3D_CACHE_CLEAN + 1) +#define V3D_MAX_QUEUES (V3D_CPU + 1) static inline char *v3d_queue_to_string(enum v3d_queue queue) { @@ -29,6 +29,7 @@ static inline char *v3d_queue_to_string(enum v3d_queue queue) case V3D_TFU: return "tfu"; case V3D_CSD: return "csd"; case V3D_CACHE_CLEAN: return "cache_clean"; + case V3D_CPU: return "cpu"; } return "UNKNOWN"; } @@ -122,6 +123,7 @@ struct v3d_dev { struct v3d_render_job *render_job; struct v3d_tfu_job *tfu_job; struct v3d_csd_job *csd_job; + struct v3d_cpu_job *cpu_job; struct v3d_queue_state queue[V3D_MAX_QUEUES]; @@ -312,6 +314,16 @@ struct v3d_csd_job { struct drm_v3d_submit_csd args; }; +enum v3d_cpu_job_type {}; + +struct v3d_cpu_job { + struct v3d_job base; + + enum v3d_cpu_job_type job_type; +}; + +typedef void (*v3d_cpu_job_fn)(struct v3d_cpu_job *); + struct v3d_submit_outsync { struct drm_syncobj *syncobj; }; @@ -414,6 +426,8 @@ int v3d_submit_tfu_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); int v3d_submit_csd_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv); +int v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv); /* v3d_irq.c */ int v3d_irq_init(struct v3d_dev *v3d); diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index fccbea2a5f2e..c89e92fc614c 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -55,6 +55,12 @@ to_csd_job(struct drm_sched_job *sched_job) return container_of(sched_job, struct v3d_csd_job, base.base); } +static struct v3d_cpu_job * +to_cpu_job(struct drm_sched_job *sched_job) +{ + return container_of(sched_job, struct v3d_cpu_job, base.base); +} + static void v3d_sched_job_free(struct drm_sched_job *sched_job) { @@ -262,6 +268,42 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) return fence; } +static const v3d_cpu_job_fn cpu_job_function[] = { }; + +static struct dma_fence * +v3d_cpu_job_run(struct drm_sched_job *sched_job) +{ + struct v3d_cpu_job *job = to_cpu_job(sched_job); + struct v3d_dev *v3d = job->base.v3d; + struct v3d_file_priv *file = job->base.file->driver_priv; + u64 runtime; + + v3d->cpu_job = job; + + if (job->job_type >= ARRAY_SIZE(cpu_job_function)) { + DRM_DEBUG_DRIVER("Unknown CPU job: %d\n", job->job_type); + return NULL; + } + + file->start_ns[V3D_CPU] = local_clock(); + v3d->queue[V3D_CPU].start_ns = file->start_ns[V3D_CPU]; + + cpu_job_function[job->job_type](job); + + runtime = local_clock() - file->start_ns[V3D_CPU]; + + file->enabled_ns[V3D_CPU] += runtime; + v3d->queue[V3D_CPU].enabled_ns += runtime; + + file->jobs_sent[V3D_CPU]++; + v3d->queue[V3D_CPU].jobs_sent++; + + file->start_ns[V3D_CPU] = 0; + v3d->queue[V3D_CPU].start_ns = 0; + + return NULL; +} + static struct dma_fence * v3d_cache_clean_job_run(struct drm_sched_job *sched_job) { @@ -416,6 +458,12 @@ static const struct drm_sched_backend_ops v3d_cache_clean_sched_ops = { .free_job = v3d_sched_job_free }; +static const struct drm_sched_backend_ops v3d_cpu_sched_ops = { + .run_job = v3d_cpu_job_run, + .timedout_job = v3d_generic_job_timedout, + .free_job = v3d_sched_job_free +}; + int v3d_sched_init(struct v3d_dev *v3d) { @@ -471,6 +519,15 @@ v3d_sched_init(struct v3d_dev *v3d) goto fail; } + ret = drm_sched_init(&v3d->queue[V3D_CPU].sched, + &v3d_cpu_sched_ops, NULL, + DRM_SCHED_PRIORITY_COUNT, + 1, job_hang_limit, + msecs_to_jiffies(hang_limit_ms), NULL, + NULL, "v3d_cpu", v3d->drm.dev); + if (ret) + goto fail; + return 0; fail: diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 148900283c2a..918e28b1d00a 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -772,3 +772,89 @@ fail: return ret; } + +static const unsigned int cpu_job_bo_handle_count[] = { }; + +/** + * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D. + * @dev: DRM device + * @data: ioctl argument + * @file_priv: DRM file for this fd + * + * Userspace specifies the CPU job type and data required to perform its + * operations through the drm_v3d_extension struct. + */ +int +v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, + struct drm_file *file_priv) +{ + struct v3d_dev *v3d = to_v3d_dev(dev); + struct drm_v3d_submit_cpu *args = data; + struct v3d_submit_ext se = {0}; + struct v3d_cpu_job *cpu_job = NULL; + struct ww_acquire_ctx acquire_ctx; + int ret; + + if (args->flags && !(args->flags & DRM_V3D_SUBMIT_EXTENSION)) { + DRM_INFO("Invalid flags: %d\n", args->flags); + return -EINVAL; + } + + ret = v3d_job_allocate((void *)&cpu_job, sizeof(*cpu_job)); + if (ret) + return ret; + + if (args->flags & DRM_V3D_SUBMIT_EXTENSION) { + ret = v3d_get_extensions(file_priv, args->extensions, &se); + if (ret) { + DRM_DEBUG("Failed to get extensions.\n"); + goto fail; + } + } + + /* Every CPU job must have a CPU job user extension */ + if (!cpu_job->job_type) { + DRM_DEBUG("CPU job must have a CPU job user extension.\n"); + goto fail; + } + + if (args->bo_handle_count != cpu_job_bo_handle_count[cpu_job->job_type]) { + DRM_DEBUG("This CPU job was not submitted with the proper number of BOs.\n"); + goto fail; + } + + ret = v3d_job_init(v3d, file_priv, &cpu_job->base, + v3d_job_free, 0, &se, V3D_CPU); + if (ret) + goto fail; + + if (args->bo_handle_count) { + ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base, + args->bo_handles, args->bo_handle_count); + if (ret) + goto fail; + + ret = v3d_lock_bo_reservations(&cpu_job->base, &acquire_ctx); + if (ret) + goto fail; + } + + mutex_lock(&v3d->sched_lock); + v3d_push_job(&cpu_job->base); + mutex_unlock(&v3d->sched_lock); + + v3d_attach_fences_and_unlock_reservation(file_priv, + &cpu_job->base, + &acquire_ctx, 0, + NULL, cpu_job->base.done_fence); + + v3d_job_put(&cpu_job->base); + + return 0; + +fail: + v3d_job_cleanup((void *)cpu_job); + v3d_put_multisync_post_deps(&se); + + return ret; +} diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h index 1a7d7a689de3..00abef9d0db7 100644 --- a/include/uapi/drm/v3d_drm.h +++ b/include/uapi/drm/v3d_drm.h @@ -41,6 +41,7 @@ extern "C" { #define DRM_V3D_PERFMON_CREATE 0x08 #define DRM_V3D_PERFMON_DESTROY 0x09 #define DRM_V3D_PERFMON_GET_VALUES 0x0a +#define DRM_V3D_SUBMIT_CPU 0x0b #define DRM_IOCTL_V3D_SUBMIT_CL DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_SUBMIT_CL, struct drm_v3d_submit_cl) #define DRM_IOCTL_V3D_WAIT_BO DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_WAIT_BO, struct drm_v3d_wait_bo) @@ -56,6 +57,7 @@ extern "C" { struct drm_v3d_perfmon_destroy) #define DRM_IOCTL_V3D_PERFMON_GET_VALUES DRM_IOWR(DRM_COMMAND_BASE + DRM_V3D_PERFMON_GET_VALUES, \ struct drm_v3d_perfmon_get_values) +#define DRM_IOCTL_V3D_SUBMIT_CPU DRM_IOW(DRM_COMMAND_BASE + DRM_V3D_SUBMIT_CPU, struct drm_v3d_submit_cpu) #define DRM_V3D_SUBMIT_CL_FLUSH_CACHE 0x01 #define DRM_V3D_SUBMIT_EXTENSION 0x02 @@ -93,6 +95,7 @@ enum v3d_queue { V3D_TFU, V3D_CSD, V3D_CACHE_CLEAN, + V3D_CPU, }; /** @@ -276,6 +279,7 @@ enum drm_v3d_param { DRM_V3D_PARAM_SUPPORTS_CACHE_FLUSH, DRM_V3D_PARAM_SUPPORTS_PERFMON, DRM_V3D_PARAM_SUPPORTS_MULTISYNC_EXT, + DRM_V3D_PARAM_SUPPORTS_CPU_QUEUE, }; struct drm_v3d_get_param { @@ -361,6 +365,19 @@ struct drm_v3d_submit_csd { __u32 pad; }; +struct drm_v3d_submit_cpu { + /* Pointer to a u32 array of the BOs that are referenced by the job. */ + __u64 bo_handles; + + /* Number of BO handles passed in (size is that times 4). */ + __u32 bo_handle_count; + + __u32 flags; + + /* Pointer to an array of ioctl extensions*/ + __u64 extensions; +}; + enum { V3D_PERFCNT_FEP_VALID_PRIMTS_NO_PIXELS, V3D_PERFCNT_FEP_VALID_PRIMS, -- cgit v1.2.3 From 18b8413b25b7070fa2e55858a2c808e6909581d0 Mon Sep 17 00:00:00 2001 From: Maíra Canal Date: Thu, 30 Nov 2023 13:40:35 -0300 Subject: drm/v3d: Create a CPU job extension for a indirect CSD job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CPU job is a type of job that performs operations that requires CPU intervention. An indirect CSD job is a job that, when executed in the queue, will map the indirect buffer, read the dispatch parameters, and submit a regular dispatch. Therefore, it is a job that needs CPU intervention. So, create a user extension for the CPU job that enables the creation of an indirect CSD. This user extension will allow the creation of a CSD job linked to a CPU job. The CPU job will wait for the indirect CSD job dependencies and, once they are signaled, it will update the CSD job parameters. Co-developed-by: Melissa Wen Signed-off-by: Melissa Wen Signed-off-by: Maíra Canal Reviewed-by: Iago Toral Quiroga Link: https://patchwork.freedesktop.org/patch/msgid/20231130164420.932823-14-mcanal@igalia.com --- drivers/gpu/drm/v3d/v3d_drv.h | 31 +++++++++++- drivers/gpu/drm/v3d/v3d_sched.c | 41 ++++++++++++++- drivers/gpu/drm/v3d/v3d_submit.c | 104 ++++++++++++++++++++++++++++++++++++++- include/uapi/drm/v3d_drm.h | 43 +++++++++++++++- 4 files changed, 213 insertions(+), 6 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 39d62915cdd6..202c0d4b04a5 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -316,12 +316,41 @@ struct v3d_csd_job { struct drm_v3d_submit_csd args; }; -enum v3d_cpu_job_type {}; +enum v3d_cpu_job_type { + V3D_CPU_JOB_TYPE_INDIRECT_CSD = 1, +}; + +struct v3d_indirect_csd_info { + /* Indirect CSD */ + struct v3d_csd_job *job; + + /* Clean cache job associated to the Indirect CSD job */ + struct v3d_job *clean_job; + + /* Offset within the BO where the workgroup counts are stored */ + u32 offset; + + /* Workgroups size */ + u32 wg_size; + + /* Indices of the uniforms with the workgroup dispatch counts + * in the uniform stream. + */ + u32 wg_uniform_offsets[3]; + + /* Indirect BO */ + struct drm_gem_object *indirect; + + /* Context of the Indirect CSD job */ + struct ww_acquire_ctx acquire_ctx; +}; struct v3d_cpu_job { struct v3d_job base; enum v3d_cpu_job_type job_type; + + struct v3d_indirect_csd_info indirect_csd; }; typedef void (*v3d_cpu_job_fn)(struct v3d_cpu_job *); diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index ebbd00840a73..307257ab0e4a 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -25,6 +25,8 @@ #include "v3d_regs.h" #include "v3d_trace.h" +#define V3D_CSD_CFG012_WG_COUNT_SHIFT 16 + static struct v3d_job * to_v3d_job(struct drm_sched_job *sched_job) { @@ -268,7 +270,44 @@ v3d_csd_job_run(struct drm_sched_job *sched_job) return fence; } -static const v3d_cpu_job_fn cpu_job_function[] = { }; +static void +v3d_rewrite_csd_job_wg_counts_from_indirect(struct v3d_cpu_job *job) +{ + struct v3d_indirect_csd_info *indirect_csd = &job->indirect_csd; + struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]); + struct v3d_bo *indirect = to_v3d_bo(indirect_csd->indirect); + struct drm_v3d_submit_csd *args = &indirect_csd->job->args; + u32 *wg_counts; + + v3d_get_bo_vaddr(bo); + v3d_get_bo_vaddr(indirect); + + wg_counts = (uint32_t *)(bo->vaddr + indirect_csd->offset); + + if (wg_counts[0] == 0 || wg_counts[1] == 0 || wg_counts[2] == 0) + return; + + args->cfg[0] = wg_counts[0] << V3D_CSD_CFG012_WG_COUNT_SHIFT; + args->cfg[1] = wg_counts[1] << V3D_CSD_CFG012_WG_COUNT_SHIFT; + args->cfg[2] = wg_counts[2] << V3D_CSD_CFG012_WG_COUNT_SHIFT; + args->cfg[4] = DIV_ROUND_UP(indirect_csd->wg_size, 16) * + (wg_counts[0] * wg_counts[1] * wg_counts[2]) - 1; + + for (int i = 0; i < 3; i++) { + /* 0xffffffff indicates that the uniform rewrite is not needed */ + if (indirect_csd->wg_uniform_offsets[i] != 0xffffffff) { + u32 uniform_idx = indirect_csd->wg_uniform_offsets[i]; + ((uint32_t *)indirect->vaddr)[uniform_idx] = wg_counts[i]; + } + } + + v3d_put_bo_vaddr(indirect); + v3d_put_bo_vaddr(bo); +} + +static const v3d_cpu_job_fn cpu_job_function[] = { + [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = v3d_rewrite_csd_job_wg_counts_from_indirect, +}; static struct dma_fence * v3d_cpu_job_run(struct drm_sched_job *sched_job) diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index eb26fe1e27e3..0320695b941b 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -391,6 +391,48 @@ v3d_get_multisync_submit_deps(struct drm_file *file_priv, return 0; } +/* Get data for the indirect CSD job submission. */ +static int +v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, + struct drm_v3d_extension __user *ext, + struct v3d_cpu_job *job) +{ + struct v3d_file_priv *v3d_priv = file_priv->driver_priv; + struct v3d_dev *v3d = v3d_priv->v3d; + struct drm_v3d_indirect_csd indirect_csd; + struct v3d_indirect_csd_info *info = &job->indirect_csd; + + if (!job) { + DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); + return -EINVAL; + } + + if (job->job_type) { + DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); + return -EINVAL; + } + + if (copy_from_user(&indirect_csd, ext, sizeof(indirect_csd))) + return -EFAULT; + + if (!v3d_has_csd(v3d)) { + DRM_DEBUG("Attempting CSD submit on non-CSD hardware.\n"); + return -EINVAL; + } + + job->job_type = V3D_CPU_JOB_TYPE_INDIRECT_CSD; + info->offset = indirect_csd.offset; + info->wg_size = indirect_csd.wg_size; + memcpy(&info->wg_uniform_offsets, &indirect_csd.wg_uniform_offsets, + sizeof(indirect_csd.wg_uniform_offsets)); + + info->indirect = drm_gem_object_lookup(file_priv, indirect_csd.indirect); + + return v3d_setup_csd_jobs_and_bos(file_priv, v3d, &indirect_csd.submit, + &info->job, &info->clean_job, + NULL, &info->acquire_ctx); +} + /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data * according to the extension id (name). */ @@ -416,6 +458,9 @@ v3d_get_extensions(struct drm_file *file_priv, case DRM_V3D_EXT_ID_MULTI_SYNC: ret = v3d_get_multisync_submit_deps(file_priv, user_ext, se); break; + case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD: + ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job); + break; default: DRM_DEBUG_DRIVER("Unknown extension id: %d\n", ext.id); return -EINVAL; @@ -790,7 +835,9 @@ fail: return ret; } -static const unsigned int cpu_job_bo_handle_count[] = { }; +static const unsigned int cpu_job_bo_handle_count[] = { + [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, +}; /** * v3d_submit_cpu_ioctl() - Submits a CPU job to the V3D. @@ -808,7 +855,10 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, struct v3d_dev *v3d = to_v3d_dev(dev); struct drm_v3d_submit_cpu *args = data; struct v3d_submit_ext se = {0}; + struct v3d_submit_ext *out_se = NULL; struct v3d_cpu_job *cpu_job = NULL; + struct v3d_csd_job *csd_job = NULL; + struct v3d_job *clean_job = NULL; struct ww_acquire_ctx acquire_ctx; int ret; @@ -847,6 +897,9 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, if (ret) goto fail; + clean_job = cpu_job->indirect_csd.clean_job; + csd_job = cpu_job->indirect_csd.job; + if (args->bo_handle_count) { ret = v3d_lookup_bos(dev, file_priv, &cpu_job->base, args->bo_handles, args->bo_handle_count); @@ -860,19 +913,66 @@ v3d_submit_cpu_ioctl(struct drm_device *dev, void *data, mutex_lock(&v3d->sched_lock); v3d_push_job(&cpu_job->base); + + switch (cpu_job->job_type) { + case V3D_CPU_JOB_TYPE_INDIRECT_CSD: + ret = drm_sched_job_add_dependency(&csd_job->base.base, + dma_fence_get(cpu_job->base.done_fence)); + if (ret) + goto fail_unreserve; + + v3d_push_job(&csd_job->base); + + ret = drm_sched_job_add_dependency(&clean_job->base, + dma_fence_get(csd_job->base.done_fence)); + if (ret) + goto fail_unreserve; + + v3d_push_job(clean_job); + + break; + default: + break; + } mutex_unlock(&v3d->sched_lock); + out_se = (cpu_job->job_type == V3D_CPU_JOB_TYPE_INDIRECT_CSD) ? NULL : &se; + v3d_attach_fences_and_unlock_reservation(file_priv, &cpu_job->base, &acquire_ctx, 0, - NULL, cpu_job->base.done_fence); + out_se, cpu_job->base.done_fence); + + switch (cpu_job->job_type) { + case V3D_CPU_JOB_TYPE_INDIRECT_CSD: + v3d_attach_fences_and_unlock_reservation(file_priv, + clean_job, + &cpu_job->indirect_csd.acquire_ctx, + 0, &se, clean_job->done_fence); + break; + default: + break; + } v3d_job_put(&cpu_job->base); + v3d_job_put(&csd_job->base); + v3d_job_put(clean_job); return 0; +fail_unreserve: + mutex_unlock(&v3d->sched_lock); + + drm_gem_unlock_reservations(cpu_job->base.bo, cpu_job->base.bo_count, + &acquire_ctx); + + drm_gem_unlock_reservations(clean_job->bo, clean_job->bo_count, + &cpu_job->indirect_csd.acquire_ctx); + fail: v3d_job_cleanup((void *)cpu_job); + v3d_job_cleanup((void *)csd_job); + v3d_job_cleanup(clean_job); v3d_put_multisync_post_deps(&se); return ret; diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h index 00abef9d0db7..0c0f47782528 100644 --- a/include/uapi/drm/v3d_drm.h +++ b/include/uapi/drm/v3d_drm.h @@ -71,7 +71,8 @@ extern "C" { struct drm_v3d_extension { __u64 next; __u32 id; -#define DRM_V3D_EXT_ID_MULTI_SYNC 0x01 +#define DRM_V3D_EXT_ID_MULTI_SYNC 0x01 +#define DRM_V3D_EXT_ID_CPU_INDIRECT_CSD 0x02 __u32 flags; /* mbz */ }; @@ -365,8 +366,46 @@ struct drm_v3d_submit_csd { __u32 pad; }; +/** + * struct drm_v3d_indirect_csd - ioctl extension for the CPU job to create an + * indirect CSD + * + * When an extension of DRM_V3D_EXT_ID_CPU_INDIRECT_CSD id is defined, it + * points to this extension to define a indirect CSD submission. It creates a + * CPU job linked to a CSD job. The CPU job waits for the indirect CSD + * dependencies and, once they are signaled, it updates the CSD job config + * before allowing the CSD job execution. + */ +struct drm_v3d_indirect_csd { + struct drm_v3d_extension base; + + /* Indirect CSD */ + struct drm_v3d_submit_csd submit; + + /* Handle of the indirect BO, that should be also attached to the + * indirect CSD. + */ + __u32 indirect; + + /* Offset within the BO where the workgroup counts are stored */ + __u32 offset; + + /* Workgroups size */ + __u32 wg_size; + + /* Indices of the uniforms with the workgroup dispatch counts + * in the uniform stream. If the uniform rewrite is not needed, + * the offset must be 0xffffffff. + */ + __u32 wg_uniform_offsets[3]; +}; + struct drm_v3d_submit_cpu { - /* Pointer to a u32 array of the BOs that are referenced by the job. */ + /* Pointer to a u32 array of the BOs that are referenced by the job. + * + * For DRM_V3D_EXT_ID_CPU_INDIRECT_CSD, it must contain only one BO, + * that contains the workgroup counts. + */ __u64 bo_handles; /* Number of BO handles passed in (size is that times 4). */ -- cgit v1.2.3 From 9ba0ff3e083f6a4a0b6698f06bfff74805fefa5f Mon Sep 17 00:00:00 2001 From: Maíra Canal Date: Thu, 30 Nov 2023 13:40:36 -0300 Subject: drm/v3d: Create a CPU job extension for the timestamp query job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CPU job is a type of job that performs operations that requires CPU intervention. A timestamp query job is a job that calculates the query timestamp and updates the query availability by signaling a syncobj. As V3D doesn't provide any mechanism to obtain a timestamp from the GPU, it is a job that needs CPU intervention. So, create a user extension for the CPU job that enables the creation of a timestamp query job. This user extension will allow the creation of a CPU job that performs the timestamp query calculation and updates the timestamp BO with the proper value. Signed-off-by: Maíra Canal Reviewed-by: Iago Toral Quiroga Link: https://patchwork.freedesktop.org/patch/msgid/20231130164420.932823-15-mcanal@igalia.com --- drivers/gpu/drm/v3d/v3d_drv.h | 17 +++++++++++ drivers/gpu/drm/v3d/v3d_sched.c | 40 ++++++++++++++++++++++++- drivers/gpu/drm/v3d/v3d_submit.c | 63 ++++++++++++++++++++++++++++++++++++++++ include/uapi/drm/v3d_drm.h | 30 +++++++++++++++++++ 4 files changed, 149 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 202c0d4b04a5..dd86e745c260 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -318,6 +318,15 @@ struct v3d_csd_job { enum v3d_cpu_job_type { V3D_CPU_JOB_TYPE_INDIRECT_CSD = 1, + V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY, +}; + +struct v3d_timestamp_query { + /* Offset of this query in the timestamp BO for its value. */ + u32 offset; + + /* Syncobj that indicates the timestamp availability */ + struct drm_syncobj *syncobj; }; struct v3d_indirect_csd_info { @@ -345,12 +354,20 @@ struct v3d_indirect_csd_info { struct ww_acquire_ctx acquire_ctx; }; +struct v3d_timestamp_query_info { + struct v3d_timestamp_query *queries; + + u32 count; +}; + struct v3d_cpu_job { struct v3d_job base; enum v3d_cpu_job_type job_type; struct v3d_indirect_csd_info indirect_csd; + + struct v3d_timestamp_query_info timestamp_query; }; typedef void (*v3d_cpu_job_fn)(struct v3d_cpu_job *); diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 307257ab0e4a..5e692f087105 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -21,6 +21,8 @@ #include #include +#include + #include "v3d_drv.h" #include "v3d_regs.h" #include "v3d_trace.h" @@ -71,6 +73,21 @@ v3d_sched_job_free(struct drm_sched_job *sched_job) v3d_job_cleanup(job); } +static void +v3d_cpu_job_free(struct drm_sched_job *sched_job) +{ + struct v3d_cpu_job *job = to_cpu_job(sched_job); + struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query; + + if (timestamp_query->queries) { + for (int i = 0; i < timestamp_query->count; i++) + drm_syncobj_put(timestamp_query->queries[i].syncobj); + kvfree(timestamp_query->queries); + } + + v3d_job_cleanup(&job->base); +} + static void v3d_switch_perfmon(struct v3d_dev *v3d, struct v3d_job *job) { @@ -305,8 +322,29 @@ v3d_rewrite_csd_job_wg_counts_from_indirect(struct v3d_cpu_job *job) v3d_put_bo_vaddr(bo); } +static void +v3d_timestamp_query(struct v3d_cpu_job *job) +{ + struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query; + struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]); + u8 *value_addr; + + v3d_get_bo_vaddr(bo); + + for (int i = 0; i < timestamp_query->count; i++) { + value_addr = ((u8 *)bo->vaddr) + timestamp_query->queries[i].offset; + *((u64 *)value_addr) = i == 0 ? ktime_get_ns() : 0ull; + + drm_syncobj_replace_fence(timestamp_query->queries[i].syncobj, + job->base.done_fence); + } + + v3d_put_bo_vaddr(bo); +} + static const v3d_cpu_job_fn cpu_job_function[] = { [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = v3d_rewrite_csd_job_wg_counts_from_indirect, + [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = v3d_timestamp_query, }; static struct dma_fence * @@ -504,7 +542,7 @@ static const struct drm_sched_backend_ops v3d_cache_clean_sched_ops = { static const struct drm_sched_backend_ops v3d_cpu_sched_ops = { .run_job = v3d_cpu_job_run, .timedout_job = v3d_generic_job_timedout, - .free_job = v3d_sched_job_free + .free_job = v3d_cpu_job_free }; int diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 0320695b941b..83e029e786ea 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -433,6 +433,64 @@ v3d_get_cpu_indirect_csd_params(struct drm_file *file_priv, NULL, &info->acquire_ctx); } +/* Get data for the query timestamp job submission. */ +static int +v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv, + struct drm_v3d_extension __user *ext, + struct v3d_cpu_job *job) +{ + u32 __user *offsets, *syncs; + struct drm_v3d_timestamp_query timestamp; + + if (!job) { + DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); + return -EINVAL; + } + + if (job->job_type) { + DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); + return -EINVAL; + } + + if (copy_from_user(×tamp, ext, sizeof(timestamp))) + return -EFAULT; + + if (timestamp.pad) + return -EINVAL; + + job->job_type = V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY; + + job->timestamp_query.queries = kvmalloc_array(timestamp.count, + sizeof(struct v3d_timestamp_query), + GFP_KERNEL); + if (!job->timestamp_query.queries) + return -ENOMEM; + + offsets = u64_to_user_ptr(timestamp.offsets); + syncs = u64_to_user_ptr(timestamp.syncs); + + for (int i = 0; i < timestamp.count; i++) { + u32 offset, sync; + + if (copy_from_user(&offset, offsets++, sizeof(offset))) { + kvfree(job->timestamp_query.queries); + return -EFAULT; + } + + job->timestamp_query.queries[i].offset = offset; + + if (copy_from_user(&sync, syncs++, sizeof(sync))) { + kvfree(job->timestamp_query.queries); + return -EFAULT; + } + + job->timestamp_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); + } + job->timestamp_query.count = timestamp.count; + + return 0; +} + /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data * according to the extension id (name). */ @@ -461,6 +519,9 @@ v3d_get_extensions(struct drm_file *file_priv, case DRM_V3D_EXT_ID_CPU_INDIRECT_CSD: ret = v3d_get_cpu_indirect_csd_params(file_priv, user_ext, job); break; + case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY: + ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job); + break; default: DRM_DEBUG_DRIVER("Unknown extension id: %d\n", ext.id); return -EINVAL; @@ -837,6 +898,7 @@ fail: static const unsigned int cpu_job_bo_handle_count[] = { [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, + [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, }; /** @@ -974,6 +1036,7 @@ fail: v3d_job_cleanup((void *)csd_job); v3d_job_cleanup(clean_job); v3d_put_multisync_post_deps(&se); + kvfree(cpu_job->timestamp_query.queries); return ret; } diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h index 0c0f47782528..239801b5e117 100644 --- a/include/uapi/drm/v3d_drm.h +++ b/include/uapi/drm/v3d_drm.h @@ -73,6 +73,7 @@ struct drm_v3d_extension { __u32 id; #define DRM_V3D_EXT_ID_MULTI_SYNC 0x01 #define DRM_V3D_EXT_ID_CPU_INDIRECT_CSD 0x02 +#define DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY 0x03 __u32 flags; /* mbz */ }; @@ -400,11 +401,40 @@ struct drm_v3d_indirect_csd { __u32 wg_uniform_offsets[3]; }; +/** + * struct drm_v3d_timestamp_query - ioctl extension for the CPU job to calculate + * a timestamp query + * + * When an extension DRM_V3D_EXT_ID_TIMESTAMP_QUERY is defined, it points to + * this extension to define a timestamp query submission. This CPU job will + * calculate the timestamp query and update the query value within the + * timestamp BO. Moreover, it will signal the timestamp syncobj to indicate + * query availability. + */ +struct drm_v3d_timestamp_query { + struct drm_v3d_extension base; + + /* Array of queries' offsets within the timestamp BO for their value */ + __u64 offsets; + + /* Array of timestamp's syncobjs to indicate its availability */ + __u64 syncs; + + /* Number of queries */ + __u32 count; + + /* mbz */ + __u32 pad; +}; + struct drm_v3d_submit_cpu { /* Pointer to a u32 array of the BOs that are referenced by the job. * * For DRM_V3D_EXT_ID_CPU_INDIRECT_CSD, it must contain only one BO, * that contains the workgroup counts. + * + * For DRM_V3D_EXT_ID_TIMESTAMP_QUERY, it must contain only one BO, + * that will contain the timestamp. */ __u64 bo_handles; -- cgit v1.2.3 From 34a101e64296c736b14ce27e647fcebd70cb7bf8 Mon Sep 17 00:00:00 2001 From: Maíra Canal Date: Thu, 30 Nov 2023 13:40:37 -0300 Subject: drm/v3d: Create a CPU job extension for the reset timestamp job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CPU job is a type of job that performs operations that requires CPU intervention. A reset timestamp job is a job that resets the timestamp queries based on the value offset of the first query. As V3D doesn't provide any mechanism to obtain a timestamp from the GPU, it is a job that needs CPU intervention. So, create a user extension for the CPU job that enables the creation of a reset timestamp job. This user extension will allow the creation of a CPU job that resets the timestamp value in the timestamp BO and resets the availability syncobj. Signed-off-by: Maíra Canal Reviewed-by: Iago Toral Quiroga Link: https://patchwork.freedesktop.org/patch/msgid/20231130164420.932823-16-mcanal@igalia.com --- drivers/gpu/drm/v3d/v3d_drv.h | 1 + drivers/gpu/drm/v3d/v3d_sched.c | 21 ++++++++++++++++ drivers/gpu/drm/v3d/v3d_submit.c | 52 ++++++++++++++++++++++++++++++++++++++++ include/uapi/drm/v3d_drm.h | 27 +++++++++++++++++++++ 4 files changed, 101 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index dd86e745c260..3988407635ed 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -319,6 +319,7 @@ struct v3d_csd_job { enum v3d_cpu_job_type { V3D_CPU_JOB_TYPE_INDIRECT_CSD = 1, V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY, + V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY, }; struct v3d_timestamp_query { diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index 5e692f087105..e287f42d3621 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -342,9 +342,30 @@ v3d_timestamp_query(struct v3d_cpu_job *job) v3d_put_bo_vaddr(bo); } +static void +v3d_reset_timestamp_queries(struct v3d_cpu_job *job) +{ + struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query; + struct v3d_timestamp_query *queries = timestamp_query->queries; + struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]); + u8 *value_addr; + + v3d_get_bo_vaddr(bo); + + for (int i = 0; i < timestamp_query->count; i++) { + value_addr = ((u8 *)bo->vaddr) + queries[i].offset; + *((u64 *)value_addr) = 0; + + drm_syncobj_replace_fence(queries[i].syncobj, NULL); + } + + v3d_put_bo_vaddr(bo); +} + static const v3d_cpu_job_fn cpu_job_function[] = { [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = v3d_rewrite_csd_job_wg_counts_from_indirect, [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = v3d_timestamp_query, + [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = v3d_reset_timestamp_queries, }; static struct dma_fence * diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 83e029e786ea..1c719416e26a 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -491,6 +491,54 @@ v3d_get_cpu_timestamp_query_params(struct drm_file *file_priv, return 0; } +static int +v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv, + struct drm_v3d_extension __user *ext, + struct v3d_cpu_job *job) +{ + u32 __user *syncs; + struct drm_v3d_reset_timestamp_query reset; + + if (!job) { + DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); + return -EINVAL; + } + + if (job->job_type) { + DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); + return -EINVAL; + } + + if (copy_from_user(&reset, ext, sizeof(reset))) + return -EFAULT; + + job->job_type = V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY; + + job->timestamp_query.queries = kvmalloc_array(reset.count, + sizeof(struct v3d_timestamp_query), + GFP_KERNEL); + if (!job->timestamp_query.queries) + return -ENOMEM; + + syncs = u64_to_user_ptr(reset.syncs); + + for (int i = 0; i < reset.count; i++) { + u32 sync; + + job->timestamp_query.queries[i].offset = reset.offset + 8 * i; + + if (copy_from_user(&sync, syncs++, sizeof(sync))) { + kvfree(job->timestamp_query.queries); + return -EFAULT; + } + + job->timestamp_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); + } + job->timestamp_query.count = reset.count; + + return 0; +} + /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data * according to the extension id (name). */ @@ -522,6 +570,9 @@ v3d_get_extensions(struct drm_file *file_priv, case DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY: ret = v3d_get_cpu_timestamp_query_params(file_priv, user_ext, job); break; + case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY: + ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job); + break; default: DRM_DEBUG_DRIVER("Unknown extension id: %d\n", ext.id); return -EINVAL; @@ -899,6 +950,7 @@ fail: static const unsigned int cpu_job_bo_handle_count[] = { [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, + [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, }; /** diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h index 239801b5e117..930f8d07f088 100644 --- a/include/uapi/drm/v3d_drm.h +++ b/include/uapi/drm/v3d_drm.h @@ -74,6 +74,7 @@ struct drm_v3d_extension { #define DRM_V3D_EXT_ID_MULTI_SYNC 0x01 #define DRM_V3D_EXT_ID_CPU_INDIRECT_CSD 0x02 #define DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY 0x03 +#define DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY 0x04 __u32 flags; /* mbz */ }; @@ -427,6 +428,29 @@ struct drm_v3d_timestamp_query { __u32 pad; }; +/** + * struct drm_v3d_reset_timestamp_query - ioctl extension for the CPU job to + * reset timestamp queries + * + * When an extension DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY is defined, it + * points to this extension to define a reset timestamp submission. This CPU + * job will reset the timestamp queries based on value offset of the first + * query. Moreover, it will reset the timestamp syncobj to reset query + * availability. + */ +struct drm_v3d_reset_timestamp_query { + struct drm_v3d_extension base; + + /* Array of timestamp's syncobjs to indicate its availability */ + __u64 syncs; + + /* Offset of the first query within the timestamp BO for its value */ + __u32 offset; + + /* Number of queries */ + __u32 count; +}; + struct drm_v3d_submit_cpu { /* Pointer to a u32 array of the BOs that are referenced by the job. * @@ -435,6 +459,9 @@ struct drm_v3d_submit_cpu { * * For DRM_V3D_EXT_ID_TIMESTAMP_QUERY, it must contain only one BO, * that will contain the timestamp. + * + * For DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY, it must contain only + * one BO, that contains the timestamp. */ __u64 bo_handles; -- cgit v1.2.3 From 6745f3e44a20ac18e7e5a40a3c7f62225983d544 Mon Sep 17 00:00:00 2001 From: Maíra Canal Date: Thu, 30 Nov 2023 13:40:38 -0300 Subject: drm/v3d: Create a CPU job extension to copy timestamp query to a buffer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CPU job is a type of job that performs operations that requires CPU intervention. A copy timestamp query job is a job that copy the complete or partial result of a query to a buffer. As V3D doesn't provide any mechanism to obtain a timestamp from the GPU, it is a job that needs CPU intervention. So, create a user extension for the CPU job that enables the creation of a copy timestamp query job. This user extension will allow the creation of a CPU job that copy the results of a timestamp query to a BO with the possibility to indicate the timestamp availability with a availability bit. Signed-off-by: Maíra Canal Reviewed-by: Iago Toral Quiroga Link: https://patchwork.freedesktop.org/patch/msgid/20231130164420.932823-17-mcanal@igalia.com --- drivers/gpu/drm/v3d/v3d_drv.h | 20 ++++++++++++ drivers/gpu/drm/v3d/v3d_sched.c | 56 ++++++++++++++++++++++++++++++++ drivers/gpu/drm/v3d/v3d_submit.c | 69 ++++++++++++++++++++++++++++++++++++++++ include/uapi/drm/v3d_drm.h | 45 ++++++++++++++++++++++++++ 4 files changed, 190 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 3988407635ed..5058a354fffd 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -320,6 +320,7 @@ enum v3d_cpu_job_type { V3D_CPU_JOB_TYPE_INDIRECT_CSD = 1, V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY, V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY, + V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY, }; struct v3d_timestamp_query { @@ -361,6 +362,23 @@ struct v3d_timestamp_query_info { u32 count; }; +struct v3d_copy_query_results_info { + /* Define if should write to buffer using 64 or 32 bits */ + bool do_64bit; + + /* Define if it can write to buffer even if the query is not available */ + bool do_partial; + + /* Define if it should write availability bit to buffer */ + bool availability_bit; + + /* Offset of the copy buffer in the BO */ + u32 offset; + + /* Stride of the copy buffer in the BO */ + u32 stride; +}; + struct v3d_cpu_job { struct v3d_job base; @@ -369,6 +387,8 @@ struct v3d_cpu_job { struct v3d_indirect_csd_info indirect_csd; struct v3d_timestamp_query_info timestamp_query; + + struct v3d_copy_query_results_info copy; }; typedef void (*v3d_cpu_job_fn)(struct v3d_cpu_job *); diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index e287f42d3621..b1662d32a929 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -362,10 +362,66 @@ v3d_reset_timestamp_queries(struct v3d_cpu_job *job) v3d_put_bo_vaddr(bo); } +static void +write_to_buffer(void *dst, u32 idx, bool do_64bit, u64 value) +{ + if (do_64bit) { + u64 *dst64 = (u64 *)dst; + + dst64[idx] = value; + } else { + u32 *dst32 = (u32 *)dst; + + dst32[idx] = (u32)value; + } +} + +static void +v3d_copy_query_results(struct v3d_cpu_job *job) +{ + struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query; + struct v3d_timestamp_query *queries = timestamp_query->queries; + struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]); + struct v3d_bo *timestamp = to_v3d_bo(job->base.bo[1]); + struct v3d_copy_query_results_info *copy = &job->copy; + struct dma_fence *fence; + u8 *query_addr; + bool available, write_result; + u8 *data; + int i; + + v3d_get_bo_vaddr(bo); + v3d_get_bo_vaddr(timestamp); + + data = ((u8 *)bo->vaddr) + copy->offset; + + for (i = 0; i < timestamp_query->count; i++) { + fence = drm_syncobj_fence_get(queries[i].syncobj); + available = fence ? dma_fence_is_signaled(fence) : false; + + write_result = available || copy->do_partial; + if (write_result) { + query_addr = ((u8 *)timestamp->vaddr) + queries[i].offset; + write_to_buffer(data, 0, copy->do_64bit, *((u64 *)query_addr)); + } + + if (copy->availability_bit) + write_to_buffer(data, 1, copy->do_64bit, available ? 1u : 0u); + + data += copy->stride; + + dma_fence_put(fence); + } + + v3d_put_bo_vaddr(timestamp); + v3d_put_bo_vaddr(bo); +} + static const v3d_cpu_job_fn cpu_job_function[] = { [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = v3d_rewrite_csd_job_wg_counts_from_indirect, [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = v3d_timestamp_query, [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = v3d_reset_timestamp_queries, + [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = v3d_copy_query_results, }; static struct dma_fence * diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 1c719416e26a..bafd49c6440c 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -539,6 +539,71 @@ v3d_get_cpu_reset_timestamp_params(struct drm_file *file_priv, return 0; } +/* Get data for the copy timestamp query results job submission. */ +static int +v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv, + struct drm_v3d_extension __user *ext, + struct v3d_cpu_job *job) +{ + u32 __user *offsets, *syncs; + struct drm_v3d_copy_timestamp_query copy; + int i; + + if (!job) { + DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); + return -EINVAL; + } + + if (job->job_type) { + DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); + return -EINVAL; + } + + if (copy_from_user(©, ext, sizeof(copy))) + return -EFAULT; + + if (copy.pad) + return -EINVAL; + + job->job_type = V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY; + + job->timestamp_query.queries = kvmalloc_array(copy.count, + sizeof(struct v3d_timestamp_query), + GFP_KERNEL); + if (!job->timestamp_query.queries) + return -ENOMEM; + + offsets = u64_to_user_ptr(copy.offsets); + syncs = u64_to_user_ptr(copy.syncs); + + for (i = 0; i < copy.count; i++) { + u32 offset, sync; + + if (copy_from_user(&offset, offsets++, sizeof(offset))) { + kvfree(job->timestamp_query.queries); + return -EFAULT; + } + + job->timestamp_query.queries[i].offset = offset; + + if (copy_from_user(&sync, syncs++, sizeof(sync))) { + kvfree(job->timestamp_query.queries); + return -EFAULT; + } + + job->timestamp_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); + } + job->timestamp_query.count = copy.count; + + job->copy.do_64bit = copy.do_64bit; + job->copy.do_partial = copy.do_partial; + job->copy.availability_bit = copy.availability_bit; + job->copy.offset = copy.offset; + job->copy.stride = copy.stride; + + return 0; +} + /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data * according to the extension id (name). */ @@ -573,6 +638,9 @@ v3d_get_extensions(struct drm_file *file_priv, case DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY: ret = v3d_get_cpu_reset_timestamp_params(file_priv, user_ext, job); break; + case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY: + ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job); + break; default: DRM_DEBUG_DRIVER("Unknown extension id: %d\n", ext.id); return -EINVAL; @@ -951,6 +1019,7 @@ static const unsigned int cpu_job_bo_handle_count[] = { [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = 1, [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, + [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2, }; /** diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h index 930f8d07f088..f7defbf3d043 100644 --- a/include/uapi/drm/v3d_drm.h +++ b/include/uapi/drm/v3d_drm.h @@ -75,6 +75,7 @@ struct drm_v3d_extension { #define DRM_V3D_EXT_ID_CPU_INDIRECT_CSD 0x02 #define DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY 0x03 #define DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY 0x04 +#define DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY 0x05 __u32 flags; /* mbz */ }; @@ -451,6 +452,46 @@ struct drm_v3d_reset_timestamp_query { __u32 count; }; +/** + * struct drm_v3d_copy_timestamp_query - ioctl extension for the CPU job to copy + * query results to a buffer + * + * When an extension DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY is defined, it + * points to this extension to define a copy timestamp query submission. This + * CPU job will copy the timestamp queries results to a BO with the offset + * and stride defined in the extension. + */ +struct drm_v3d_copy_timestamp_query { + struct drm_v3d_extension base; + + /* Define if should write to buffer using 64 or 32 bits */ + __u8 do_64bit; + + /* Define if it can write to buffer even if the query is not available */ + __u8 do_partial; + + /* Define if it should write availability bit to buffer */ + __u8 availability_bit; + + /* mbz */ + __u8 pad; + + /* Offset of the buffer in the BO */ + __u32 offset; + + /* Stride of the buffer in the BO */ + __u32 stride; + + /* Number of queries */ + __u32 count; + + /* Array of queries' offsets within the timestamp BO for their value */ + __u64 offsets; + + /* Array of timestamp's syncobjs to indicate its availability */ + __u64 syncs; +}; + struct drm_v3d_submit_cpu { /* Pointer to a u32 array of the BOs that are referenced by the job. * @@ -462,6 +503,10 @@ struct drm_v3d_submit_cpu { * * For DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY, it must contain only * one BO, that contains the timestamp. + * + * For DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY, it must contain two + * BOs. The first is the BO where the timestamp queries will be written + * to. The second is the BO that contains the timestamp. */ __u64 bo_handles; -- cgit v1.2.3 From bae7cb5d68001a8d4ceec5964dda74bb9aab7220 Mon Sep 17 00:00:00 2001 From: Maíra Canal Date: Thu, 30 Nov 2023 13:40:39 -0300 Subject: drm/v3d: Create a CPU job extension for the reset performance query job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CPU job is a type of job that performs operations that requires CPU intervention. A reset performance query job is a job that resets the performance queries by resetting the values of the perfmons. Moreover, we also reset the syncobjs related to the availability of the query. So, create a user extension for the CPU job that enables the creation of a reset performance job. This user extension will allow the creation of a CPU job that resets the perfmons values and resets the availability syncobj. Signed-off-by: Maíra Canal Reviewed-by: Iago Toral Quiroga Link: https://patchwork.freedesktop.org/patch/msgid/20231130164420.932823-18-mcanal@igalia.com --- drivers/gpu/drm/v3d/v3d_drv.h | 28 +++++++++++++++ drivers/gpu/drm/v3d/v3d_sched.c | 36 ++++++++++++++++++++ drivers/gpu/drm/v3d/v3d_submit.c | 73 ++++++++++++++++++++++++++++++++++++++++ include/uapi/drm/v3d_drm.h | 30 +++++++++++++++++ 4 files changed, 167 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 5058a354fffd..0f7f80ad8d88 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -321,6 +321,7 @@ enum v3d_cpu_job_type { V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY, V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY, V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY, + V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY, }; struct v3d_timestamp_query { @@ -331,6 +332,18 @@ struct v3d_timestamp_query { struct drm_syncobj *syncobj; }; +/* Number of perfmons required to handle all supported performance counters */ +#define V3D_MAX_PERFMONS DIV_ROUND_UP(V3D_PERFCNT_NUM, \ + DRM_V3D_MAX_PERF_COUNTERS) + +struct v3d_performance_query { + /* Performance monitor IDs for this query */ + u32 kperfmon_ids[V3D_MAX_PERFMONS]; + + /* Syncobj that indicates the query availability */ + struct drm_syncobj *syncobj; +}; + struct v3d_indirect_csd_info { /* Indirect CSD */ struct v3d_csd_job *job; @@ -362,6 +375,19 @@ struct v3d_timestamp_query_info { u32 count; }; +struct v3d_performance_query_info { + struct v3d_performance_query *queries; + + /* Number of performance queries */ + u32 count; + + /* Number of performance monitors related to that query pool */ + u32 nperfmons; + + /* Number of performance counters related to that query pool */ + u32 ncounters; +}; + struct v3d_copy_query_results_info { /* Define if should write to buffer using 64 or 32 bits */ bool do_64bit; @@ -389,6 +415,8 @@ struct v3d_cpu_job { struct v3d_timestamp_query_info timestamp_query; struct v3d_copy_query_results_info copy; + + struct v3d_performance_query_info performance_query; }; typedef void (*v3d_cpu_job_fn)(struct v3d_cpu_job *); diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index b1662d32a929..e83d2907bc83 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -78,6 +78,7 @@ v3d_cpu_job_free(struct drm_sched_job *sched_job) { struct v3d_cpu_job *job = to_cpu_job(sched_job); struct v3d_timestamp_query_info *timestamp_query = &job->timestamp_query; + struct v3d_performance_query_info *performance_query = &job->performance_query; if (timestamp_query->queries) { for (int i = 0; i < timestamp_query->count; i++) @@ -85,6 +86,12 @@ v3d_cpu_job_free(struct drm_sched_job *sched_job) kvfree(timestamp_query->queries); } + if (performance_query->queries) { + for (int i = 0; i < performance_query->count; i++) + drm_syncobj_put(performance_query->queries[i].syncobj); + kvfree(performance_query->queries); + } + v3d_job_cleanup(&job->base); } @@ -417,11 +424,40 @@ v3d_copy_query_results(struct v3d_cpu_job *job) v3d_put_bo_vaddr(bo); } +static void +v3d_reset_performance_queries(struct v3d_cpu_job *job) +{ + struct v3d_performance_query_info *performance_query = &job->performance_query; + struct v3d_file_priv *v3d_priv = job->base.file->driver_priv; + struct v3d_dev *v3d = job->base.v3d; + struct v3d_perfmon *perfmon; + + for (int i = 0; i < performance_query->count; i++) { + for (int j = 0; j < performance_query->nperfmons; j++) { + perfmon = v3d_perfmon_find(v3d_priv, + performance_query->queries[i].kperfmon_ids[j]); + if (!perfmon) { + DRM_DEBUG("Failed to find perfmon."); + continue; + } + + v3d_perfmon_stop(v3d, perfmon, false); + + memset(perfmon->values, 0, perfmon->ncounters * sizeof(u64)); + + v3d_perfmon_put(perfmon); + } + + drm_syncobj_replace_fence(performance_query->queries[i].syncobj, NULL); + } +} + static const v3d_cpu_job_fn cpu_job_function[] = { [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = v3d_rewrite_csd_job_wg_counts_from_indirect, [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = v3d_timestamp_query, [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = v3d_reset_timestamp_queries, [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = v3d_copy_query_results, + [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = v3d_reset_performance_queries, }; static struct dma_fence * diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index bafd49c6440c..20af8ae14831 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -604,6 +604,74 @@ v3d_get_cpu_copy_query_results_params(struct drm_file *file_priv, return 0; } +static int +v3d_get_cpu_reset_performance_params(struct drm_file *file_priv, + struct drm_v3d_extension __user *ext, + struct v3d_cpu_job *job) +{ + u32 __user *syncs; + u64 __user *kperfmon_ids; + struct drm_v3d_reset_performance_query reset; + + if (!job) { + DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); + return -EINVAL; + } + + if (job->job_type) { + DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); + return -EINVAL; + } + + if (copy_from_user(&reset, ext, sizeof(reset))) + return -EFAULT; + + job->job_type = V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY; + + job->performance_query.queries = kvmalloc_array(reset.count, + sizeof(struct v3d_performance_query), + GFP_KERNEL); + if (!job->performance_query.queries) + return -ENOMEM; + + syncs = u64_to_user_ptr(reset.syncs); + kperfmon_ids = u64_to_user_ptr(reset.kperfmon_ids); + + for (int i = 0; i < reset.count; i++) { + u32 sync; + u64 ids; + u32 __user *ids_pointer; + u32 id; + + if (copy_from_user(&sync, syncs++, sizeof(sync))) { + kvfree(job->performance_query.queries); + return -EFAULT; + } + + job->performance_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); + + if (copy_from_user(&ids, kperfmon_ids++, sizeof(ids))) { + kvfree(job->performance_query.queries); + return -EFAULT; + } + + ids_pointer = u64_to_user_ptr(ids); + + for (int j = 0; j < reset.nperfmons; j++) { + if (copy_from_user(&id, ids_pointer++, sizeof(id))) { + kvfree(job->performance_query.queries); + return -EFAULT; + } + + job->performance_query.queries[i].kperfmon_ids[j] = id; + } + } + job->performance_query.count = reset.count; + job->performance_query.nperfmons = reset.nperfmons; + + return 0; +} + /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data * according to the extension id (name). */ @@ -641,6 +709,9 @@ v3d_get_extensions(struct drm_file *file_priv, case DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY: ret = v3d_get_cpu_copy_query_results_params(file_priv, user_ext, job); break; + case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY: + ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job); + break; default: DRM_DEBUG_DRIVER("Unknown extension id: %d\n", ext.id); return -EINVAL; @@ -1020,6 +1091,7 @@ static const unsigned int cpu_job_bo_handle_count[] = { [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = 1, [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2, + [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0, }; /** @@ -1158,6 +1230,7 @@ fail: v3d_job_cleanup(clean_job); v3d_put_multisync_post_deps(&se); kvfree(cpu_job->timestamp_query.queries); + kvfree(cpu_job->performance_query.queries); return ret; } diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h index f7defbf3d043..d609bdf29fd6 100644 --- a/include/uapi/drm/v3d_drm.h +++ b/include/uapi/drm/v3d_drm.h @@ -76,6 +76,7 @@ struct drm_v3d_extension { #define DRM_V3D_EXT_ID_CPU_TIMESTAMP_QUERY 0x03 #define DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY 0x04 #define DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY 0x05 +#define DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY 0x06 __u32 flags; /* mbz */ }; @@ -492,6 +493,32 @@ struct drm_v3d_copy_timestamp_query { __u64 syncs; }; +/** + * struct drm_v3d_reset_performance_query - ioctl extension for the CPU job to + * reset performance queries + * + * When an extension DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY is defined, it + * points to this extension to define a reset performance submission. This CPU + * job will reset the performance queries by resetting the values of the + * performance monitors. Moreover, it will reset the syncobj to reset query + * availability. + */ +struct drm_v3d_reset_performance_query { + struct drm_v3d_extension base; + + /* Array of performance queries's syncobjs to indicate its availability */ + __u64 syncs; + + /* Number of queries */ + __u32 count; + + /* Number of performance monitors */ + __u32 nperfmons; + + /* Array of u64 user-pointers that point to an array of kperfmon_ids */ + __u64 kperfmon_ids; +}; + struct drm_v3d_submit_cpu { /* Pointer to a u32 array of the BOs that are referenced by the job. * @@ -507,6 +534,9 @@ struct drm_v3d_submit_cpu { * For DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY, it must contain two * BOs. The first is the BO where the timestamp queries will be written * to. The second is the BO that contains the timestamp. + * + * For DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY, it must contain no + * BOs. */ __u64 bo_handles; -- cgit v1.2.3 From 209e8d2695ee7a67a5b0487bbd1aa75e290d0f41 Mon Sep 17 00:00:00 2001 From: Maíra Canal Date: Thu, 30 Nov 2023 13:40:40 -0300 Subject: drm/v3d: Create a CPU job extension for the copy performance query job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A CPU job is a type of job that performs operations that requires CPU intervention. A copy performance query job is a job that copy the complete or partial result of a query to a buffer. In order to copy the result of a performance query to a buffer, we need to get the values from the performance monitors. So, create a user extension for the CPU job that enables the creation of a copy performance query job. This user extension will allow the creation of a CPU job that copy the results of a performance query to a BO with the possibility to indicate the availability with a availability bit. Signed-off-by: Maíra Canal Reviewed-by: Iago Toral Quiroga Link: https://patchwork.freedesktop.org/patch/msgid/20231130164420.932823-19-mcanal@igalia.com --- drivers/gpu/drm/v3d/v3d_drv.h | 1 + drivers/gpu/drm/v3d/v3d_sched.c | 65 +++++++++++++++++++++++++++++++ drivers/gpu/drm/v3d/v3d_submit.c | 82 ++++++++++++++++++++++++++++++++++++++++ include/uapi/drm/v3d_drm.h | 50 ++++++++++++++++++++++++ 4 files changed, 198 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/v3d/v3d_drv.h b/drivers/gpu/drm/v3d/v3d_drv.h index 0f7f80ad8d88..3c7d58866570 100644 --- a/drivers/gpu/drm/v3d/v3d_drv.h +++ b/drivers/gpu/drm/v3d/v3d_drv.h @@ -322,6 +322,7 @@ enum v3d_cpu_job_type { V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY, V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY, V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY, + V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY, }; struct v3d_timestamp_query { diff --git a/drivers/gpu/drm/v3d/v3d_sched.c b/drivers/gpu/drm/v3d/v3d_sched.c index e83d2907bc83..54015ad765c7 100644 --- a/drivers/gpu/drm/v3d/v3d_sched.c +++ b/drivers/gpu/drm/v3d/v3d_sched.c @@ -452,12 +452,77 @@ v3d_reset_performance_queries(struct v3d_cpu_job *job) } } +static void +v3d_write_performance_query_result(struct v3d_cpu_job *job, void *data, u32 query) +{ + struct v3d_performance_query_info *performance_query = &job->performance_query; + struct v3d_copy_query_results_info *copy = &job->copy; + struct v3d_file_priv *v3d_priv = job->base.file->driver_priv; + struct v3d_dev *v3d = job->base.v3d; + struct v3d_perfmon *perfmon; + u64 counter_values[V3D_PERFCNT_NUM]; + + for (int i = 0; i < performance_query->nperfmons; i++) { + perfmon = v3d_perfmon_find(v3d_priv, + performance_query->queries[query].kperfmon_ids[i]); + if (!perfmon) { + DRM_DEBUG("Failed to find perfmon."); + continue; + } + + v3d_perfmon_stop(v3d, perfmon, true); + + memcpy(&counter_values[i * DRM_V3D_MAX_PERF_COUNTERS], perfmon->values, + perfmon->ncounters * sizeof(u64)); + + v3d_perfmon_put(perfmon); + } + + for (int i = 0; i < performance_query->ncounters; i++) + write_to_buffer(data, i, copy->do_64bit, counter_values[i]); +} + +static void +v3d_copy_performance_query(struct v3d_cpu_job *job) +{ + struct v3d_performance_query_info *performance_query = &job->performance_query; + struct v3d_copy_query_results_info *copy = &job->copy; + struct v3d_bo *bo = to_v3d_bo(job->base.bo[0]); + struct dma_fence *fence; + bool available, write_result; + u8 *data; + + v3d_get_bo_vaddr(bo); + + data = ((u8 *)bo->vaddr) + copy->offset; + + for (int i = 0; i < performance_query->count; i++) { + fence = drm_syncobj_fence_get(performance_query->queries[i].syncobj); + available = fence ? dma_fence_is_signaled(fence) : false; + + write_result = available || copy->do_partial; + if (write_result) + v3d_write_performance_query_result(job, data, i); + + if (copy->availability_bit) + write_to_buffer(data, performance_query->ncounters, + copy->do_64bit, available ? 1u : 0u); + + data += copy->stride; + + dma_fence_put(fence); + } + + v3d_put_bo_vaddr(bo); +} + static const v3d_cpu_job_fn cpu_job_function[] = { [V3D_CPU_JOB_TYPE_INDIRECT_CSD] = v3d_rewrite_csd_job_wg_counts_from_indirect, [V3D_CPU_JOB_TYPE_TIMESTAMP_QUERY] = v3d_timestamp_query, [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = v3d_reset_timestamp_queries, [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = v3d_copy_query_results, [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = v3d_reset_performance_queries, + [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = v3d_copy_performance_query, }; static struct dma_fence * diff --git a/drivers/gpu/drm/v3d/v3d_submit.c b/drivers/gpu/drm/v3d/v3d_submit.c index 20af8ae14831..d7a9da2484fd 100644 --- a/drivers/gpu/drm/v3d/v3d_submit.c +++ b/drivers/gpu/drm/v3d/v3d_submit.c @@ -672,6 +672,84 @@ v3d_get_cpu_reset_performance_params(struct drm_file *file_priv, return 0; } +static int +v3d_get_cpu_copy_performance_query_params(struct drm_file *file_priv, + struct drm_v3d_extension __user *ext, + struct v3d_cpu_job *job) +{ + u32 __user *syncs; + u64 __user *kperfmon_ids; + struct drm_v3d_copy_performance_query copy; + + if (!job) { + DRM_DEBUG("CPU job extension was attached to a GPU job.\n"); + return -EINVAL; + } + + if (job->job_type) { + DRM_DEBUG("Two CPU job extensions were added to the same CPU job.\n"); + return -EINVAL; + } + + if (copy_from_user(©, ext, sizeof(copy))) + return -EFAULT; + + if (copy.pad) + return -EINVAL; + + job->job_type = V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY; + + job->performance_query.queries = kvmalloc_array(copy.count, + sizeof(struct v3d_performance_query), + GFP_KERNEL); + if (!job->performance_query.queries) + return -ENOMEM; + + syncs = u64_to_user_ptr(copy.syncs); + kperfmon_ids = u64_to_user_ptr(copy.kperfmon_ids); + + for (int i = 0; i < copy.count; i++) { + u32 sync; + u64 ids; + u32 __user *ids_pointer; + u32 id; + + if (copy_from_user(&sync, syncs++, sizeof(sync))) { + kvfree(job->performance_query.queries); + return -EFAULT; + } + + job->performance_query.queries[i].syncobj = drm_syncobj_find(file_priv, sync); + + if (copy_from_user(&ids, kperfmon_ids++, sizeof(ids))) { + kvfree(job->performance_query.queries); + return -EFAULT; + } + + ids_pointer = u64_to_user_ptr(ids); + + for (int j = 0; j < copy.nperfmons; j++) { + if (copy_from_user(&id, ids_pointer++, sizeof(id))) { + kvfree(job->performance_query.queries); + return -EFAULT; + } + + job->performance_query.queries[i].kperfmon_ids[j] = id; + } + } + job->performance_query.count = copy.count; + job->performance_query.nperfmons = copy.nperfmons; + job->performance_query.ncounters = copy.ncounters; + + job->copy.do_64bit = copy.do_64bit; + job->copy.do_partial = copy.do_partial; + job->copy.availability_bit = copy.availability_bit; + job->copy.offset = copy.offset; + job->copy.stride = copy.stride; + + return 0; +} + /* Whenever userspace sets ioctl extensions, v3d_get_extensions parses data * according to the extension id (name). */ @@ -712,6 +790,9 @@ v3d_get_extensions(struct drm_file *file_priv, case DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY: ret = v3d_get_cpu_reset_performance_params(file_priv, user_ext, job); break; + case DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY: + ret = v3d_get_cpu_copy_performance_query_params(file_priv, user_ext, job); + break; default: DRM_DEBUG_DRIVER("Unknown extension id: %d\n", ext.id); return -EINVAL; @@ -1092,6 +1173,7 @@ static const unsigned int cpu_job_bo_handle_count[] = { [V3D_CPU_JOB_TYPE_RESET_TIMESTAMP_QUERY] = 1, [V3D_CPU_JOB_TYPE_COPY_TIMESTAMP_QUERY] = 2, [V3D_CPU_JOB_TYPE_RESET_PERFORMANCE_QUERY] = 0, + [V3D_CPU_JOB_TYPE_COPY_PERFORMANCE_QUERY] = 1, }; /** diff --git a/include/uapi/drm/v3d_drm.h b/include/uapi/drm/v3d_drm.h index d609bdf29fd6..dce1835eced4 100644 --- a/include/uapi/drm/v3d_drm.h +++ b/include/uapi/drm/v3d_drm.h @@ -77,6 +77,7 @@ struct drm_v3d_extension { #define DRM_V3D_EXT_ID_CPU_RESET_TIMESTAMP_QUERY 0x04 #define DRM_V3D_EXT_ID_CPU_COPY_TIMESTAMP_QUERY 0x05 #define DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY 0x06 +#define DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY 0x07 __u32 flags; /* mbz */ }; @@ -519,6 +520,52 @@ struct drm_v3d_reset_performance_query { __u64 kperfmon_ids; }; +/** + * struct drm_v3d_copy_performance_query - ioctl extension for the CPU job to copy + * performance query results to a buffer + * + * When an extension DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY is defined, it + * points to this extension to define a copy performance query submission. This + * CPU job will copy the performance queries results to a BO with the offset + * and stride defined in the extension. + */ +struct drm_v3d_copy_performance_query { + struct drm_v3d_extension base; + + /* Define if should write to buffer using 64 or 32 bits */ + __u8 do_64bit; + + /* Define if it can write to buffer even if the query is not available */ + __u8 do_partial; + + /* Define if it should write availability bit to buffer */ + __u8 availability_bit; + + /* mbz */ + __u8 pad; + + /* Offset of the buffer in the BO */ + __u32 offset; + + /* Stride of the buffer in the BO */ + __u32 stride; + + /* Number of performance monitors */ + __u32 nperfmons; + + /* Number of performance counters related to this query pool */ + __u32 ncounters; + + /* Number of queries */ + __u32 count; + + /* Array of performance queries's syncobjs to indicate its availability */ + __u64 syncs; + + /* Array of u64 user-pointers that point to an array of kperfmon_ids */ + __u64 kperfmon_ids; +}; + struct drm_v3d_submit_cpu { /* Pointer to a u32 array of the BOs that are referenced by the job. * @@ -537,6 +584,9 @@ struct drm_v3d_submit_cpu { * * For DRM_V3D_EXT_ID_CPU_RESET_PERFORMANCE_QUERY, it must contain no * BOs. + * + * For DRM_V3D_EXT_ID_CPU_COPY_PERFORMANCE_QUERY, it must contain one + * BO, where the performance queries will be written. */ __u64 bo_handles; -- cgit v1.2.3 From 8570c27932e132d2663e8120311891deb2a853de Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Wed, 23 Aug 2023 14:54:54 -0700 Subject: drm/syncobj: Add deadline support for syncobj waits Add a new flag to let userspace provide a deadline as a hint for syncobj and timeline waits. This gives a hint to the driver signaling the backing fences about how soon userspace needs it to compete work, so it can adjust GPU frequency accordingly. An immediate deadline can be given to provide something equivalent to i915 "wait boost". v2: Use absolute u64 ns value for deadline hint, drop cap and driver feature flag in favor of allowing count_handles==0 as a way for userspace to probe kernel for support of new flag v3: More verbose comments about UAPI v4: Fix negative zero, s/deadline_ns/deadline_nsec/ for consistency with existing ioctl struct fields v5: Comment/description typo fixes Signed-off-by: Rob Clark Reviewed-by: Tvrtko Ursulin [DB: fixed checkpatch warnings] Signed-off-by: Dmitry Baryshkov Link: https://patchwork.freedesktop.org/patch/msgid/20230823215458.203366-2-robdclark@gmail.com --- drivers/gpu/drm/drm_syncobj.c | 64 ++++++++++++++++++++++++++++++++++--------- include/uapi/drm/drm.h | 17 ++++++++++++ 2 files changed, 68 insertions(+), 13 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/drm_syncobj.c b/drivers/gpu/drm/drm_syncobj.c index 01da6789d044..cbb65b7ba425 100644 --- a/drivers/gpu/drm/drm_syncobj.c +++ b/drivers/gpu/drm/drm_syncobj.c @@ -126,6 +126,11 @@ * synchronize between the two. * This requirement is inherited from the Vulkan fence API. * + * If &DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE is set, the ioctl will also set + * a fence deadline hint on the backing fences before waiting, to provide the + * fence signaler with an appropriate sense of urgency. The deadline is + * specified as an absolute &CLOCK_MONOTONIC value in units of ns. + * * Similarly, &DRM_IOCTL_SYNCOBJ_TIMELINE_WAIT takes an array of syncobj * handles as well as an array of u64 points and does a host-side wait on all * of syncobj fences at the given points simultaneously. @@ -1027,7 +1032,8 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, uint32_t count, uint32_t flags, signed long timeout, - uint32_t *idx) + uint32_t *idx, + ktime_t *deadline) { struct syncobj_wait_entry *entries; struct dma_fence *fence; @@ -1108,6 +1114,15 @@ static signed long drm_syncobj_array_wait_timeout(struct drm_syncobj **syncobjs, drm_syncobj_fence_add_wait(syncobjs[i], &entries[i]); } + if (deadline) { + for (i = 0; i < count; ++i) { + fence = entries[i].fence; + if (!fence) + continue; + dma_fence_set_deadline(fence, *deadline); + } + } + do { set_current_state(TASK_INTERRUPTIBLE); @@ -1206,7 +1221,8 @@ static int drm_syncobj_array_wait(struct drm_device *dev, struct drm_file *file_private, struct drm_syncobj_wait *wait, struct drm_syncobj_timeline_wait *timeline_wait, - struct drm_syncobj **syncobjs, bool timeline) + struct drm_syncobj **syncobjs, bool timeline, + ktime_t *deadline) { signed long timeout = 0; uint32_t first = ~0; @@ -1217,7 +1233,8 @@ static int drm_syncobj_array_wait(struct drm_device *dev, NULL, wait->count_handles, wait->flags, - timeout, &first); + timeout, &first, + deadline); if (timeout < 0) return timeout; wait->first_signaled = first; @@ -1227,7 +1244,8 @@ static int drm_syncobj_array_wait(struct drm_device *dev, u64_to_user_ptr(timeline_wait->points), timeline_wait->count_handles, timeline_wait->flags, - timeout, &first); + timeout, &first, + deadline); if (timeout < 0) return timeout; timeline_wait->first_signaled = first; @@ -1298,17 +1316,22 @@ drm_syncobj_wait_ioctl(struct drm_device *dev, void *data, { struct drm_syncobj_wait *args = data; struct drm_syncobj **syncobjs; + unsigned int possible_flags; + ktime_t t, *tp = NULL; int ret = 0; if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ)) return -EOPNOTSUPP; - if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | - DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT)) + possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | + DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | + DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; + + if (args->flags & ~possible_flags) return -EINVAL; if (args->count_handles == 0) - return -EINVAL; + return 0; ret = drm_syncobj_array_find(file_private, u64_to_user_ptr(args->handles), @@ -1317,8 +1340,13 @@ drm_syncobj_wait_ioctl(struct drm_device *dev, void *data, if (ret < 0) return ret; + if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { + t = ns_to_ktime(args->deadline_nsec); + tp = &t; + } + ret = drm_syncobj_array_wait(dev, file_private, - args, NULL, syncobjs, false); + args, NULL, syncobjs, false, tp); drm_syncobj_array_free(syncobjs, args->count_handles); @@ -1331,18 +1359,23 @@ drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data, { struct drm_syncobj_timeline_wait *args = data; struct drm_syncobj **syncobjs; + unsigned int possible_flags; + ktime_t t, *tp = NULL; int ret = 0; if (!drm_core_check_feature(dev, DRIVER_SYNCOBJ_TIMELINE)) return -EOPNOTSUPP; - if (args->flags & ~(DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | - DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | - DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE)) + possible_flags = DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL | + DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT | + DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE | + DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE; + + if (args->flags & ~possible_flags) return -EINVAL; if (args->count_handles == 0) - return -EINVAL; + return 0; ret = drm_syncobj_array_find(file_private, u64_to_user_ptr(args->handles), @@ -1351,8 +1384,13 @@ drm_syncobj_timeline_wait_ioctl(struct drm_device *dev, void *data, if (ret < 0) return ret; + if (args->flags & DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE) { + t = ns_to_ktime(args->deadline_nsec); + tp = &t; + } + ret = drm_syncobj_array_wait(dev, file_private, - NULL, args, syncobjs, true); + NULL, args, syncobjs, true, tp); drm_syncobj_array_free(syncobjs, args->count_handles); diff --git a/include/uapi/drm/drm.h b/include/uapi/drm/drm.h index 91943e5ce010..16122819edfe 100644 --- a/include/uapi/drm/drm.h +++ b/include/uapi/drm/drm.h @@ -926,6 +926,7 @@ struct drm_syncobj_transfer { #define DRM_SYNCOBJ_WAIT_FLAGS_WAIT_ALL (1 << 0) #define DRM_SYNCOBJ_WAIT_FLAGS_WAIT_FOR_SUBMIT (1 << 1) #define DRM_SYNCOBJ_WAIT_FLAGS_WAIT_AVAILABLE (1 << 2) /* wait for time point to become available */ +#define DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE (1 << 3) /* set fence deadline to deadline_nsec */ struct drm_syncobj_wait { __u64 handles; /* absolute timeout */ @@ -934,6 +935,14 @@ struct drm_syncobj_wait { __u32 flags; __u32 first_signaled; /* only valid when not waiting all */ __u32 pad; + /** + * @deadline_nsec - fence deadline hint + * + * Deadline hint, in absolute CLOCK_MONOTONIC, to set on backing + * fence(s) if the DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE flag is + * set. + */ + __u64 deadline_nsec; }; struct drm_syncobj_timeline_wait { @@ -946,6 +955,14 @@ struct drm_syncobj_timeline_wait { __u32 flags; __u32 first_signaled; /* only valid when not waiting all */ __u32 pad; + /** + * @deadline_nsec - fence deadline hint + * + * Deadline hint, in absolute CLOCK_MONOTONIC, to set on backing + * fence(s) if the DRM_SYNCOBJ_WAIT_FLAGS_WAIT_DEADLINE flag is + * set. + */ + __u64 deadline_nsec; }; /** -- cgit v1.2.3 From 63ee44540205d993854f143a5ab1d7d9e63ffcf1 Mon Sep 17 00:00:00 2001 From: Rob Clark Date: Wed, 23 Aug 2023 14:54:55 -0700 Subject: dma-buf/sync_file: Add SET_DEADLINE ioctl The initial purpose is for igt tests, but this would also be useful for compositors that wait until close to vblank deadline to make decisions about which frame to show. The igt tests can be found at: https://gitlab.freedesktop.org/robclark/igt-gpu-tools/-/commits/fence-deadline v2: Clarify the timebase, add link to igt tests v3: Use u64 value in ns to express deadline. v4: More doc Signed-off-by: Rob Clark Acked-by: Pekka Paalanen Signed-off-by: Dmitry Baryshkov Link: https://patchwork.freedesktop.org/patch/msgid/20230823215458.203366-3-robdclark@gmail.com --- drivers/dma-buf/dma-fence.c | 3 ++- drivers/dma-buf/sync_file.c | 19 +++++++++++++++++++ include/uapi/linux/sync_file.h | 22 ++++++++++++++++++++++ 3 files changed, 43 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/drivers/dma-buf/dma-fence.c b/drivers/dma-buf/dma-fence.c index 8aa8f8cb7071..e0fd99e61a2d 100644 --- a/drivers/dma-buf/dma-fence.c +++ b/drivers/dma-buf/dma-fence.c @@ -934,7 +934,8 @@ EXPORT_SYMBOL(dma_fence_wait_any_timeout); * the GPU's devfreq to reduce frequency, when in fact the opposite is what is * needed. * - * To this end, deadline hint(s) can be set on a &dma_fence via &dma_fence_set_deadline. + * To this end, deadline hint(s) can be set on a &dma_fence via &dma_fence_set_deadline + * (or indirectly via userspace facing ioctls like &sync_set_deadline). * The deadline hint provides a way for the waiting driver, or userspace, to * convey an appropriate sense of urgency to the signaling driver. * diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c index 2e9a316c596a..d9b1c1b2a72b 100644 --- a/drivers/dma-buf/sync_file.c +++ b/drivers/dma-buf/sync_file.c @@ -347,6 +347,22 @@ out: return ret; } +static int sync_file_ioctl_set_deadline(struct sync_file *sync_file, + unsigned long arg) +{ + struct sync_set_deadline ts; + + if (copy_from_user(&ts, (void __user *)arg, sizeof(ts))) + return -EFAULT; + + if (ts.pad) + return -EINVAL; + + dma_fence_set_deadline(sync_file->fence, ns_to_ktime(ts.deadline_ns)); + + return 0; +} + static long sync_file_ioctl(struct file *file, unsigned int cmd, unsigned long arg) { @@ -359,6 +375,9 @@ static long sync_file_ioctl(struct file *file, unsigned int cmd, case SYNC_IOC_FILE_INFO: return sync_file_ioctl_fence_info(sync_file, arg); + case SYNC_IOC_SET_DEADLINE: + return sync_file_ioctl_set_deadline(sync_file, arg); + default: return -ENOTTY; } diff --git a/include/uapi/linux/sync_file.h b/include/uapi/linux/sync_file.h index ff0a931833e2..ff1f38889dcf 100644 --- a/include/uapi/linux/sync_file.h +++ b/include/uapi/linux/sync_file.h @@ -76,6 +76,27 @@ struct sync_file_info { __u64 sync_fence_info; }; +/** + * struct sync_set_deadline - SYNC_IOC_SET_DEADLINE - set a deadline hint on a fence + * @deadline_ns: absolute time of the deadline + * @pad: must be zero + * + * Allows userspace to set a deadline on a fence, see &dma_fence_set_deadline + * + * The timebase for the deadline is CLOCK_MONOTONIC (same as vblank). For + * example + * + * clock_gettime(CLOCK_MONOTONIC, &t); + * deadline_ns = (t.tv_sec * 1000000000L) + t.tv_nsec + ns_until_deadline + */ +struct sync_set_deadline { + __u64 deadline_ns; + /* Not strictly needed for alignment but gives some possibility + * for future extension: + */ + __u64 pad; +}; + #define SYNC_IOC_MAGIC '>' /* @@ -87,5 +108,6 @@ struct sync_file_info { #define SYNC_IOC_MERGE _IOWR(SYNC_IOC_MAGIC, 3, struct sync_merge_data) #define SYNC_IOC_FILE_INFO _IOWR(SYNC_IOC_MAGIC, 4, struct sync_file_info) +#define SYNC_IOC_SET_DEADLINE _IOW(SYNC_IOC_MAGIC, 5, struct sync_set_deadline) #endif /* _UAPI_LINUX_SYNC_H */ -- cgit v1.2.3 From 85863a4e16e77079ee14865905ddc3ef9483a640 Mon Sep 17 00:00:00 2001 From: Jessica Zhang Date: Fri, 27 Oct 2023 15:32:52 -0700 Subject: drm: Introduce solid fill DRM plane property Document and add support for solid_fill property to drm_plane. In addition, add support for setting and getting the values for solid_fill. To enable solid fill planes, userspace must assign a property blob to the "solid_fill" plane property containing the following information: struct drm_mode_solid_fill { u32 r, g, b, pad; }; Acked-by: Harry Wentland Acked-by: Sebastian Wick Signed-off-by: Jessica Zhang Reviewed-by: Dmitry Baryshkov Signed-off-by: Dmitry Baryshkov Link: https://patchwork.freedesktop.org/patch/msgid/20231027-solid-fill-v7-2-780188bfa7b2@quicinc.com --- drivers/gpu/drm/drm_atomic_state_helper.c | 9 ++++++++ drivers/gpu/drm/drm_atomic_uapi.c | 26 ++++++++++++++++++++++ drivers/gpu/drm/drm_blend.c | 30 ++++++++++++++++++++++++++ include/drm/drm_blend.h | 1 + include/drm/drm_plane.h | 36 +++++++++++++++++++++++++++++++ include/uapi/drm/drm_mode.h | 24 +++++++++++++++++++++ 6 files changed, 126 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c index 311b04edf742..f6c76cea8be4 100644 --- a/drivers/gpu/drm/drm_atomic_state_helper.c +++ b/drivers/gpu/drm/drm_atomic_state_helper.c @@ -254,6 +254,11 @@ void __drm_atomic_helper_plane_state_reset(struct drm_plane_state *plane_state, plane_state->pixel_blend_mode = DRM_MODE_BLEND_PREMULTI; plane_state->pixel_source = DRM_PLANE_PIXEL_SOURCE_FB; + if (plane_state->solid_fill_blob) { + drm_property_blob_put(plane_state->solid_fill_blob); + plane_state->solid_fill_blob = NULL; + } + if (plane->color_encoding_property) { if (!drm_object_property_get_default_value(&plane->base, plane->color_encoding_property, @@ -350,6 +355,9 @@ void __drm_atomic_helper_plane_duplicate_state(struct drm_plane *plane, if (state->fb) drm_framebuffer_get(state->fb); + if (state->solid_fill_blob) + drm_property_blob_get(state->solid_fill_blob); + state->fence = NULL; state->commit = NULL; state->fb_damage_clips = NULL; @@ -399,6 +407,7 @@ void __drm_atomic_helper_plane_destroy_state(struct drm_plane_state *state) drm_crtc_commit_put(state->commit); drm_property_blob_put(state->fb_damage_clips); + drm_property_blob_put(state->solid_fill_blob); } EXPORT_SYMBOL(__drm_atomic_helper_plane_destroy_state); diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index bd7140531948..d7ae8e2c0265 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -316,6 +316,20 @@ drm_atomic_set_crtc_for_connector(struct drm_connector_state *conn_state, } EXPORT_SYMBOL(drm_atomic_set_crtc_for_connector); +static void drm_atomic_set_solid_fill_prop(struct drm_plane_state *state) +{ + struct drm_mode_solid_fill *user_info; + + if (!state->solid_fill_blob) + return; + + user_info = (struct drm_mode_solid_fill *)state->solid_fill_blob->data; + + state->solid_fill.r = user_info->r; + state->solid_fill.g = user_info->g; + state->solid_fill.b = user_info->b; +} + static void set_out_fence_for_crtc(struct drm_atomic_state *state, struct drm_crtc *crtc, s32 __user *fence_ptr) { @@ -564,6 +578,15 @@ static int drm_atomic_plane_set_property(struct drm_plane *plane, state->src_h = val; } else if (property == plane->pixel_source_property) { state->pixel_source = val; + } else if (property == plane->solid_fill_property) { + ret = drm_atomic_replace_property_blob_from_id(dev, + &state->solid_fill_blob, + val, sizeof(struct drm_mode_solid_fill), + -1, &replaced); + if (ret) + return ret; + + drm_atomic_set_solid_fill_prop(state); } else if (property == plane->alpha_property) { state->alpha = val; } else if (property == plane->blend_mode_property) { @@ -654,6 +677,9 @@ drm_atomic_plane_get_property(struct drm_plane *plane, *val = state->src_h; } else if (property == plane->pixel_source_property) { *val = state->pixel_source; + } else if (property == plane->solid_fill_property) { + *val = state->solid_fill_blob ? + state->solid_fill_blob->base.id : 0; } else if (property == plane->alpha_property) { *val = state->alpha; } else if (property == plane->blend_mode_property) { diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c index fce734cdb85b..665c5d9b056f 100644 --- a/drivers/gpu/drm/drm_blend.c +++ b/drivers/gpu/drm/drm_blend.c @@ -204,6 +204,10 @@ * "FB": * Framebuffer source set by the "FB_ID" property. * + * solid_fill: + * solid_fill is set up with drm_plane_create_solid_fill_property(). It + * contains pixel data that drivers can use to fill a plane. + * * Note that all the property extensions described here apply either to the * plane or the CRTC (e.g. for the background color, which currently is not * exposed and assumed to be black). @@ -709,3 +713,29 @@ int drm_plane_create_pixel_source_property(struct drm_plane *plane, return 0; } EXPORT_SYMBOL(drm_plane_create_pixel_source_property); + +/** + * drm_plane_create_solid_fill_property - create a new solid_fill property + * @plane: drm plane + * + * This creates a new property blob that holds pixel data for solid fill planes. + * The property is exposed to userspace as a property blob called "solid_fill". + * + * For information on what the blob contains, see `drm_mode_solid_fill`. + */ +int drm_plane_create_solid_fill_property(struct drm_plane *plane) +{ + struct drm_property *prop; + + prop = drm_property_create(plane->dev, + DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB, + "solid_fill", 0); + if (!prop) + return -ENOMEM; + + drm_object_attach_property(&plane->base, prop, 0); + plane->solid_fill_property = prop; + + return 0; +} +EXPORT_SYMBOL(drm_plane_create_solid_fill_property); diff --git a/include/drm/drm_blend.h b/include/drm/drm_blend.h index 122bbfbaae33..e7158fbee389 100644 --- a/include/drm/drm_blend.h +++ b/include/drm/drm_blend.h @@ -60,4 +60,5 @@ int drm_plane_create_blend_mode_property(struct drm_plane *plane, unsigned int supported_modes); int drm_plane_create_pixel_source_property(struct drm_plane *plane, unsigned long extra_sources); +int drm_plane_create_solid_fill_property(struct drm_plane *plane); #endif diff --git a/include/drm/drm_plane.h b/include/drm/drm_plane.h index bc0176ba25be..5bac644d4cc3 100644 --- a/include/drm/drm_plane.h +++ b/include/drm/drm_plane.h @@ -46,6 +46,18 @@ enum drm_plane_pixel_source { DRM_PLANE_PIXEL_SOURCE_MAX }; +/** + * struct solid_fill_property - RGB values for solid fill plane + * + * TODO: Add solid fill source and corresponding pixel source + * that supports RGBA color + */ +struct drm_solid_fill { + uint32_t r; + uint32_t g; + uint32_t b; +}; + /** * struct drm_plane_state - mutable plane state * @@ -134,6 +146,23 @@ struct drm_plane_state { */ enum drm_plane_pixel_source pixel_source; + /** + * @solid_fill_blob: + * + * Blob containing relevant information for a solid fill plane + * including RGB color values. See + * drm_plane_create_solid_fill_property() for more details. + */ + struct drm_property_blob *solid_fill_blob; + + /** + * @solid_fill: + * + * Pixel data for solid fill planes. See + * drm_plane_create_solid_fill_property() for more details. + */ + struct drm_solid_fill solid_fill; + /** * @alpha: * Opacity of the plane with 0 as completely transparent and 0xffff as @@ -734,6 +763,13 @@ struct drm_plane { */ struct drm_property *pixel_source_property; + /** + * @solid_fill_property: + * Optional solid_fill property for this plane. See + * drm_plane_create_solid_fill_property(). + */ + struct drm_property *solid_fill_property; + /** * @alpha_property: * Optional alpha property for this plane. See diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index 95630f170110..cbd943a8c88b 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -259,6 +259,30 @@ struct drm_mode_modeinfo { char name[DRM_DISPLAY_MODE_LEN]; }; +/** + * struct drm_mode_solid_fill - User info for solid fill planes + * + * This is the userspace API solid fill information structure. + * + * Userspace can enable solid fill planes by assigning the plane "solid_fill" + * property to a blob containing a single drm_mode_solid_fill struct populated with an RGB323232 + * color and setting the pixel source to "SOLID_FILL". + * + * For information on the plane property, see drm_plane_create_solid_fill_property() + * + * @r: Red color value of single pixel + * @g: Green color value of single pixel + * @b: Blue color value of single pixel + * @pad: padding, must be zero + */ +struct drm_mode_solid_fill { + __u32 r; + __u32 g; + __u32 b; + __u32 pad; +}; + + struct drm_mode_card_res { __u64 fb_id_ptr; __u64 crtc_id_ptr; -- cgit v1.2.3 From e5fba1ada1c1d676438138d815acd8f427a1eaf0 Mon Sep 17 00:00:00 2001 From: Dmitry Baryshkov Date: Mon, 4 Dec 2023 15:13:53 +0200 Subject: Revert "drm: Introduce solid fill DRM plane property" This reverts commit 85863a4e16e77079ee14865905ddc3ef9483a640. Although the Solid Fill planes patchset got all reviews and acknowledgements, it doesn't fulfill requirements for the new uABI. It has neither corresponding open-source userspace implementation nor the IGT tests coverage. Reverting this patchset until userspace obligations are fulfilled. Acked-by: Simon Ser Acked-by: Maxime Ripard Signed-off-by: Dmitry Baryshkov Link: https://patchwork.freedesktop.org/patch/msgid/20231204131455.19023-7-dmitry.baryshkov@linaro.org --- drivers/gpu/drm/drm_atomic_state_helper.c | 9 -------- drivers/gpu/drm/drm_atomic_uapi.c | 26 ---------------------- drivers/gpu/drm/drm_blend.c | 30 -------------------------- include/drm/drm_blend.h | 1 - include/drm/drm_plane.h | 36 ------------------------------- include/uapi/drm/drm_mode.h | 24 --------------------- 6 files changed, 126 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/drm_atomic_state_helper.c b/drivers/gpu/drm/drm_atomic_state_helper.c index f6c76cea8be4..311b04edf742 100644 --- a/drivers/gpu/drm/drm_atomic_state_helper.c +++ b/drivers/gpu/drm/drm_atomic_state_helper.c @@ -254,11 +254,6 @@ void __drm_atomic_helper_plane_state_reset(struct drm_plane_state *plane_state, plane_state->pixel_blend_mode = DRM_MODE_BLEND_PREMULTI; plane_state->pixel_source = DRM_PLANE_PIXEL_SOURCE_FB; - if (plane_state->solid_fill_blob) { - drm_property_blob_put(plane_state->solid_fill_blob); - plane_state->solid_fill_blob = NULL; - } - if (plane->color_encoding_property) { if (!drm_object_property_get_default_value(&plane->base, plane->color_encoding_property, @@ -355,9 +350,6 @@ void __drm_atomic_helper_plane_duplicate_state(struct drm_plane *plane, if (state->fb) drm_framebuffer_get(state->fb); - if (state->solid_fill_blob) - drm_property_blob_get(state->solid_fill_blob); - state->fence = NULL; state->commit = NULL; state->fb_damage_clips = NULL; @@ -407,7 +399,6 @@ void __drm_atomic_helper_plane_destroy_state(struct drm_plane_state *state) drm_crtc_commit_put(state->commit); drm_property_blob_put(state->fb_damage_clips); - drm_property_blob_put(state->solid_fill_blob); } EXPORT_SYMBOL(__drm_atomic_helper_plane_destroy_state); diff --git a/drivers/gpu/drm/drm_atomic_uapi.c b/drivers/gpu/drm/drm_atomic_uapi.c index d7ae8e2c0265..bd7140531948 100644 --- a/drivers/gpu/drm/drm_atomic_uapi.c +++ b/drivers/gpu/drm/drm_atomic_uapi.c @@ -316,20 +316,6 @@ drm_atomic_set_crtc_for_connector(struct drm_connector_state *conn_state, } EXPORT_SYMBOL(drm_atomic_set_crtc_for_connector); -static void drm_atomic_set_solid_fill_prop(struct drm_plane_state *state) -{ - struct drm_mode_solid_fill *user_info; - - if (!state->solid_fill_blob) - return; - - user_info = (struct drm_mode_solid_fill *)state->solid_fill_blob->data; - - state->solid_fill.r = user_info->r; - state->solid_fill.g = user_info->g; - state->solid_fill.b = user_info->b; -} - static void set_out_fence_for_crtc(struct drm_atomic_state *state, struct drm_crtc *crtc, s32 __user *fence_ptr) { @@ -578,15 +564,6 @@ static int drm_atomic_plane_set_property(struct drm_plane *plane, state->src_h = val; } else if (property == plane->pixel_source_property) { state->pixel_source = val; - } else if (property == plane->solid_fill_property) { - ret = drm_atomic_replace_property_blob_from_id(dev, - &state->solid_fill_blob, - val, sizeof(struct drm_mode_solid_fill), - -1, &replaced); - if (ret) - return ret; - - drm_atomic_set_solid_fill_prop(state); } else if (property == plane->alpha_property) { state->alpha = val; } else if (property == plane->blend_mode_property) { @@ -677,9 +654,6 @@ drm_atomic_plane_get_property(struct drm_plane *plane, *val = state->src_h; } else if (property == plane->pixel_source_property) { *val = state->pixel_source; - } else if (property == plane->solid_fill_property) { - *val = state->solid_fill_blob ? - state->solid_fill_blob->base.id : 0; } else if (property == plane->alpha_property) { *val = state->alpha; } else if (property == plane->blend_mode_property) { diff --git a/drivers/gpu/drm/drm_blend.c b/drivers/gpu/drm/drm_blend.c index 665c5d9b056f..fce734cdb85b 100644 --- a/drivers/gpu/drm/drm_blend.c +++ b/drivers/gpu/drm/drm_blend.c @@ -204,10 +204,6 @@ * "FB": * Framebuffer source set by the "FB_ID" property. * - * solid_fill: - * solid_fill is set up with drm_plane_create_solid_fill_property(). It - * contains pixel data that drivers can use to fill a plane. - * * Note that all the property extensions described here apply either to the * plane or the CRTC (e.g. for the background color, which currently is not * exposed and assumed to be black). @@ -713,29 +709,3 @@ int drm_plane_create_pixel_source_property(struct drm_plane *plane, return 0; } EXPORT_SYMBOL(drm_plane_create_pixel_source_property); - -/** - * drm_plane_create_solid_fill_property - create a new solid_fill property - * @plane: drm plane - * - * This creates a new property blob that holds pixel data for solid fill planes. - * The property is exposed to userspace as a property blob called "solid_fill". - * - * For information on what the blob contains, see `drm_mode_solid_fill`. - */ -int drm_plane_create_solid_fill_property(struct drm_plane *plane) -{ - struct drm_property *prop; - - prop = drm_property_create(plane->dev, - DRM_MODE_PROP_ATOMIC | DRM_MODE_PROP_BLOB, - "solid_fill", 0); - if (!prop) - return -ENOMEM; - - drm_object_attach_property(&plane->base, prop, 0); - plane->solid_fill_property = prop; - - return 0; -} -EXPORT_SYMBOL(drm_plane_create_solid_fill_property); diff --git a/include/drm/drm_blend.h b/include/drm/drm_blend.h index e7158fbee389..122bbfbaae33 100644 --- a/include/drm/drm_blend.h +++ b/include/drm/drm_blend.h @@ -60,5 +60,4 @@ int drm_plane_create_blend_mode_property(struct drm_plane *plane, unsigned int supported_modes); int drm_plane_create_pixel_source_property(struct drm_plane *plane, unsigned long extra_sources); -int drm_plane_create_solid_fill_property(struct drm_plane *plane); #endif diff --git a/include/drm/drm_plane.h b/include/drm/drm_plane.h index 5bac644d4cc3..bc0176ba25be 100644 --- a/include/drm/drm_plane.h +++ b/include/drm/drm_plane.h @@ -46,18 +46,6 @@ enum drm_plane_pixel_source { DRM_PLANE_PIXEL_SOURCE_MAX }; -/** - * struct solid_fill_property - RGB values for solid fill plane - * - * TODO: Add solid fill source and corresponding pixel source - * that supports RGBA color - */ -struct drm_solid_fill { - uint32_t r; - uint32_t g; - uint32_t b; -}; - /** * struct drm_plane_state - mutable plane state * @@ -146,23 +134,6 @@ struct drm_plane_state { */ enum drm_plane_pixel_source pixel_source; - /** - * @solid_fill_blob: - * - * Blob containing relevant information for a solid fill plane - * including RGB color values. See - * drm_plane_create_solid_fill_property() for more details. - */ - struct drm_property_blob *solid_fill_blob; - - /** - * @solid_fill: - * - * Pixel data for solid fill planes. See - * drm_plane_create_solid_fill_property() for more details. - */ - struct drm_solid_fill solid_fill; - /** * @alpha: * Opacity of the plane with 0 as completely transparent and 0xffff as @@ -763,13 +734,6 @@ struct drm_plane { */ struct drm_property *pixel_source_property; - /** - * @solid_fill_property: - * Optional solid_fill property for this plane. See - * drm_plane_create_solid_fill_property(). - */ - struct drm_property *solid_fill_property; - /** * @alpha_property: * Optional alpha property for this plane. See diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index cbd943a8c88b..95630f170110 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -259,30 +259,6 @@ struct drm_mode_modeinfo { char name[DRM_DISPLAY_MODE_LEN]; }; -/** - * struct drm_mode_solid_fill - User info for solid fill planes - * - * This is the userspace API solid fill information structure. - * - * Userspace can enable solid fill planes by assigning the plane "solid_fill" - * property to a blob containing a single drm_mode_solid_fill struct populated with an RGB323232 - * color and setting the pixel source to "SOLID_FILL". - * - * For information on the plane property, see drm_plane_create_solid_fill_property() - * - * @r: Red color value of single pixel - * @g: Green color value of single pixel - * @b: Blue color value of single pixel - * @pad: padding, must be zero - */ -struct drm_mode_solid_fill { - __u32 r; - __u32 g; - __u32 b; - __u32 pad; -}; - - struct drm_mode_card_res { __u64 fb_id_ptr; __u64 crtc_id_ptr; -- cgit v1.2.3 From 44a88fa45665318473bfdbb832eba1da2d0a3740 Mon Sep 17 00:00:00 2001 From: Connor Abbott Date: Thu, 7 Dec 2023 21:30:48 +0000 Subject: drm/msm: Add param for the highest bank bit This parameter is programmed by the kernel and influences the tiling layout of images. Exposing it to userspace will allow it to tile/untile images correctly without guessing what value the kernel programmed, and allow us to change it in the future without breaking userspace. Signed-off-by: Connor Abbott Patchwork: https://patchwork.freedesktop.org/patch/571181/ Signed-off-by: Rob Clark --- drivers/gpu/drm/msm/adreno/adreno_gpu.c | 3 +++ include/uapi/drm/msm_drm.h | 1 + 2 files changed, 4 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index 3fe9fd240cc7..074fb498706f 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -373,6 +373,9 @@ int adreno_get_param(struct msm_gpu *gpu, struct msm_file_private *ctx, return -EINVAL; *value = ctx->aspace->va_size; return 0; + case MSM_PARAM_HIGHEST_BANK_BIT: + *value = adreno_gpu->ubwc_config.highest_bank_bit; + return 0; default: DBG("%s: invalid param: %u", gpu->name, param); return -EINVAL; diff --git a/include/uapi/drm/msm_drm.h b/include/uapi/drm/msm_drm.h index 6f2a7ad04aa4..d8a6b3472760 100644 --- a/include/uapi/drm/msm_drm.h +++ b/include/uapi/drm/msm_drm.h @@ -86,6 +86,7 @@ struct drm_msm_timespec { #define MSM_PARAM_CMDLINE 0x0d /* WO: override for task cmdline */ #define MSM_PARAM_VA_START 0x0e /* RO: start of valid GPU iova range */ #define MSM_PARAM_VA_SIZE 0x0f /* RO: size of valid GPU iova range (bytes) */ +#define MSM_PARAM_HIGHEST_BANK_BIT 0x10 /* RO */ /* For backwards compat. The original support for preemption was based on * a single ring per priority level so # of priority levels equals the # -- cgit v1.2.3 From dd08ebf6c3525a7ea2186e636df064ea47281987 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 30 Mar 2023 17:31:57 -0400 Subject: drm/xe: Introduce a new DRM driver for Intel GPUs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Xe, is a new driver for Intel GPUs that supports both integrated and discrete platforms starting with Tiger Lake (first Intel Xe Architecture). The code is at a stage where it is already functional and has experimental support for multiple platforms starting from Tiger Lake, with initial support implemented in Mesa (for Iris and Anv, our OpenGL and Vulkan drivers), as well as in NEO (for OpenCL and Level0). The new Xe driver leverages a lot from i915. As for display, the intent is to share the display code with the i915 driver so that there is maximum reuse there. But it is not added in this patch. This initial work is a collaboration of many people and unfortunately the big squashed patch won't fully honor the proper credits. But let's get some git quick stats so we can at least try to preserve some of the credits: Co-developed-by: Matthew Brost Co-developed-by: Matthew Auld Co-developed-by: Matt Roper Co-developed-by: Thomas Hellström Co-developed-by: Francois Dugast Co-developed-by: Lucas De Marchi Co-developed-by: Maarten Lankhorst Co-developed-by: Philippe Lecluse Co-developed-by: Nirmoy Das Co-developed-by: Jani Nikula Co-developed-by: José Roberto de Souza Co-developed-by: Rodrigo Vivi Co-developed-by: Dave Airlie Co-developed-by: Faith Ekstrand Co-developed-by: Daniel Vetter Co-developed-by: Mauro Carvalho Chehab Signed-off-by: Rodrigo Vivi Signed-off-by: Matthew Brost --- Documentation/gpu/drivers.rst | 1 + Documentation/gpu/xe/index.rst | 23 + Documentation/gpu/xe/xe_cs.rst | 8 + Documentation/gpu/xe/xe_firmware.rst | 34 + Documentation/gpu/xe/xe_gt_mcr.rst | 13 + Documentation/gpu/xe/xe_map.rst | 8 + Documentation/gpu/xe/xe_migrate.rst | 8 + Documentation/gpu/xe/xe_mm.rst | 14 + Documentation/gpu/xe/xe_pcode.rst | 14 + Documentation/gpu/xe/xe_pm.rst | 14 + Documentation/gpu/xe/xe_rtp.rst | 20 + Documentation/gpu/xe/xe_wa.rst | 14 + drivers/gpu/drm/Kconfig | 2 + drivers/gpu/drm/Makefile | 1 + drivers/gpu/drm/xe/.gitignore | 2 + drivers/gpu/drm/xe/Kconfig | 63 + drivers/gpu/drm/xe/Kconfig.debug | 96 + drivers/gpu/drm/xe/Makefile | 121 + drivers/gpu/drm/xe/abi/guc_actions_abi.h | 219 ++ drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h | 249 ++ drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h | 189 ++ .../gpu/drm/xe/abi/guc_communication_mmio_abi.h | 49 + drivers/gpu/drm/xe/abi/guc_errors_abi.h | 37 + drivers/gpu/drm/xe/abi/guc_klvs_abi.h | 322 ++ drivers/gpu/drm/xe/abi/guc_messages_abi.h | 234 ++ drivers/gpu/drm/xe/tests/Makefile | 4 + drivers/gpu/drm/xe/tests/xe_bo.c | 303 ++ drivers/gpu/drm/xe/tests/xe_bo_test.c | 25 + drivers/gpu/drm/xe/tests/xe_dma_buf.c | 259 ++ drivers/gpu/drm/xe/tests/xe_dma_buf_test.c | 23 + drivers/gpu/drm/xe/tests/xe_migrate.c | 378 +++ drivers/gpu/drm/xe/tests/xe_migrate_test.c | 23 + drivers/gpu/drm/xe/tests/xe_test.h | 66 + drivers/gpu/drm/xe/xe_bb.c | 97 + drivers/gpu/drm/xe/xe_bb.h | 27 + drivers/gpu/drm/xe/xe_bb_types.h | 20 + drivers/gpu/drm/xe/xe_bo.c | 1698 ++++++++++ drivers/gpu/drm/xe/xe_bo.h | 290 ++ drivers/gpu/drm/xe/xe_bo_doc.h | 179 + drivers/gpu/drm/xe/xe_bo_evict.c | 225 ++ drivers/gpu/drm/xe/xe_bo_evict.h | 15 + drivers/gpu/drm/xe/xe_bo_types.h | 73 + drivers/gpu/drm/xe/xe_debugfs.c | 129 + drivers/gpu/drm/xe/xe_debugfs.h | 13 + drivers/gpu/drm/xe/xe_device.c | 359 +++ drivers/gpu/drm/xe/xe_device.h | 126 + drivers/gpu/drm/xe/xe_device_types.h | 214 ++ drivers/gpu/drm/xe/xe_dma_buf.c | 307 ++ drivers/gpu/drm/xe/xe_dma_buf.h | 15 + drivers/gpu/drm/xe/xe_drv.h | 24 + drivers/gpu/drm/xe/xe_engine.c | 734 +++++ drivers/gpu/drm/xe/xe_engine.h | 54 + drivers/gpu/drm/xe/xe_engine_types.h | 208 ++ drivers/gpu/drm/xe/xe_exec.c | 390 +++ drivers/gpu/drm/xe/xe_exec.h | 14 + drivers/gpu/drm/xe/xe_execlist.c | 489 +++ drivers/gpu/drm/xe/xe_execlist.h | 21 + drivers/gpu/drm/xe/xe_execlist_types.h | 49 + drivers/gpu/drm/xe/xe_force_wake.c | 203 ++ drivers/gpu/drm/xe/xe_force_wake.h | 40 + drivers/gpu/drm/xe/xe_force_wake_types.h | 84 + drivers/gpu/drm/xe/xe_ggtt.c | 304 ++ drivers/gpu/drm/xe/xe_ggtt.h | 28 + drivers/gpu/drm/xe/xe_ggtt_types.h | 28 + drivers/gpu/drm/xe/xe_gpu_scheduler.c | 101 + drivers/gpu/drm/xe/xe_gpu_scheduler.h | 73 + drivers/gpu/drm/xe/xe_gpu_scheduler_types.h | 57 + drivers/gpu/drm/xe/xe_gt.c | 830 +++++ drivers/gpu/drm/xe/xe_gt.h | 64 + drivers/gpu/drm/xe/xe_gt_clock.c | 83 + drivers/gpu/drm/xe/xe_gt_clock.h | 13 + drivers/gpu/drm/xe/xe_gt_debugfs.c | 160 + drivers/gpu/drm/xe/xe_gt_debugfs.h | 13 + drivers/gpu/drm/xe/xe_gt_mcr.c | 552 ++++ drivers/gpu/drm/xe/xe_gt_mcr.h | 26 + drivers/gpu/drm/xe/xe_gt_pagefault.c | 750 +++++ drivers/gpu/drm/xe/xe_gt_pagefault.h | 22 + drivers/gpu/drm/xe/xe_gt_sysfs.c | 55 + drivers/gpu/drm/xe/xe_gt_sysfs.h | 19 + drivers/gpu/drm/xe/xe_gt_sysfs_types.h | 26 + drivers/gpu/drm/xe/xe_gt_topology.c | 144 + drivers/gpu/drm/xe/xe_gt_topology.h | 20 + drivers/gpu/drm/xe/xe_gt_types.h | 320 ++ drivers/gpu/drm/xe/xe_guc.c | 875 +++++ drivers/gpu/drm/xe/xe_guc.h | 57 + drivers/gpu/drm/xe/xe_guc_ads.c | 676 ++++ drivers/gpu/drm/xe/xe_guc_ads.h | 17 + drivers/gpu/drm/xe/xe_guc_ads_types.h | 25 + drivers/gpu/drm/xe/xe_guc_ct.c | 1196 +++++++ drivers/gpu/drm/xe/xe_guc_ct.h | 62 + drivers/gpu/drm/xe/xe_guc_ct_types.h | 87 + drivers/gpu/drm/xe/xe_guc_debugfs.c | 105 + drivers/gpu/drm/xe/xe_guc_debugfs.h | 14 + drivers/gpu/drm/xe/xe_guc_engine_types.h | 52 + drivers/gpu/drm/xe/xe_guc_fwif.h | 392 +++ drivers/gpu/drm/xe/xe_guc_hwconfig.c | 125 + drivers/gpu/drm/xe/xe_guc_hwconfig.h | 17 + drivers/gpu/drm/xe/xe_guc_log.c | 109 + drivers/gpu/drm/xe/xe_guc_log.h | 48 + drivers/gpu/drm/xe/xe_guc_log_types.h | 23 + drivers/gpu/drm/xe/xe_guc_pc.c | 843 +++++ drivers/gpu/drm/xe/xe_guc_pc.h | 15 + drivers/gpu/drm/xe/xe_guc_pc_types.h | 34 + drivers/gpu/drm/xe/xe_guc_reg.h | 147 + drivers/gpu/drm/xe/xe_guc_submit.c | 1695 ++++++++++ drivers/gpu/drm/xe/xe_guc_submit.h | 30 + drivers/gpu/drm/xe/xe_guc_types.h | 71 + drivers/gpu/drm/xe/xe_huc.c | 131 + drivers/gpu/drm/xe/xe_huc.h | 19 + drivers/gpu/drm/xe/xe_huc_debugfs.c | 71 + drivers/gpu/drm/xe/xe_huc_debugfs.h | 14 + drivers/gpu/drm/xe/xe_huc_types.h | 19 + drivers/gpu/drm/xe/xe_hw_engine.c | 658 ++++ drivers/gpu/drm/xe/xe_hw_engine.h | 27 + drivers/gpu/drm/xe/xe_hw_engine_types.h | 107 + drivers/gpu/drm/xe/xe_hw_fence.c | 230 ++ drivers/gpu/drm/xe/xe_hw_fence.h | 27 + drivers/gpu/drm/xe/xe_hw_fence_types.h | 72 + drivers/gpu/drm/xe/xe_irq.c | 565 ++++ drivers/gpu/drm/xe/xe_irq.h | 18 + drivers/gpu/drm/xe/xe_lrc.c | 841 +++++ drivers/gpu/drm/xe/xe_lrc.h | 50 + drivers/gpu/drm/xe/xe_lrc_types.h | 47 + drivers/gpu/drm/xe/xe_macros.h | 20 + drivers/gpu/drm/xe/xe_map.h | 93 + drivers/gpu/drm/xe/xe_migrate.c | 1168 +++++++ drivers/gpu/drm/xe/xe_migrate.h | 88 + drivers/gpu/drm/xe/xe_migrate_doc.h | 88 + drivers/gpu/drm/xe/xe_mmio.c | 466 +++ drivers/gpu/drm/xe/xe_mmio.h | 110 + drivers/gpu/drm/xe/xe_mocs.c | 557 ++++ drivers/gpu/drm/xe/xe_mocs.h | 29 + drivers/gpu/drm/xe/xe_module.c | 76 + drivers/gpu/drm/xe/xe_module.h | 13 + drivers/gpu/drm/xe/xe_pci.c | 651 ++++ drivers/gpu/drm/xe/xe_pci.h | 21 + drivers/gpu/drm/xe/xe_pcode.c | 296 ++ drivers/gpu/drm/xe/xe_pcode.h | 25 + drivers/gpu/drm/xe/xe_pcode_api.h | 40 + drivers/gpu/drm/xe/xe_platform_types.h | 32 + drivers/gpu/drm/xe/xe_pm.c | 207 ++ drivers/gpu/drm/xe/xe_pm.h | 24 + drivers/gpu/drm/xe/xe_preempt_fence.c | 157 + drivers/gpu/drm/xe/xe_preempt_fence.h | 61 + drivers/gpu/drm/xe/xe_preempt_fence_types.h | 33 + drivers/gpu/drm/xe/xe_pt.c | 1542 +++++++++ drivers/gpu/drm/xe/xe_pt.h | 54 + drivers/gpu/drm/xe/xe_pt_types.h | 57 + drivers/gpu/drm/xe/xe_pt_walk.c | 160 + drivers/gpu/drm/xe/xe_pt_walk.h | 161 + drivers/gpu/drm/xe/xe_query.c | 387 +++ drivers/gpu/drm/xe/xe_query.h | 14 + drivers/gpu/drm/xe/xe_reg_sr.c | 248 ++ drivers/gpu/drm/xe/xe_reg_sr.h | 28 + drivers/gpu/drm/xe/xe_reg_sr_types.h | 44 + drivers/gpu/drm/xe/xe_reg_whitelist.c | 73 + drivers/gpu/drm/xe/xe_reg_whitelist.h | 13 + drivers/gpu/drm/xe/xe_res_cursor.h | 226 ++ drivers/gpu/drm/xe/xe_ring_ops.c | 373 +++ drivers/gpu/drm/xe/xe_ring_ops.h | 17 + drivers/gpu/drm/xe/xe_ring_ops_types.h | 22 + drivers/gpu/drm/xe/xe_rtp.c | 144 + drivers/gpu/drm/xe/xe_rtp.h | 340 ++ drivers/gpu/drm/xe/xe_rtp_types.h | 105 + drivers/gpu/drm/xe/xe_sa.c | 96 + drivers/gpu/drm/xe/xe_sa.h | 42 + drivers/gpu/drm/xe/xe_sa_types.h | 19 + drivers/gpu/drm/xe/xe_sched_job.c | 246 ++ drivers/gpu/drm/xe/xe_sched_job.h | 76 + drivers/gpu/drm/xe/xe_sched_job_types.h | 46 + drivers/gpu/drm/xe/xe_step.c | 189 ++ drivers/gpu/drm/xe/xe_step.h | 18 + drivers/gpu/drm/xe/xe_step_types.h | 51 + drivers/gpu/drm/xe/xe_sync.c | 276 ++ drivers/gpu/drm/xe/xe_sync.h | 27 + drivers/gpu/drm/xe/xe_sync_types.h | 27 + drivers/gpu/drm/xe/xe_trace.c | 9 + drivers/gpu/drm/xe/xe_trace.h | 513 +++ drivers/gpu/drm/xe/xe_ttm_gtt_mgr.c | 130 + drivers/gpu/drm/xe/xe_ttm_gtt_mgr.h | 16 + drivers/gpu/drm/xe/xe_ttm_gtt_mgr_types.h | 18 + drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 403 +++ drivers/gpu/drm/xe/xe_ttm_vram_mgr.h | 41 + drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h | 44 + drivers/gpu/drm/xe/xe_tuning.c | 39 + drivers/gpu/drm/xe/xe_tuning.h | 13 + drivers/gpu/drm/xe/xe_uc.c | 226 ++ drivers/gpu/drm/xe/xe_uc.h | 21 + drivers/gpu/drm/xe/xe_uc_debugfs.c | 26 + drivers/gpu/drm/xe/xe_uc_debugfs.h | 14 + drivers/gpu/drm/xe/xe_uc_fw.c | 406 +++ drivers/gpu/drm/xe/xe_uc_fw.h | 180 ++ drivers/gpu/drm/xe/xe_uc_fw_abi.h | 81 + drivers/gpu/drm/xe/xe_uc_fw_types.h | 112 + drivers/gpu/drm/xe/xe_uc_types.h | 25 + drivers/gpu/drm/xe/xe_vm.c | 3407 ++++++++++++++++++++ drivers/gpu/drm/xe/xe_vm.h | 141 + drivers/gpu/drm/xe/xe_vm_doc.h | 555 ++++ drivers/gpu/drm/xe/xe_vm_madvise.c | 347 ++ drivers/gpu/drm/xe/xe_vm_madvise.h | 15 + drivers/gpu/drm/xe/xe_vm_types.h | 337 ++ drivers/gpu/drm/xe/xe_wa.c | 326 ++ drivers/gpu/drm/xe/xe_wa.h | 18 + drivers/gpu/drm/xe/xe_wait_user_fence.c | 202 ++ drivers/gpu/drm/xe/xe_wait_user_fence.h | 15 + drivers/gpu/drm/xe/xe_wopcm.c | 263 ++ drivers/gpu/drm/xe/xe_wopcm.h | 16 + drivers/gpu/drm/xe/xe_wopcm_types.h | 26 + include/drm/xe_pciids.h | 195 ++ include/uapi/drm/xe_drm.h | 787 +++++ 210 files changed, 40575 insertions(+) create mode 100644 Documentation/gpu/xe/index.rst create mode 100644 Documentation/gpu/xe/xe_cs.rst create mode 100644 Documentation/gpu/xe/xe_firmware.rst create mode 100644 Documentation/gpu/xe/xe_gt_mcr.rst create mode 100644 Documentation/gpu/xe/xe_map.rst create mode 100644 Documentation/gpu/xe/xe_migrate.rst create mode 100644 Documentation/gpu/xe/xe_mm.rst create mode 100644 Documentation/gpu/xe/xe_pcode.rst create mode 100644 Documentation/gpu/xe/xe_pm.rst create mode 100644 Documentation/gpu/xe/xe_rtp.rst create mode 100644 Documentation/gpu/xe/xe_wa.rst create mode 100644 drivers/gpu/drm/xe/.gitignore create mode 100644 drivers/gpu/drm/xe/Kconfig create mode 100644 drivers/gpu/drm/xe/Kconfig.debug create mode 100644 drivers/gpu/drm/xe/Makefile create mode 100644 drivers/gpu/drm/xe/abi/guc_actions_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_communication_mmio_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_errors_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_klvs_abi.h create mode 100644 drivers/gpu/drm/xe/abi/guc_messages_abi.h create mode 100644 drivers/gpu/drm/xe/tests/Makefile create mode 100644 drivers/gpu/drm/xe/tests/xe_bo.c create mode 100644 drivers/gpu/drm/xe/tests/xe_bo_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_dma_buf.c create mode 100644 drivers/gpu/drm/xe/tests/xe_dma_buf_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_migrate.c create mode 100644 drivers/gpu/drm/xe/tests/xe_migrate_test.c create mode 100644 drivers/gpu/drm/xe/tests/xe_test.h create mode 100644 drivers/gpu/drm/xe/xe_bb.c create mode 100644 drivers/gpu/drm/xe/xe_bb.h create mode 100644 drivers/gpu/drm/xe/xe_bb_types.h create mode 100644 drivers/gpu/drm/xe/xe_bo.c create mode 100644 drivers/gpu/drm/xe/xe_bo.h create mode 100644 drivers/gpu/drm/xe/xe_bo_doc.h create mode 100644 drivers/gpu/drm/xe/xe_bo_evict.c create mode 100644 drivers/gpu/drm/xe/xe_bo_evict.h create mode 100644 drivers/gpu/drm/xe/xe_bo_types.h create mode 100644 drivers/gpu/drm/xe/xe_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_device.c create mode 100644 drivers/gpu/drm/xe/xe_device.h create mode 100644 drivers/gpu/drm/xe/xe_device_types.h create mode 100644 drivers/gpu/drm/xe/xe_dma_buf.c create mode 100644 drivers/gpu/drm/xe/xe_dma_buf.h create mode 100644 drivers/gpu/drm/xe/xe_drv.h create mode 100644 drivers/gpu/drm/xe/xe_engine.c create mode 100644 drivers/gpu/drm/xe/xe_engine.h create mode 100644 drivers/gpu/drm/xe/xe_engine_types.h create mode 100644 drivers/gpu/drm/xe/xe_exec.c create mode 100644 drivers/gpu/drm/xe/xe_exec.h create mode 100644 drivers/gpu/drm/xe/xe_execlist.c create mode 100644 drivers/gpu/drm/xe/xe_execlist.h create mode 100644 drivers/gpu/drm/xe/xe_execlist_types.h create mode 100644 drivers/gpu/drm/xe/xe_force_wake.c create mode 100644 drivers/gpu/drm/xe/xe_force_wake.h create mode 100644 drivers/gpu/drm/xe/xe_force_wake_types.h create mode 100644 drivers/gpu/drm/xe/xe_ggtt.c create mode 100644 drivers/gpu/drm/xe/xe_ggtt.h create mode 100644 drivers/gpu/drm/xe/xe_ggtt_types.h create mode 100644 drivers/gpu/drm/xe/xe_gpu_scheduler.c create mode 100644 drivers/gpu/drm/xe/xe_gpu_scheduler.h create mode 100644 drivers/gpu/drm/xe/xe_gpu_scheduler_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt.c create mode 100644 drivers/gpu/drm/xe/xe_gt.h create mode 100644 drivers/gpu/drm/xe/xe_gt_clock.c create mode 100644 drivers/gpu/drm/xe/xe_gt_clock.h create mode 100644 drivers/gpu/drm/xe/xe_gt_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_gt_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_gt_mcr.c create mode 100644 drivers/gpu/drm/xe/xe_gt_mcr.h create mode 100644 drivers/gpu/drm/xe/xe_gt_pagefault.c create mode 100644 drivers/gpu/drm/xe/xe_gt_pagefault.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sysfs.c create mode 100644 drivers/gpu/drm/xe/xe_gt_sysfs.h create mode 100644 drivers/gpu/drm/xe/xe_gt_sysfs_types.h create mode 100644 drivers/gpu/drm/xe/xe_gt_topology.c create mode 100644 drivers/gpu/drm/xe/xe_gt_topology.h create mode 100644 drivers/gpu/drm/xe/xe_gt_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc.c create mode 100644 drivers/gpu/drm/xe/xe_guc.h create mode 100644 drivers/gpu/drm/xe/xe_guc_ads.c create mode 100644 drivers/gpu/drm/xe/xe_guc_ads.h create mode 100644 drivers/gpu/drm/xe/xe_guc_ads_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_ct.c create mode 100644 drivers/gpu/drm/xe/xe_guc_ct.h create mode 100644 drivers/gpu/drm/xe/xe_guc_ct_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_guc_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_guc_engine_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_fwif.h create mode 100644 drivers/gpu/drm/xe/xe_guc_hwconfig.c create mode 100644 drivers/gpu/drm/xe/xe_guc_hwconfig.h create mode 100644 drivers/gpu/drm/xe/xe_guc_log.c create mode 100644 drivers/gpu/drm/xe/xe_guc_log.h create mode 100644 drivers/gpu/drm/xe/xe_guc_log_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_pc.c create mode 100644 drivers/gpu/drm/xe/xe_guc_pc.h create mode 100644 drivers/gpu/drm/xe/xe_guc_pc_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_reg.h create mode 100644 drivers/gpu/drm/xe/xe_guc_submit.c create mode 100644 drivers/gpu/drm/xe/xe_guc_submit.h create mode 100644 drivers/gpu/drm/xe/xe_guc_types.h create mode 100644 drivers/gpu/drm/xe/xe_huc.c create mode 100644 drivers/gpu/drm/xe/xe_huc.h create mode 100644 drivers/gpu/drm/xe/xe_huc_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_huc_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_huc_types.h create mode 100644 drivers/gpu/drm/xe/xe_hw_engine.c create mode 100644 drivers/gpu/drm/xe/xe_hw_engine.h create mode 100644 drivers/gpu/drm/xe/xe_hw_engine_types.h create mode 100644 drivers/gpu/drm/xe/xe_hw_fence.c create mode 100644 drivers/gpu/drm/xe/xe_hw_fence.h create mode 100644 drivers/gpu/drm/xe/xe_hw_fence_types.h create mode 100644 drivers/gpu/drm/xe/xe_irq.c create mode 100644 drivers/gpu/drm/xe/xe_irq.h create mode 100644 drivers/gpu/drm/xe/xe_lrc.c create mode 100644 drivers/gpu/drm/xe/xe_lrc.h create mode 100644 drivers/gpu/drm/xe/xe_lrc_types.h create mode 100644 drivers/gpu/drm/xe/xe_macros.h create mode 100644 drivers/gpu/drm/xe/xe_map.h create mode 100644 drivers/gpu/drm/xe/xe_migrate.c create mode 100644 drivers/gpu/drm/xe/xe_migrate.h create mode 100644 drivers/gpu/drm/xe/xe_migrate_doc.h create mode 100644 drivers/gpu/drm/xe/xe_mmio.c create mode 100644 drivers/gpu/drm/xe/xe_mmio.h create mode 100644 drivers/gpu/drm/xe/xe_mocs.c create mode 100644 drivers/gpu/drm/xe/xe_mocs.h create mode 100644 drivers/gpu/drm/xe/xe_module.c create mode 100644 drivers/gpu/drm/xe/xe_module.h create mode 100644 drivers/gpu/drm/xe/xe_pci.c create mode 100644 drivers/gpu/drm/xe/xe_pci.h create mode 100644 drivers/gpu/drm/xe/xe_pcode.c create mode 100644 drivers/gpu/drm/xe/xe_pcode.h create mode 100644 drivers/gpu/drm/xe/xe_pcode_api.h create mode 100644 drivers/gpu/drm/xe/xe_platform_types.h create mode 100644 drivers/gpu/drm/xe/xe_pm.c create mode 100644 drivers/gpu/drm/xe/xe_pm.h create mode 100644 drivers/gpu/drm/xe/xe_preempt_fence.c create mode 100644 drivers/gpu/drm/xe/xe_preempt_fence.h create mode 100644 drivers/gpu/drm/xe/xe_preempt_fence_types.h create mode 100644 drivers/gpu/drm/xe/xe_pt.c create mode 100644 drivers/gpu/drm/xe/xe_pt.h create mode 100644 drivers/gpu/drm/xe/xe_pt_types.h create mode 100644 drivers/gpu/drm/xe/xe_pt_walk.c create mode 100644 drivers/gpu/drm/xe/xe_pt_walk.h create mode 100644 drivers/gpu/drm/xe/xe_query.c create mode 100644 drivers/gpu/drm/xe/xe_query.h create mode 100644 drivers/gpu/drm/xe/xe_reg_sr.c create mode 100644 drivers/gpu/drm/xe/xe_reg_sr.h create mode 100644 drivers/gpu/drm/xe/xe_reg_sr_types.h create mode 100644 drivers/gpu/drm/xe/xe_reg_whitelist.c create mode 100644 drivers/gpu/drm/xe/xe_reg_whitelist.h create mode 100644 drivers/gpu/drm/xe/xe_res_cursor.h create mode 100644 drivers/gpu/drm/xe/xe_ring_ops.c create mode 100644 drivers/gpu/drm/xe/xe_ring_ops.h create mode 100644 drivers/gpu/drm/xe/xe_ring_ops_types.h create mode 100644 drivers/gpu/drm/xe/xe_rtp.c create mode 100644 drivers/gpu/drm/xe/xe_rtp.h create mode 100644 drivers/gpu/drm/xe/xe_rtp_types.h create mode 100644 drivers/gpu/drm/xe/xe_sa.c create mode 100644 drivers/gpu/drm/xe/xe_sa.h create mode 100644 drivers/gpu/drm/xe/xe_sa_types.h create mode 100644 drivers/gpu/drm/xe/xe_sched_job.c create mode 100644 drivers/gpu/drm/xe/xe_sched_job.h create mode 100644 drivers/gpu/drm/xe/xe_sched_job_types.h create mode 100644 drivers/gpu/drm/xe/xe_step.c create mode 100644 drivers/gpu/drm/xe/xe_step.h create mode 100644 drivers/gpu/drm/xe/xe_step_types.h create mode 100644 drivers/gpu/drm/xe/xe_sync.c create mode 100644 drivers/gpu/drm/xe/xe_sync.h create mode 100644 drivers/gpu/drm/xe/xe_sync_types.h create mode 100644 drivers/gpu/drm/xe/xe_trace.c create mode 100644 drivers/gpu/drm/xe/xe_trace.h create mode 100644 drivers/gpu/drm/xe/xe_ttm_gtt_mgr.c create mode 100644 drivers/gpu/drm/xe/xe_ttm_gtt_mgr.h create mode 100644 drivers/gpu/drm/xe/xe_ttm_gtt_mgr_types.h create mode 100644 drivers/gpu/drm/xe/xe_ttm_vram_mgr.c create mode 100644 drivers/gpu/drm/xe/xe_ttm_vram_mgr.h create mode 100644 drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h create mode 100644 drivers/gpu/drm/xe/xe_tuning.c create mode 100644 drivers/gpu/drm/xe/xe_tuning.h create mode 100644 drivers/gpu/drm/xe/xe_uc.c create mode 100644 drivers/gpu/drm/xe/xe_uc.h create mode 100644 drivers/gpu/drm/xe/xe_uc_debugfs.c create mode 100644 drivers/gpu/drm/xe/xe_uc_debugfs.h create mode 100644 drivers/gpu/drm/xe/xe_uc_fw.c create mode 100644 drivers/gpu/drm/xe/xe_uc_fw.h create mode 100644 drivers/gpu/drm/xe/xe_uc_fw_abi.h create mode 100644 drivers/gpu/drm/xe/xe_uc_fw_types.h create mode 100644 drivers/gpu/drm/xe/xe_uc_types.h create mode 100644 drivers/gpu/drm/xe/xe_vm.c create mode 100644 drivers/gpu/drm/xe/xe_vm.h create mode 100644 drivers/gpu/drm/xe/xe_vm_doc.h create mode 100644 drivers/gpu/drm/xe/xe_vm_madvise.c create mode 100644 drivers/gpu/drm/xe/xe_vm_madvise.h create mode 100644 drivers/gpu/drm/xe/xe_vm_types.h create mode 100644 drivers/gpu/drm/xe/xe_wa.c create mode 100644 drivers/gpu/drm/xe/xe_wa.h create mode 100644 drivers/gpu/drm/xe/xe_wait_user_fence.c create mode 100644 drivers/gpu/drm/xe/xe_wait_user_fence.h create mode 100644 drivers/gpu/drm/xe/xe_wopcm.c create mode 100644 drivers/gpu/drm/xe/xe_wopcm.h create mode 100644 drivers/gpu/drm/xe/xe_wopcm_types.h create mode 100644 include/drm/xe_pciids.h create mode 100644 include/uapi/drm/xe_drm.h (limited to 'include/uapi') diff --git a/Documentation/gpu/drivers.rst b/Documentation/gpu/drivers.rst index cc6535f5f28c..b899cbc5c2b4 100644 --- a/Documentation/gpu/drivers.rst +++ b/Documentation/gpu/drivers.rst @@ -18,6 +18,7 @@ GPU Driver Documentation vkms bridge/dw-hdmi xen-front + xe/index afbc komeda-kms panfrost diff --git a/Documentation/gpu/xe/index.rst b/Documentation/gpu/xe/index.rst new file mode 100644 index 000000000000..2fddf9ed251e --- /dev/null +++ b/Documentation/gpu/xe/index.rst @@ -0,0 +1,23 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +======================= +drm/xe Intel GFX Driver +======================= + +The drm/xe driver supports some future GFX cards with rendering, display, +compute and media. Support for currently available platforms like TGL, ADL, +DG2, etc is provided to prototype the driver. + +.. toctree:: + :titlesonly: + + xe_mm + xe_map + xe_migrate + xe_cs + xe_pm + xe_pcode + xe_gt_mcr + xe_wa + xe_rtp + xe_firmware diff --git a/Documentation/gpu/xe/xe_cs.rst b/Documentation/gpu/xe/xe_cs.rst new file mode 100644 index 000000000000..e379aed4f5a8 --- /dev/null +++ b/Documentation/gpu/xe/xe_cs.rst @@ -0,0 +1,8 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +================== +Command submission +================== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_exec.c + :doc: Execbuf (User GPU command submission) diff --git a/Documentation/gpu/xe/xe_firmware.rst b/Documentation/gpu/xe/xe_firmware.rst new file mode 100644 index 000000000000..c01246ae99f5 --- /dev/null +++ b/Documentation/gpu/xe/xe_firmware.rst @@ -0,0 +1,34 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +======== +Firmware +======== + +Firmware Layout +=============== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_uc_fw_abi.h + :doc: Firmware Layout + +Write Once Protected Content Memory (WOPCM) Layout +================================================== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_wopcm.c + :doc: Write Once Protected Content Memory (WOPCM) Layout + +GuC CTB Blob +============ + +.. kernel-doc:: drivers/gpu/drm/xe/xe_guc_ct.c + :doc: GuC CTB Blob + +GuC Power Conservation (PC) +=========================== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_guc_pc.c + :doc: GuC Power Conservation (PC) + +Internal API +============ + +TODO diff --git a/Documentation/gpu/xe/xe_gt_mcr.rst b/Documentation/gpu/xe/xe_gt_mcr.rst new file mode 100644 index 000000000000..848c07bc36d0 --- /dev/null +++ b/Documentation/gpu/xe/xe_gt_mcr.rst @@ -0,0 +1,13 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +============================================== +GT Multicast/Replicated (MCR) Register Support +============================================== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_gt_mcr.c + :doc: GT Multicast/Replicated (MCR) Register Support + +Internal API +============ + +TODO diff --git a/Documentation/gpu/xe/xe_map.rst b/Documentation/gpu/xe/xe_map.rst new file mode 100644 index 000000000000..a098cfd2df04 --- /dev/null +++ b/Documentation/gpu/xe/xe_map.rst @@ -0,0 +1,8 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +========= +Map Layer +========= + +.. kernel-doc:: drivers/gpu/drm/xe/xe_map.h + :doc: Map layer diff --git a/Documentation/gpu/xe/xe_migrate.rst b/Documentation/gpu/xe/xe_migrate.rst new file mode 100644 index 000000000000..f92faec0ac94 --- /dev/null +++ b/Documentation/gpu/xe/xe_migrate.rst @@ -0,0 +1,8 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +============= +Migrate Layer +============= + +.. kernel-doc:: drivers/gpu/drm/xe/xe_migrate_doc.h + :doc: Migrate Layer diff --git a/Documentation/gpu/xe/xe_mm.rst b/Documentation/gpu/xe/xe_mm.rst new file mode 100644 index 000000000000..6c8fd8b4a466 --- /dev/null +++ b/Documentation/gpu/xe/xe_mm.rst @@ -0,0 +1,14 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +================= +Memory Management +================= + +.. kernel-doc:: drivers/gpu/drm/xe/xe_bo_doc.h + :doc: Buffer Objects (BO) + +Pagetable building +================== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_pt.c + :doc: Pagetable building diff --git a/Documentation/gpu/xe/xe_pcode.rst b/Documentation/gpu/xe/xe_pcode.rst new file mode 100644 index 000000000000..d2e22cc45061 --- /dev/null +++ b/Documentation/gpu/xe/xe_pcode.rst @@ -0,0 +1,14 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +===== +Pcode +===== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_pcode.c + :doc: PCODE + +Internal API +============ + +.. kernel-doc:: drivers/gpu/drm/xe/xe_pcode.c + :internal: diff --git a/Documentation/gpu/xe/xe_pm.rst b/Documentation/gpu/xe/xe_pm.rst new file mode 100644 index 000000000000..6781cdfb24f6 --- /dev/null +++ b/Documentation/gpu/xe/xe_pm.rst @@ -0,0 +1,14 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +======================== +Runtime Power Management +======================== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_pm.c + :doc: Xe Power Management + +Internal API +============ + +.. kernel-doc:: drivers/gpu/drm/xe/xe_pm.c + :internal: diff --git a/Documentation/gpu/xe/xe_rtp.rst b/Documentation/gpu/xe/xe_rtp.rst new file mode 100644 index 000000000000..7fdf4b6c1a04 --- /dev/null +++ b/Documentation/gpu/xe/xe_rtp.rst @@ -0,0 +1,20 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +========================= +Register Table Processing +========================= + +.. kernel-doc:: drivers/gpu/drm/xe/xe_rtp.c + :doc: Register Table Processing + +Internal API +============ + +.. kernel-doc:: drivers/gpu/drm/xe/xe_rtp_types.h + :internal: + +.. kernel-doc:: drivers/gpu/drm/xe/xe_rtp.h + :internal: + +.. kernel-doc:: drivers/gpu/drm/xe/xe_rtp.c + :internal: diff --git a/Documentation/gpu/xe/xe_wa.rst b/Documentation/gpu/xe/xe_wa.rst new file mode 100644 index 000000000000..f8811cc6adcc --- /dev/null +++ b/Documentation/gpu/xe/xe_wa.rst @@ -0,0 +1,14 @@ +.. SPDX-License-Identifier: (GPL-2.0+ OR MIT) + +==================== +Hardware workarounds +==================== + +.. kernel-doc:: drivers/gpu/drm/xe/xe_wa.c + :doc: Hardware workarounds + +Internal API +============ + +.. kernel-doc:: drivers/gpu/drm/xe/xe_wa.c + :internal: diff --git a/drivers/gpu/drm/Kconfig b/drivers/gpu/drm/Kconfig index 31cfe2c2a2af..2520db0b776e 100644 --- a/drivers/gpu/drm/Kconfig +++ b/drivers/gpu/drm/Kconfig @@ -276,6 +276,8 @@ source "drivers/gpu/drm/nouveau/Kconfig" source "drivers/gpu/drm/i915/Kconfig" +source "drivers/gpu/drm/xe/Kconfig" + source "drivers/gpu/drm/kmb/Kconfig" config DRM_VGEM diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index 8ac6f4b9546e..104b42df2e95 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -134,6 +134,7 @@ obj-$(CONFIG_DRM_RADEON)+= radeon/ obj-$(CONFIG_DRM_AMDGPU)+= amd/amdgpu/ obj-$(CONFIG_DRM_AMDGPU)+= amd/amdxcp/ obj-$(CONFIG_DRM_I915) += i915/ +obj-$(CONFIG_DRM_XE) += xe/ obj-$(CONFIG_DRM_KMB_DISPLAY) += kmb/ obj-$(CONFIG_DRM_MGAG200) += mgag200/ obj-$(CONFIG_DRM_V3D) += v3d/ diff --git a/drivers/gpu/drm/xe/.gitignore b/drivers/gpu/drm/xe/.gitignore new file mode 100644 index 000000000000..81972dce1aff --- /dev/null +++ b/drivers/gpu/drm/xe/.gitignore @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +*.hdrtest diff --git a/drivers/gpu/drm/xe/Kconfig b/drivers/gpu/drm/xe/Kconfig new file mode 100644 index 000000000000..62f54e6d62d9 --- /dev/null +++ b/drivers/gpu/drm/xe/Kconfig @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: GPL-2.0-only +config DRM_XE + tristate "Intel Xe Graphics" + depends on DRM && PCI && MMU + select INTERVAL_TREE + # we need shmfs for the swappable backing store, and in particular + # the shmem_readpage() which depends upon tmpfs + select SHMEM + select TMPFS + select DRM_BUDDY + select DRM_KMS_HELPER + select DRM_PANEL + select DRM_SUBALLOC_HELPER + select RELAY + select IRQ_WORK + select SYNC_FILE + select IOSF_MBI + select CRC32 + select SND_HDA_I915 if SND_HDA_CORE + select CEC_CORE if CEC_NOTIFIER + select VMAP_PFN + select DRM_TTM + select DRM_TTM_HELPER + select DRM_SCHED + select MMU_NOTIFIER + help + Experimental driver for Intel Xe series GPUs + + If "M" is selected, the module will be called xe. + +config DRM_XE_FORCE_PROBE + string "Force probe xe for selected Intel hardware IDs" + depends on DRM_XE + help + This is the default value for the xe.force_probe module + parameter. Using the module parameter overrides this option. + + Force probe the xe for Intel graphics devices that are + recognized but not properly supported by this kernel version. It is + recommended to upgrade to a kernel version with proper support as soon + as it is available. + + It can also be used to block the probe of recognized and fully + supported devices. + + Use "" to disable force probe. If in doubt, use this. + + Use "[,,...]" to force probe the xe for listed + devices. For example, "4500" or "4500,4571". + + Use "*" to force probe the driver for all known devices. + + Use "!" right before the ID to block the probe of the device. For + example, "4500,!4571" forces the probe of 4500 and blocks the probe of + 4571. + + Use "!*" to block the probe of the driver for all known devices. + +menu "drm/Xe Debugging" +depends on DRM_XE +depends on EXPERT +source "drivers/gpu/drm/xe/Kconfig.debug" +endmenu diff --git a/drivers/gpu/drm/xe/Kconfig.debug b/drivers/gpu/drm/xe/Kconfig.debug new file mode 100644 index 000000000000..b61fd43a76fe --- /dev/null +++ b/drivers/gpu/drm/xe/Kconfig.debug @@ -0,0 +1,96 @@ +# SPDX-License-Identifier: GPL-2.0-only +config DRM_XE_WERROR + bool "Force GCC to throw an error instead of a warning when compiling" + # As this may inadvertently break the build, only allow the user + # to shoot oneself in the foot iff they aim really hard + depends on EXPERT + # We use the dependency on !COMPILE_TEST to not be enabled in + # allmodconfig or allyesconfig configurations + depends on !COMPILE_TEST + default n + help + Add -Werror to the build flags for (and only for) xe.ko. + Do not enable this unless you are writing code for the xe.ko module. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_DEBUG + bool "Enable additional driver debugging" + depends on DRM_XE + depends on EXPERT + depends on !COMPILE_TEST + default n + help + Choose this option to turn on extra driver debugging that may affect + performance but will catch some internal issues. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_DEBUG_VM + bool "Enable extra VM debugging info" + default n + help + Enable extra VM debugging info + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_DEBUG_MEM + bool "Enable passing SYS/LMEM addresses to user space" + default n + help + Pass object location trough uapi. Intended for extended + testing and development only. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_SIMPLE_ERROR_CAPTURE + bool "Enable simple error capture to dmesg on job timeout" + default n + help + Choose this option when debugging an unexpected job timeout + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_KUNIT_TEST + tristate "KUnit tests for the drm xe driver" if !KUNIT_ALL_TESTS + depends on DRM_XE && KUNIT + default KUNIT_ALL_TESTS + select DRM_EXPORT_FOR_TESTS if m + help + Choose this option to allow the driver to perform selftests under + the kunit framework + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_LARGE_GUC_BUFFER + bool "Enable larger guc log buffer" + default n + help + Choose this option when debugging guc issues. + Buffer should be large enough for complex issues. + + Recommended for driver developers only. + + If in doubt, say "N". + +config DRM_XE_USERPTR_INVAL_INJECT + bool "Inject userptr invalidation -EINVAL errors" + default n + help + Choose this option when debugging error paths that + are hit during checks for userptr invalidations. + + Recomended for driver developers only. + If in doubt, say "N". diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile new file mode 100644 index 000000000000..228a87f2fe7b --- /dev/null +++ b/drivers/gpu/drm/xe/Makefile @@ -0,0 +1,121 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the drm device driver. This driver provides support for the +# Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher. + +# Add a set of useful warning flags and enable -Werror for CI to prevent +# trivial mistakes from creeping in. We have to do this piecemeal as we reject +# any patch that isn't warning clean, so turning on -Wall -Wextra (or W=1) we +# need to filter out dubious warnings. Still it is our interest +# to keep running locally with W=1 C=1 until we are completely clean. +# +# Note the danger in using -Wall -Wextra is that when CI updates gcc we +# will most likely get a sudden build breakage... Hopefully we will fix +# new warnings before CI updates! +subdir-ccflags-y := -Wall -Wextra +# making these call cc-disable-warning breaks when trying to build xe.mod.o +# by calling make M=drivers/gpu/drm/xe. This doesn't happen in upstream tree, +# so it was somehow fixed by the changes in the build system. Move it back to +# $(call cc-disable-warning, ...) after rebase. +subdir-ccflags-y += -Wno-unused-parameter +subdir-ccflags-y += -Wno-type-limits +#subdir-ccflags-y += $(call cc-disable-warning, unused-parameter) +#subdir-ccflags-y += $(call cc-disable-warning, type-limits) +subdir-ccflags-y += $(call cc-disable-warning, missing-field-initializers) +subdir-ccflags-y += $(call cc-disable-warning, unused-but-set-variable) +# clang warnings +subdir-ccflags-y += $(call cc-disable-warning, sign-compare) +subdir-ccflags-y += $(call cc-disable-warning, sometimes-uninitialized) +subdir-ccflags-y += $(call cc-disable-warning, initializer-overrides) +subdir-ccflags-y += $(call cc-disable-warning, frame-address) +subdir-ccflags-$(CONFIG_DRM_XE_WERROR) += -Werror + +# Fine grained warnings disable +CFLAGS_xe_pci.o = $(call cc-disable-warning, override-init) + +subdir-ccflags-y += -I$(srctree)/$(src) + +# Please keep these build lists sorted! + +# core driver code + +xe-y += xe_bb.o \ + xe_bo.o \ + xe_bo_evict.o \ + xe_debugfs.o \ + xe_device.o \ + xe_dma_buf.o \ + xe_engine.o \ + xe_exec.o \ + xe_execlist.o \ + xe_force_wake.o \ + xe_ggtt.o \ + xe_gpu_scheduler.o \ + xe_gt.o \ + xe_gt_clock.o \ + xe_gt_debugfs.o \ + xe_gt_mcr.o \ + xe_gt_pagefault.o \ + xe_gt_sysfs.o \ + xe_gt_topology.o \ + xe_guc.o \ + xe_guc_ads.o \ + xe_guc_ct.o \ + xe_guc_debugfs.o \ + xe_guc_hwconfig.o \ + xe_guc_log.o \ + xe_guc_pc.o \ + xe_guc_submit.o \ + xe_hw_engine.o \ + xe_hw_fence.o \ + xe_huc.o \ + xe_huc_debugfs.o \ + xe_irq.o \ + xe_lrc.o \ + xe_migrate.o \ + xe_mmio.o \ + xe_mocs.o \ + xe_module.o \ + xe_pci.o \ + xe_pcode.o \ + xe_pm.o \ + xe_preempt_fence.o \ + xe_pt.o \ + xe_pt_walk.o \ + xe_query.o \ + xe_reg_sr.o \ + xe_reg_whitelist.o \ + xe_rtp.o \ + xe_ring_ops.o \ + xe_sa.o \ + xe_sched_job.o \ + xe_step.o \ + xe_sync.o \ + xe_trace.o \ + xe_ttm_gtt_mgr.o \ + xe_ttm_vram_mgr.o \ + xe_tuning.o \ + xe_uc.o \ + xe_uc_debugfs.o \ + xe_uc_fw.o \ + xe_vm.o \ + xe_vm_madvise.o \ + xe_wait_user_fence.o \ + xe_wa.o \ + xe_wopcm.o + +# XXX: Needed for i915 register definitions. Will be removed after xe-regs. +subdir-ccflags-y += -I$(srctree)/drivers/gpu/drm/i915/ + +obj-$(CONFIG_DRM_XE) += xe.o +obj-$(CONFIG_DRM_XE_KUNIT_TEST) += tests/ +\ +# header test +always-$(CONFIG_DRM_XE_WERROR) += \ + $(patsubst %.h,%.hdrtest, $(shell cd $(srctree)/$(src) && find * -name '*.h')) + +quiet_cmd_hdrtest = HDRTEST $(patsubst %.hdrtest,%.h,$@) + cmd_hdrtest = $(CC) -DHDRTEST $(filter-out $(CFLAGS_GCOV), $(c_flags)) -S -o /dev/null -x c /dev/null -include $<; touch $@ + +$(obj)/%.hdrtest: $(src)/%.h FORCE + $(call if_changed_dep,hdrtest) diff --git a/drivers/gpu/drm/xe/abi/guc_actions_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_abi.h new file mode 100644 index 000000000000..3062e0e0d467 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_actions_abi.h @@ -0,0 +1,219 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_ACTIONS_ABI_H +#define _ABI_GUC_ACTIONS_ABI_H + +/** + * DOC: HOST2GUC_SELF_CFG + * + * This message is used by Host KMD to setup of the `GuC Self Config KLVs`_. + * + * This message must be sent as `MMIO HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_SELF_CFG` = 0x0508 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:16 | **KLV_KEY** - KLV key, see `GuC Self Config KLVs`_ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **KLV_LEN** - KLV length | + * | | | | + * | | | - 32 bit KLV = 1 | + * | | | - 64 bit KLV = 2 | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **VALUE32** - Bits 31-0 of the KLV value | + * +---+-------+--------------------------------------------------------------+ + * | 3 | 31:0 | **VALUE64** - Bits 63-32 of the KLV value (**KLV_LEN** = 2) | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = **NUM** - 1 if KLV was parsed, 0 if not recognized | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_HOST2GUC_SELF_CFG 0x0508 + +#define HOST2GUC_SELF_CFG_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 3u) +#define HOST2GUC_SELF_CFG_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY (0xffff << 16) +#define HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN (0xffff << 0) +#define HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32 GUC_HXG_REQUEST_MSG_n_DATAn +#define HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64 GUC_HXG_REQUEST_MSG_n_DATAn + +#define HOST2GUC_SELF_CFG_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define HOST2GUC_SELF_CFG_RESPONSE_MSG_0_NUM GUC_HXG_RESPONSE_MSG_0_DATA0 + +/** + * DOC: HOST2GUC_CONTROL_CTB + * + * This H2G action allows Vf Host to enable or disable H2G and G2H `CT Buffer`_. + * + * This message must be sent as `MMIO HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_CONTROL_CTB` = 0x4509 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **CONTROL** - control `CTB based communication`_ | + * | | | | + * | | | - _`GUC_CTB_CONTROL_DISABLE` = 0 | + * | | | - _`GUC_CTB_CONTROL_ENABLE` = 1 | + * +---+-------+--------------------------------------------------------------+ + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_GUC_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | DATA0 = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ +#define GUC_ACTION_HOST2GUC_CONTROL_CTB 0x4509 + +#define HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define HOST2GUC_CONTROL_CTB_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL GUC_HXG_REQUEST_MSG_n_DATAn +#define GUC_CTB_CONTROL_DISABLE 0u +#define GUC_CTB_CONTROL_ENABLE 1u + +#define HOST2GUC_CONTROL_CTB_RESPONSE_MSG_LEN GUC_HXG_RESPONSE_MSG_MIN_LEN +#define HOST2GUC_CONTROL_CTB_RESPONSE_MSG_0_MBZ GUC_HXG_RESPONSE_MSG_0_DATA0 + +/* legacy definitions */ + +enum xe_guc_action { + XE_GUC_ACTION_DEFAULT = 0x0, + XE_GUC_ACTION_REQUEST_PREEMPTION = 0x2, + XE_GUC_ACTION_REQUEST_ENGINE_RESET = 0x3, + XE_GUC_ACTION_ALLOCATE_DOORBELL = 0x10, + XE_GUC_ACTION_DEALLOCATE_DOORBELL = 0x20, + XE_GUC_ACTION_LOG_BUFFER_FILE_FLUSH_COMPLETE = 0x30, + XE_GUC_ACTION_UK_LOG_ENABLE_LOGGING = 0x40, + XE_GUC_ACTION_FORCE_LOG_BUFFER_FLUSH = 0x302, + XE_GUC_ACTION_ENTER_S_STATE = 0x501, + XE_GUC_ACTION_EXIT_S_STATE = 0x502, + XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE = 0x506, + XE_GUC_ACTION_SCHED_CONTEXT = 0x1000, + XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET = 0x1001, + XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE = 0x1002, + XE_GUC_ACTION_SCHED_ENGINE_MODE_SET = 0x1003, + XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE = 0x1004, + XE_GUC_ACTION_SET_CONTEXT_PRIORITY = 0x1005, + XE_GUC_ACTION_SET_CONTEXT_EXECUTION_QUANTUM = 0x1006, + XE_GUC_ACTION_SET_CONTEXT_PREEMPTION_TIMEOUT = 0x1007, + XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION = 0x1008, + XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION = 0x1009, + XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES = 0x100B, + XE_GUC_ACTION_SETUP_PC_GUCRC = 0x3004, + XE_GUC_ACTION_AUTHENTICATE_HUC = 0x4000, + XE_GUC_ACTION_GET_HWCONFIG = 0x4100, + XE_GUC_ACTION_REGISTER_CONTEXT = 0x4502, + XE_GUC_ACTION_DEREGISTER_CONTEXT = 0x4503, + XE_GUC_ACTION_REGISTER_COMMAND_TRANSPORT_BUFFER = 0x4505, + XE_GUC_ACTION_DEREGISTER_COMMAND_TRANSPORT_BUFFER = 0x4506, + XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE = 0x4600, + XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC = 0x4601, + XE_GUC_ACTION_CLIENT_SOFT_RESET = 0x5507, + XE_GUC_ACTION_SET_ENG_UTIL_BUFF = 0x550A, + XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR = 0x6000, + XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC = 0x6002, + XE_GUC_ACTION_PAGE_FAULT_RES_DESC = 0x6003, + XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY = 0x6004, + XE_GUC_ACTION_TLB_INVALIDATION = 0x7000, + XE_GUC_ACTION_TLB_INVALIDATION_DONE = 0x7001, + XE_GUC_ACTION_TLB_INVALIDATION_ALL = 0x7002, + XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION = 0x8002, + XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE = 0x8003, + XE_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED = 0x8004, + XE_GUC_ACTION_NOTIFY_EXCEPTION = 0x8005, + XE_GUC_ACTION_LIMIT +}; + +enum xe_guc_rc_options { + XE_GUCRC_HOST_CONTROL, + XE_GUCRC_FIRMWARE_CONTROL, +}; + +enum xe_guc_preempt_options { + XE_GUC_PREEMPT_OPTION_DROP_WORK_Q = 0x4, + XE_GUC_PREEMPT_OPTION_DROP_SUBMIT_Q = 0x8, +}; + +enum xe_guc_report_status { + XE_GUC_REPORT_STATUS_UNKNOWN = 0x0, + XE_GUC_REPORT_STATUS_ACKED = 0x1, + XE_GUC_REPORT_STATUS_ERROR = 0x2, + XE_GUC_REPORT_STATUS_COMPLETE = 0x4, +}; + +enum xe_guc_sleep_state_status { + XE_GUC_SLEEP_STATE_SUCCESS = 0x1, + XE_GUC_SLEEP_STATE_PREEMPT_TO_IDLE_FAILED = 0x2, + XE_GUC_SLEEP_STATE_ENGINE_RESET_FAILED = 0x3 +#define XE_GUC_SLEEP_STATE_INVALID_MASK 0x80000000 +}; + +#define GUC_LOG_CONTROL_LOGGING_ENABLED (1 << 0) +#define GUC_LOG_CONTROL_VERBOSITY_SHIFT 4 +#define GUC_LOG_CONTROL_VERBOSITY_MASK (0xF << GUC_LOG_CONTROL_VERBOSITY_SHIFT) +#define GUC_LOG_CONTROL_DEFAULT_LOGGING (1 << 8) + +#define XE_GUC_TLB_INVAL_TYPE_SHIFT 0 +#define XE_GUC_TLB_INVAL_MODE_SHIFT 8 +/* Flush PPC or SMRO caches along with TLB invalidation request */ +#define XE_GUC_TLB_INVAL_FLUSH_CACHE (1 << 31) + +enum xe_guc_tlb_invalidation_type { + XE_GUC_TLB_INVAL_FULL = 0x0, + XE_GUC_TLB_INVAL_PAGE_SELECTIVE = 0x1, + XE_GUC_TLB_INVAL_PAGE_SELECTIVE_CTX = 0x2, + XE_GUC_TLB_INVAL_GUC = 0x3, +}; + +/* + * 0: Heavy mode of Invalidation: + * The pipeline of the engine(s) for which the invalidation is targeted to is + * blocked, and all the in-flight transactions are guaranteed to be Globally + * Observed before completing the TLB invalidation + * 1: Lite mode of Invalidation: + * TLBs of the targeted engine(s) are immediately invalidated. + * In-flight transactions are NOT guaranteed to be Globally Observed before + * completing TLB invalidation. + * Light Invalidation Mode is to be used only when + * it can be guaranteed (by SW) that the address translations remain invariant + * for the in-flight transactions across the TLB invalidation. In other words, + * this mode can be used when the TLB invalidation is intended to clear out the + * stale cached translations that are no longer in use. Light Invalidation Mode + * is much faster than the Heavy Invalidation Mode, as it does not wait for the + * in-flight transactions to be GOd. + */ +enum xe_guc_tlb_inval_mode { + XE_GUC_TLB_INVAL_MODE_HEAVY = 0x0, + XE_GUC_TLB_INVAL_MODE_LITE = 0x1, +}; + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h new file mode 100644 index 000000000000..811add10c30d --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h @@ -0,0 +1,249 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _GUC_ACTIONS_SLPC_ABI_H_ +#define _GUC_ACTIONS_SLPC_ABI_H_ + +#include + +/** + * DOC: SLPC SHARED DATA STRUCTURE + * + * +----+------+--------------------------------------------------------------+ + * | CL | Bytes| Description | + * +====+======+==============================================================+ + * | 1 | 0-3 | SHARED DATA SIZE | + * | +------+--------------------------------------------------------------+ + * | | 4-7 | GLOBAL STATE | + * | +------+--------------------------------------------------------------+ + * | | 8-11 | DISPLAY DATA ADDRESS | + * | +------+--------------------------------------------------------------+ + * | | 12:63| PADDING | + * +----+------+--------------------------------------------------------------+ + * | | 0:63 | PADDING(PLATFORM INFO) | + * +----+------+--------------------------------------------------------------+ + * | 3 | 0-3 | TASK STATE DATA | + * + +------+--------------------------------------------------------------+ + * | | 4:63 | PADDING | + * +----+------+--------------------------------------------------------------+ + * |4-21|0:1087| OVERRIDE PARAMS AND BIT FIELDS | + * +----+------+--------------------------------------------------------------+ + * | | | PADDING + EXTRA RESERVED PAGE | + * +----+------+--------------------------------------------------------------+ + */ + +/* + * SLPC exposes certain parameters for global configuration by the host. + * These are referred to as override parameters, because in most cases + * the host will not need to modify the default values used by SLPC. + * SLPC remembers the default values which allows the host to easily restore + * them by simply unsetting the override. The host can set or unset override + * parameters during SLPC (re-)initialization using the SLPC Reset event. + * The host can also set or unset override parameters on the fly using the + * Parameter Set and Parameter Unset events + */ + +#define SLPC_MAX_OVERRIDE_PARAMETERS 256 +#define SLPC_OVERRIDE_BITFIELD_SIZE \ + (SLPC_MAX_OVERRIDE_PARAMETERS / 32) + +#define SLPC_PAGE_SIZE_BYTES 4096 +#define SLPC_CACHELINE_SIZE_BYTES 64 +#define SLPC_SHARED_DATA_SIZE_BYTE_HEADER SLPC_CACHELINE_SIZE_BYTES +#define SLPC_SHARED_DATA_SIZE_BYTE_PLATFORM_INFO SLPC_CACHELINE_SIZE_BYTES +#define SLPC_SHARED_DATA_SIZE_BYTE_TASK_STATE SLPC_CACHELINE_SIZE_BYTES +#define SLPC_SHARED_DATA_MODE_DEFN_TABLE_SIZE SLPC_PAGE_SIZE_BYTES +#define SLPC_SHARED_DATA_SIZE_BYTE_MAX (2 * SLPC_PAGE_SIZE_BYTES) + +/* + * Cacheline size aligned (Total size needed for + * SLPM_KMD_MAX_OVERRIDE_PARAMETERS=256 is 1088 bytes) + */ +#define SLPC_OVERRIDE_PARAMS_TOTAL_BYTES (((((SLPC_MAX_OVERRIDE_PARAMETERS * 4) \ + + ((SLPC_MAX_OVERRIDE_PARAMETERS / 32) * 4)) \ + + (SLPC_CACHELINE_SIZE_BYTES - 1)) / SLPC_CACHELINE_SIZE_BYTES) * \ + SLPC_CACHELINE_SIZE_BYTES) + +#define SLPC_SHARED_DATA_SIZE_BYTE_OTHER (SLPC_SHARED_DATA_SIZE_BYTE_MAX - \ + (SLPC_SHARED_DATA_SIZE_BYTE_HEADER \ + + SLPC_SHARED_DATA_SIZE_BYTE_PLATFORM_INFO \ + + SLPC_SHARED_DATA_SIZE_BYTE_TASK_STATE \ + + SLPC_OVERRIDE_PARAMS_TOTAL_BYTES \ + + SLPC_SHARED_DATA_MODE_DEFN_TABLE_SIZE)) + +enum slpc_task_enable { + SLPC_PARAM_TASK_DEFAULT = 0, + SLPC_PARAM_TASK_ENABLED, + SLPC_PARAM_TASK_DISABLED, + SLPC_PARAM_TASK_UNKNOWN +}; + +enum slpc_global_state { + SLPC_GLOBAL_STATE_NOT_RUNNING = 0, + SLPC_GLOBAL_STATE_INITIALIZING = 1, + SLPC_GLOBAL_STATE_RESETTING = 2, + SLPC_GLOBAL_STATE_RUNNING = 3, + SLPC_GLOBAL_STATE_SHUTTING_DOWN = 4, + SLPC_GLOBAL_STATE_ERROR = 5 +}; + +enum slpc_param_id { + SLPC_PARAM_TASK_ENABLE_GTPERF = 0, + SLPC_PARAM_TASK_DISABLE_GTPERF = 1, + SLPC_PARAM_TASK_ENABLE_BALANCER = 2, + SLPC_PARAM_TASK_DISABLE_BALANCER = 3, + SLPC_PARAM_TASK_ENABLE_DCC = 4, + SLPC_PARAM_TASK_DISABLE_DCC = 5, + SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ = 6, + SLPC_PARAM_GLOBAL_MAX_GT_UNSLICE_FREQ_MHZ = 7, + SLPC_PARAM_GLOBAL_MIN_GT_SLICE_FREQ_MHZ = 8, + SLPC_PARAM_GLOBAL_MAX_GT_SLICE_FREQ_MHZ = 9, + SLPC_PARAM_GTPERF_THRESHOLD_MAX_FPS = 10, + SLPC_PARAM_GLOBAL_DISABLE_GT_FREQ_MANAGEMENT = 11, + SLPC_PARAM_GTPERF_ENABLE_FRAMERATE_STALLING = 12, + SLPC_PARAM_GLOBAL_DISABLE_RC6_MODE_CHANGE = 13, + SLPC_PARAM_GLOBAL_OC_UNSLICE_FREQ_MHZ = 14, + SLPC_PARAM_GLOBAL_OC_SLICE_FREQ_MHZ = 15, + SLPC_PARAM_GLOBAL_ENABLE_IA_GT_BALANCING = 16, + SLPC_PARAM_GLOBAL_ENABLE_ADAPTIVE_BURST_TURBO = 17, + SLPC_PARAM_GLOBAL_ENABLE_EVAL_MODE = 18, + SLPC_PARAM_GLOBAL_ENABLE_BALANCER_IN_NON_GAMING_MODE = 19, + SLPC_PARAM_GLOBAL_RT_MODE_TURBO_FREQ_DELTA_MHZ = 20, + SLPC_PARAM_PWRGATE_RC_MODE = 21, + SLPC_PARAM_EDR_MODE_COMPUTE_TIMEOUT_MS = 22, + SLPC_PARAM_EDR_QOS_FREQ_MHZ = 23, + SLPC_PARAM_MEDIA_FF_RATIO_MODE = 24, + SLPC_PARAM_ENABLE_IA_FREQ_LIMITING = 25, + SLPC_PARAM_STRATEGIES = 26, + SLPC_PARAM_POWER_PROFILE = 27, + SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY = 28, + SLPC_MAX_PARAM = 32, +}; + +enum slpc_media_ratio_mode { + SLPC_MEDIA_RATIO_MODE_DYNAMIC_CONTROL = 0, + SLPC_MEDIA_RATIO_MODE_FIXED_ONE_TO_ONE = 1, + SLPC_MEDIA_RATIO_MODE_FIXED_ONE_TO_TWO = 2, +}; + +enum slpc_gucrc_mode { + SLPC_GUCRC_MODE_HW = 0, + SLPC_GUCRC_MODE_GUCRC_NO_RC6 = 1, + SLPC_GUCRC_MODE_GUCRC_STATIC_TIMEOUT = 2, + SLPC_GUCRC_MODE_GUCRC_DYNAMIC_HYSTERESIS = 3, + + SLPC_GUCRC_MODE_MAX, +}; + +enum slpc_event_id { + SLPC_EVENT_RESET = 0, + SLPC_EVENT_SHUTDOWN = 1, + SLPC_EVENT_PLATFORM_INFO_CHANGE = 2, + SLPC_EVENT_DISPLAY_MODE_CHANGE = 3, + SLPC_EVENT_FLIP_COMPLETE = 4, + SLPC_EVENT_QUERY_TASK_STATE = 5, + SLPC_EVENT_PARAMETER_SET = 6, + SLPC_EVENT_PARAMETER_UNSET = 7, +}; + +struct slpc_task_state_data { + union { + u32 task_status_padding; + struct { + u32 status; +#define SLPC_GTPERF_TASK_ENABLED REG_BIT(0) +#define SLPC_DCC_TASK_ENABLED REG_BIT(11) +#define SLPC_IN_DCC REG_BIT(12) +#define SLPC_BALANCER_ENABLED REG_BIT(15) +#define SLPC_IBC_TASK_ENABLED REG_BIT(16) +#define SLPC_BALANCER_IA_LMT_ENABLED REG_BIT(17) +#define SLPC_BALANCER_IA_LMT_ACTIVE REG_BIT(18) + }; + }; + union { + u32 freq_padding; + struct { +#define SLPC_MAX_UNSLICE_FREQ_MASK REG_GENMASK(7, 0) +#define SLPC_MIN_UNSLICE_FREQ_MASK REG_GENMASK(15, 8) +#define SLPC_MAX_SLICE_FREQ_MASK REG_GENMASK(23, 16) +#define SLPC_MIN_SLICE_FREQ_MASK REG_GENMASK(31, 24) + u32 freq; + }; + }; +} __packed; + +struct slpc_shared_data_header { + /* Total size in bytes of this shared buffer. */ + u32 size; + u32 global_state; + u32 display_data_addr; +} __packed; + +struct slpc_override_params { + u32 bits[SLPC_OVERRIDE_BITFIELD_SIZE]; + u32 values[SLPC_MAX_OVERRIDE_PARAMETERS]; +} __packed; + +struct slpc_shared_data { + struct slpc_shared_data_header header; + u8 shared_data_header_pad[SLPC_SHARED_DATA_SIZE_BYTE_HEADER - + sizeof(struct slpc_shared_data_header)]; + + u8 platform_info_pad[SLPC_SHARED_DATA_SIZE_BYTE_PLATFORM_INFO]; + + struct slpc_task_state_data task_state_data; + u8 task_state_data_pad[SLPC_SHARED_DATA_SIZE_BYTE_TASK_STATE - + sizeof(struct slpc_task_state_data)]; + + struct slpc_override_params override_params; + u8 override_params_pad[SLPC_OVERRIDE_PARAMS_TOTAL_BYTES - + sizeof(struct slpc_override_params)]; + + u8 shared_data_pad[SLPC_SHARED_DATA_SIZE_BYTE_OTHER]; + + /* PAGE 2 (4096 bytes), mode based parameter will be removed soon */ + u8 reserved_mode_definition[4096]; +} __packed; + +/** + * DOC: SLPC H2G MESSAGE FORMAT + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN = GUC_HXG_ORIGIN_HOST_ | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | DATA0 = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | ACTION = _`GUC_ACTION_HOST2GUC_PC_SLPM_REQUEST` = 0x3003 | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:8 | **EVENT_ID** | + * + +-------+--------------------------------------------------------------+ + * | | 7:0 | **EVENT_ARGC** - number of data arguments | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **EVENT_DATA1** | + * +---+-------+--------------------------------------------------------------+ + * |...| 31:0 | ... | + * +---+-------+--------------------------------------------------------------+ + * |2+n| 31:0 | **EVENT_DATAn** | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST 0x3003 + +#define HOST2GUC_PC_SLPC_REQUEST_MSG_MIN_LEN \ + (GUC_HXG_REQUEST_MSG_MIN_LEN + 1u) +#define HOST2GUC_PC_SLPC_EVENT_MAX_INPUT_ARGS 9 +#define HOST2GUC_PC_SLPC_REQUEST_MSG_MAX_LEN \ + (HOST2GUC_PC_SLPC_REQUEST_REQUEST_MSG_MIN_LEN + \ + HOST2GUC_PC_SLPC_EVENT_MAX_INPUT_ARGS) +#define HOST2GUC_PC_SLPC_REQUEST_MSG_0_MBZ GUC_HXG_REQUEST_MSG_0_DATA0 +#define HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ID (0xff << 8) +#define HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC (0xff << 0) +#define HOST2GUC_PC_SLPC_REQUEST_MSG_N_EVENT_DATA_N GUC_HXG_REQUEST_MSG_n_DATAn + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h b/drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h new file mode 100644 index 000000000000..41244055cc0c --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_communication_ctb_abi.h @@ -0,0 +1,189 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_COMMUNICATION_CTB_ABI_H +#define _ABI_GUC_COMMUNICATION_CTB_ABI_H + +#include +#include + +#include "guc_messages_abi.h" + +/** + * DOC: CT Buffer + * + * Circular buffer used to send `CTB Message`_ + */ + +/** + * DOC: CTB Descriptor + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:0 | **HEAD** - offset (in dwords) to the last dword that was | + * | | | read from the `CT Buffer`_. | + * | | | It can only be updated by the receiver. | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **TAIL** - offset (in dwords) to the last dword that was | + * | | | written to the `CT Buffer`_. | + * | | | It can only be updated by the sender. | + * +---+-------+--------------------------------------------------------------+ + * | 2 | 31:0 | **STATUS** - status of the CTB | + * | | | | + * | | | - _`GUC_CTB_STATUS_NO_ERROR` = 0 (normal operation) | + * | | | - _`GUC_CTB_STATUS_OVERFLOW` = 1 (head/tail too large) | + * | | | - _`GUC_CTB_STATUS_UNDERFLOW` = 2 (truncated message) | + * | | | - _`GUC_CTB_STATUS_MISMATCH` = 4 (head/tail modified) | + * +---+-------+--------------------------------------------------------------+ + * |...| | RESERVED = MBZ | + * +---+-------+--------------------------------------------------------------+ + * | 15| 31:0 | RESERVED = MBZ | + * +---+-------+--------------------------------------------------------------+ + */ + +struct guc_ct_buffer_desc { + u32 head; + u32 tail; + u32 status; +#define GUC_CTB_STATUS_NO_ERROR 0 +#define GUC_CTB_STATUS_OVERFLOW (1 << 0) +#define GUC_CTB_STATUS_UNDERFLOW (1 << 1) +#define GUC_CTB_STATUS_MISMATCH (1 << 2) + u32 reserved[13]; +} __packed; +static_assert(sizeof(struct guc_ct_buffer_desc) == 64); + +/** + * DOC: CTB Message + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:16 | **FENCE** - message identifier | + * | +-------+--------------------------------------------------------------+ + * | | 15:12 | **FORMAT** - format of the CTB message | + * | | | - _`GUC_CTB_FORMAT_HXG` = 0 - see `CTB HXG Message`_ | + * | +-------+--------------------------------------------------------------+ + * | | 11:8 | **RESERVED** | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | **NUM_DWORDS** - length of the CTB message (w/o header) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | optional (depends on FORMAT) | + * +---+-------+ | + * |...| | | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_CTB_HDR_LEN 1u +#define GUC_CTB_MSG_MIN_LEN GUC_CTB_HDR_LEN +#define GUC_CTB_MSG_MAX_LEN 256u +#define GUC_CTB_MSG_0_FENCE (0xffff << 16) +#define GUC_CTB_MSG_0_FORMAT (0xf << 12) +#define GUC_CTB_FORMAT_HXG 0u +#define GUC_CTB_MSG_0_RESERVED (0xf << 8) +#define GUC_CTB_MSG_0_NUM_DWORDS (0xff << 0) + +/** + * DOC: CTB HXG Message + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:16 | FENCE | + * | +-------+--------------------------------------------------------------+ + * | | 15:12 | FORMAT = GUC_CTB_FORMAT_HXG_ | + * | +-------+--------------------------------------------------------------+ + * | | 11:8 | RESERVED = MBZ | + * | +-------+--------------------------------------------------------------+ + * | | 7:0 | NUM_DWORDS = length (in dwords) of the embedded HXG message | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | [Embedded `HXG Message`_] | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_CTB_HXG_MSG_MIN_LEN (GUC_CTB_MSG_MIN_LEN + GUC_HXG_MSG_MIN_LEN) +#define GUC_CTB_HXG_MSG_MAX_LEN GUC_CTB_MSG_MAX_LEN + +/** + * DOC: CTB based communication + * + * The CTB (command transport buffer) communication between Host and GuC + * is based on u32 data stream written to the shared buffer. One buffer can + * be used to transmit data only in one direction (one-directional channel). + * + * Current status of the each buffer is stored in the buffer descriptor. + * Buffer descriptor holds tail and head fields that represents active data + * stream. The tail field is updated by the data producer (sender), and head + * field is updated by the data consumer (receiver):: + * + * +------------+ + * | DESCRIPTOR | +=================+============+========+ + * +============+ | | MESSAGE(s) | | + * | address |--------->+=================+============+========+ + * +------------+ + * | head | ^-----head--------^ + * +------------+ + * | tail | ^---------tail-----------------^ + * +------------+ + * | size | ^---------------size--------------------^ + * +------------+ + * + * Each message in data stream starts with the single u32 treated as a header, + * followed by optional set of u32 data that makes message specific payload:: + * + * +------------+---------+---------+---------+ + * | MESSAGE | + * +------------+---------+---------+---------+ + * | msg[0] | [1] | ... | [n-1] | + * +------------+---------+---------+---------+ + * | MESSAGE | MESSAGE PAYLOAD | + * + HEADER +---------+---------+---------+ + * | | 0 | ... | n | + * +======+=====+=========+=========+=========+ + * | 31:16| code| | | | + * +------+-----+ | | | + * | 15:5|flags| | | | + * +------+-----+ | | | + * | 4:0| len| | | | + * +------+-----+---------+---------+---------+ + * + * ^-------------len-------------^ + * + * The message header consists of: + * + * - **len**, indicates length of the message payload (in u32) + * - **code**, indicates message code + * - **flags**, holds various bits to control message handling + */ + +/* + * Definition of the command transport message header (DW0) + * + * bit[4..0] message len (in dwords) + * bit[7..5] reserved + * bit[8] response (G2H only) + * bit[8] write fence to desc (H2G only) + * bit[9] write status to H2G buff (H2G only) + * bit[10] send status back via G2H (H2G only) + * bit[15..11] reserved + * bit[31..16] action code + */ +#define GUC_CT_MSG_LEN_SHIFT 0 +#define GUC_CT_MSG_LEN_MASK 0x1F +#define GUC_CT_MSG_IS_RESPONSE (1 << 8) +#define GUC_CT_MSG_WRITE_FENCE_TO_DESC (1 << 8) +#define GUC_CT_MSG_WRITE_STATUS_TO_BUFF (1 << 9) +#define GUC_CT_MSG_SEND_STATUS (1 << 10) +#define GUC_CT_MSG_ACTION_SHIFT 16 +#define GUC_CT_MSG_ACTION_MASK 0xFFFF + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_communication_mmio_abi.h b/drivers/gpu/drm/xe/abi/guc_communication_mmio_abi.h new file mode 100644 index 000000000000..ef538e34f894 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_communication_mmio_abi.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_COMMUNICATION_MMIO_ABI_H +#define _ABI_GUC_COMMUNICATION_MMIO_ABI_H + +/** + * DOC: GuC MMIO based communication + * + * The MMIO based communication between Host and GuC relies on special + * hardware registers which format could be defined by the software + * (so called scratch registers). + * + * Each MMIO based message, both Host to GuC (H2G) and GuC to Host (G2H) + * messages, which maximum length depends on number of available scratch + * registers, is directly written into those scratch registers. + * + * For Gen9+, there are 16 software scratch registers 0xC180-0xC1B8, + * but no H2G command takes more than 4 parameters and the GuC firmware + * itself uses an 4-element array to store the H2G message. + * + * For Gen11+, there are additional 4 registers 0x190240-0x19024C, which + * are, regardless on lower count, preferred over legacy ones. + * + * The MMIO based communication is mainly used during driver initialization + * phase to setup the `CTB based communication`_ that will be used afterwards. + */ + +#define GUC_MAX_MMIO_MSG_LEN 4 + +/** + * DOC: MMIO HXG Message + * + * Format of the MMIO messages follows definitions of `HXG Message`_. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:0 | | + * +---+-------+ | + * |...| | [Embedded `HXG Message`_] | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_errors_abi.h b/drivers/gpu/drm/xe/abi/guc_errors_abi.h new file mode 100644 index 000000000000..ec83551bf9c0 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_errors_abi.h @@ -0,0 +1,37 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_ERRORS_ABI_H +#define _ABI_GUC_ERRORS_ABI_H + +enum xe_guc_response_status { + XE_GUC_RESPONSE_STATUS_SUCCESS = 0x0, + XE_GUC_RESPONSE_STATUS_GENERIC_FAIL = 0xF000, +}; + +enum xe_guc_load_status { + XE_GUC_LOAD_STATUS_DEFAULT = 0x00, + XE_GUC_LOAD_STATUS_START = 0x01, + XE_GUC_LOAD_STATUS_ERROR_DEVID_BUILD_MISMATCH = 0x02, + XE_GUC_LOAD_STATUS_GUC_PREPROD_BUILD_MISMATCH = 0x03, + XE_GUC_LOAD_STATUS_ERROR_DEVID_INVALID_GUCTYPE = 0x04, + XE_GUC_LOAD_STATUS_GDT_DONE = 0x10, + XE_GUC_LOAD_STATUS_IDT_DONE = 0x20, + XE_GUC_LOAD_STATUS_LAPIC_DONE = 0x30, + XE_GUC_LOAD_STATUS_GUCINT_DONE = 0x40, + XE_GUC_LOAD_STATUS_DPC_READY = 0x50, + XE_GUC_LOAD_STATUS_DPC_ERROR = 0x60, + XE_GUC_LOAD_STATUS_EXCEPTION = 0x70, + XE_GUC_LOAD_STATUS_INIT_DATA_INVALID = 0x71, + XE_GUC_LOAD_STATUS_PXP_TEARDOWN_CTRL_ENABLED = 0x72, + XE_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_START, + XE_GUC_LOAD_STATUS_MPU_DATA_INVALID = 0x73, + XE_GUC_LOAD_STATUS_INIT_MMIO_SAVE_RESTORE_INVALID = 0x74, + XE_GUC_LOAD_STATUS_INVALID_INIT_DATA_RANGE_END, + + XE_GUC_LOAD_STATUS_READY = 0xF0, +}; + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_klvs_abi.h b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h new file mode 100644 index 000000000000..47094b9b044c --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_klvs_abi.h @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _ABI_GUC_KLVS_ABI_H +#define _ABI_GUC_KLVS_ABI_H + +#include + +/** + * DOC: GuC KLV + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31:16 | **KEY** - KLV key identifier | + * | | | - `GuC Self Config KLVs`_ | + * | | | - `GuC VGT Policy KLVs`_ | + * | | | - `GuC VF Configuration KLVs`_ | + * | | | | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **LEN** - length of VALUE (in 32bit dwords) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | **VALUE** - actual value of the KLV (format depends on KEY) | + * +---+-------+ | + * |...| | | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_KLV_LEN_MIN 1u +#define GUC_KLV_0_KEY (0xffff << 16) +#define GUC_KLV_0_LEN (0xffff << 0) +#define GUC_KLV_n_VALUE (0xffffffff << 0) + +/** + * DOC: GuC Self Config KLVs + * + * `GuC KLV`_ keys available for use with HOST2GUC_SELF_CFG_. + * + * _`GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR` : 0x0900 + * Refers to 64 bit Global Gfx address (in bytes) of memory based interrupts + * status vector for use by the GuC. + * + * _`GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR` : 0x0901 + * Refers to 64 bit Global Gfx address (in bytes) of memory based interrupts + * source vector for use by the GuC. + * + * _`GUC_KLV_SELF_CFG_H2G_CTB_ADDR` : 0x0902 + * Refers to 64 bit Global Gfx address of H2G `CT Buffer`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _`GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR : 0x0903 + * Refers to 64 bit Global Gfx address of H2G `CTB Descriptor`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _`GUC_KLV_SELF_CFG_H2G_CTB_SIZE : 0x0904 + * Refers to size of H2G `CT Buffer`_ in bytes. + * Should be a multiple of 4K. + * + * _`GUC_KLV_SELF_CFG_G2H_CTB_ADDR : 0x0905 + * Refers to 64 bit Global Gfx address of G2H `CT Buffer`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR : 0x0906 + * Refers to 64 bit Global Gfx address of G2H `CTB Descriptor`_. + * Should be above WOPCM address but below APIC base address for native mode. + * + * _GUC_KLV_SELF_CFG_G2H_CTB_SIZE : 0x0907 + * Refers to size of G2H `CT Buffer`_ in bytes. + * Should be a multiple of 4K. + */ + +#define GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_KEY 0x0900 +#define GUC_KLV_SELF_CFG_MEMIRQ_STATUS_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_KEY 0x0901 +#define GUC_KLV_SELF_CFG_MEMIRQ_SOURCE_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY 0x0902 +#define GUC_KLV_SELF_CFG_H2G_CTB_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY 0x0903 +#define GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY 0x0904 +#define GUC_KLV_SELF_CFG_H2G_CTB_SIZE_LEN 1u + +#define GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY 0x0905 +#define GUC_KLV_SELF_CFG_G2H_CTB_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY 0x0906 +#define GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_LEN 2u + +#define GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY 0x0907 +#define GUC_KLV_SELF_CFG_G2H_CTB_SIZE_LEN 1u + +/* + * Per context scheduling policy update keys. + */ +enum { + GUC_CONTEXT_POLICIES_KLV_ID_EXECUTION_QUANTUM = 0x2001, + GUC_CONTEXT_POLICIES_KLV_ID_PREEMPTION_TIMEOUT = 0x2002, + GUC_CONTEXT_POLICIES_KLV_ID_SCHEDULING_PRIORITY = 0x2003, + GUC_CONTEXT_POLICIES_KLV_ID_PREEMPT_TO_IDLE_ON_QUANTUM_EXPIRY = 0x2004, + GUC_CONTEXT_POLICIES_KLV_ID_SLPM_GT_FREQUENCY = 0x2005, + + GUC_CONTEXT_POLICIES_KLV_NUM_IDS = 5, +}; + +/** + * DOC: GuC VGT Policy KLVs + * + * `GuC KLV`_ keys available for use with PF2GUC_UPDATE_VGT_POLICY. + * + * _`GUC_KLV_VGT_POLICY_SCHED_IF_IDLE` : 0x8001 + * This config sets whether strict scheduling is enabled whereby any VF + * that doesn’t have work to submit is still allocated a fixed execution + * time-slice to ensure active VFs execution is always consitent even + * during other VF reprovisiong / rebooting events. Changing this KLV + * impacts all VFs and takes effect on the next VF-Switch event. + * + * :0: don't schedule idle (default) + * :1: schedule if idle + * + * _`GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD` : 0x8002 + * This config sets the sample period for tracking adverse event counters. + * A sample period is the period in millisecs during which events are counted. + * This is applicable for all the VFs. + * + * :0: adverse events are not counted (default) + * :n: sample period in milliseconds + * + * _`GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH` : 0x8D00 + * This enum is to reset utilized HW engine after VF Switch (i.e to clean + * up Stale HW register left behind by previous VF) + * + * :0: don't reset (default) + * :1: reset + */ + +#define GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_KEY 0x8001 +#define GUC_KLV_VGT_POLICY_SCHED_IF_IDLE_LEN 1u + +#define GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_KEY 0x8002 +#define GUC_KLV_VGT_POLICY_ADVERSE_SAMPLE_PERIOD_LEN 1u + +#define GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_KEY 0x8D00 +#define GUC_KLV_VGT_POLICY_RESET_AFTER_VF_SWITCH_LEN 1u + +/** + * DOC: GuC VF Configuration KLVs + * + * `GuC KLV`_ keys available for use with PF2GUC_UPDATE_VF_CFG. + * + * _`GUC_KLV_VF_CFG_GGTT_START` : 0x0001 + * A 4K aligned start GTT address/offset assigned to VF. + * Value is 64 bits. + * + * _`GUC_KLV_VF_CFG_GGTT_SIZE` : 0x0002 + * A 4K aligned size of GGTT assigned to VF. + * Value is 64 bits. + * + * _`GUC_KLV_VF_CFG_LMEM_SIZE` : 0x0003 + * A 2M aligned size of local memory assigned to VF. + * Value is 64 bits. + * + * _`GUC_KLV_VF_CFG_NUM_CONTEXTS` : 0x0004 + * Refers to the number of contexts allocated to this VF. + * + * :0: no contexts (default) + * :1-65535: number of contexts (Gen12) + * + * _`GUC_KLV_VF_CFG_TILE_MASK` : 0x0005 + * For multi-tiled products, this field contains the bitwise-OR of tiles + * assigned to the VF. Bit-0-set means VF has access to Tile-0, + * Bit-31-set means VF has access to Tile-31, and etc. + * At least one tile will always be allocated. + * If all bits are zero, VF KMD should treat this as a fatal error. + * For, single-tile products this KLV config is ignored. + * + * _`GUC_KLV_VF_CFG_NUM_DOORBELLS` : 0x0006 + * Refers to the number of doorbells allocated to this VF. + * + * :0: no doorbells (default) + * :1-255: number of doorbells (Gen12) + * + * _`GUC_KLV_VF_CFG_EXEC_QUANTUM` : 0x8A01 + * This config sets the VFs-execution-quantum in milliseconds. + * GUC will attempt to obey the maximum values as much as HW is capable + * of and this will never be perfectly-exact (accumulated nano-second + * granularity) since the GPUs clock time runs off a different crystal + * from the CPUs clock. Changing this KLV on a VF that is currently + * running a context wont take effect until a new context is scheduled in. + * That said, when the PF is changing this value from 0xFFFFFFFF to + * something else, it might never take effect if the VF is running an + * inifinitely long compute or shader kernel. In such a scenario, the + * PF would need to trigger a VM PAUSE and then change the KLV to force + * it to take effect. Such cases might typically happen on a 1PF+1VF + * Virtualization config enabled for heavier workloads like AI/ML. + * + * :0: infinite exec quantum (default) + * + * _`GUC_KLV_VF_CFG_PREEMPT_TIMEOUT` : 0x8A02 + * This config sets the VF-preemption-timeout in microseconds. + * GUC will attempt to obey the minimum and maximum values as much as + * HW is capable and this will never be perfectly-exact (accumulated + * nano-second granularity) since the GPUs clock time runs off a + * different crystal from the CPUs clock. Changing this KLV on a VF + * that is currently running a context wont take effect until a new + * context is scheduled in. + * That said, when the PF is changing this value from 0xFFFFFFFF to + * something else, it might never take effect if the VF is running an + * inifinitely long compute or shader kernel. + * In this case, the PF would need to trigger a VM PAUSE and then change + * the KLV to force it to take effect. Such cases might typically happen + * on a 1PF+1VF Virtualization config enabled for heavier workloads like + * AI/ML. + * + * :0: no preemption timeout (default) + * + * _`GUC_KLV_VF_CFG_THRESHOLD_CAT_ERR` : 0x8A03 + * This config sets threshold for CAT errors caused by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: event occurrence count per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_ENGINE_RESET` : 0x8A04 + * This config sets threshold for engine reset caused by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: event occurrence count per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_PAGE_FAULT` : 0x8A05 + * This config sets threshold for page fault errors caused by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: event occurrence count per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_H2G_STORM` : 0x8A06 + * This config sets threshold for H2G interrupts triggered by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: time (us) per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_IRQ_STORM` : 0x8A07 + * This config sets threshold for GT interrupts triggered by the VF's + * workloads. + * + * :0: adverse events or error will not be reported (default) + * :n: time (us) per sampling interval + * + * _`GUC_KLV_VF_CFG_THRESHOLD_DOORBELL_STORM` : 0x8A08 + * This config sets threshold for doorbell's ring triggered by the VF. + * + * :0: adverse events or error will not be reported (default) + * :n: time (us) per sampling interval + * + * _`GUC_KLV_VF_CFG_BEGIN_DOORBELL_ID` : 0x8A0A + * Refers to the start index of doorbell assigned to this VF. + * + * :0: (default) + * :1-255: number of doorbells (Gen12) + * + * _`GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID` : 0x8A0B + * Refers to the start index in context array allocated to this VF’s use. + * + * :0: (default) + * :1-65535: number of contexts (Gen12) + */ + +#define GUC_KLV_VF_CFG_GGTT_START_KEY 0x0001 +#define GUC_KLV_VF_CFG_GGTT_START_LEN 2u + +#define GUC_KLV_VF_CFG_GGTT_SIZE_KEY 0x0002 +#define GUC_KLV_VF_CFG_GGTT_SIZE_LEN 2u + +#define GUC_KLV_VF_CFG_LMEM_SIZE_KEY 0x0003 +#define GUC_KLV_VF_CFG_LMEM_SIZE_LEN 2u + +#define GUC_KLV_VF_CFG_NUM_CONTEXTS_KEY 0x0004 +#define GUC_KLV_VF_CFG_NUM_CONTEXTS_LEN 1u + +#define GUC_KLV_VF_CFG_TILE_MASK_KEY 0x0005 +#define GUC_KLV_VF_CFG_TILE_MASK_LEN 1u + +#define GUC_KLV_VF_CFG_NUM_DOORBELLS_KEY 0x0006 +#define GUC_KLV_VF_CFG_NUM_DOORBELLS_LEN 1u + +#define GUC_KLV_VF_CFG_EXEC_QUANTUM_KEY 0x8a01 +#define GUC_KLV_VF_CFG_EXEC_QUANTUM_LEN 1u + +#define GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_KEY 0x8a02 +#define GUC_KLV_VF_CFG_PREEMPT_TIMEOUT_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_CAT_ERR_KEY 0x8a03 +#define GUC_KLV_VF_CFG_THRESHOLD_CAT_ERR_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_ENGINE_RESET_KEY 0x8a04 +#define GUC_KLV_VF_CFG_THRESHOLD_ENGINE_RESET_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_PAGE_FAULT_KEY 0x8a05 +#define GUC_KLV_VF_CFG_THRESHOLD_PAGE_FAULT_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_H2G_STORM_KEY 0x8a06 +#define GUC_KLV_VF_CFG_THRESHOLD_H2G_STORM_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_IRQ_STORM_KEY 0x8a07 +#define GUC_KLV_VF_CFG_THRESHOLD_IRQ_STORM_LEN 1u + +#define GUC_KLV_VF_CFG_THRESHOLD_DOORBELL_STORM_KEY 0x8a08 +#define GUC_KLV_VF_CFG_THRESHOLD_DOORBELL_STORM_LEN 1u + +#define GUC_KLV_VF_CFG_BEGIN_DOORBELL_ID_KEY 0x8a0a +#define GUC_KLV_VF_CFG_BEGIN_DOORBELL_ID_LEN 1u + +#define GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID_KEY 0x8a0b +#define GUC_KLV_VF_CFG_BEGIN_CONTEXT_ID_LEN 1u + +#endif diff --git a/drivers/gpu/drm/xe/abi/guc_messages_abi.h b/drivers/gpu/drm/xe/abi/guc_messages_abi.h new file mode 100644 index 000000000000..3d199016cf88 --- /dev/null +++ b/drivers/gpu/drm/xe/abi/guc_messages_abi.h @@ -0,0 +1,234 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2014-2021 Intel Corporation + */ + +#ifndef _ABI_GUC_MESSAGES_ABI_H +#define _ABI_GUC_MESSAGES_ABI_H + +/** + * DOC: HXG Message + * + * All messages exchanged with GuC are defined using 32 bit dwords. + * First dword is treated as a message header. Remaining dwords are optional. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | | | | + * | 0 | 31 | **ORIGIN** - originator of the message | + * | | | - _`GUC_HXG_ORIGIN_HOST` = 0 | + * | | | - _`GUC_HXG_ORIGIN_GUC` = 1 | + * | | | | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | **TYPE** - message type | + * | | | - _`GUC_HXG_TYPE_REQUEST` = 0 | + * | | | - _`GUC_HXG_TYPE_EVENT` = 1 | + * | | | - _`GUC_HXG_TYPE_NO_RESPONSE_BUSY` = 3 | + * | | | - _`GUC_HXG_TYPE_NO_RESPONSE_RETRY` = 5 | + * | | | - _`GUC_HXG_TYPE_RESPONSE_FAILURE` = 6 | + * | | | - _`GUC_HXG_TYPE_RESPONSE_SUCCESS` = 7 | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **AUX** - auxiliary data (depends on TYPE) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **PAYLOAD** - optional payload (depends on TYPE) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_MSG_MIN_LEN 1u +#define GUC_HXG_MSG_0_ORIGIN (0x1 << 31) +#define GUC_HXG_ORIGIN_HOST 0u +#define GUC_HXG_ORIGIN_GUC 1u +#define GUC_HXG_MSG_0_TYPE (0x7 << 28) +#define GUC_HXG_TYPE_REQUEST 0u +#define GUC_HXG_TYPE_EVENT 1u +#define GUC_HXG_TYPE_NO_RESPONSE_BUSY 3u +#define GUC_HXG_TYPE_NO_RESPONSE_RETRY 5u +#define GUC_HXG_TYPE_RESPONSE_FAILURE 6u +#define GUC_HXG_TYPE_RESPONSE_SUCCESS 7u +#define GUC_HXG_MSG_0_AUX (0xfffffff << 0) +#define GUC_HXG_MSG_n_PAYLOAD (0xffffffff << 0) + +/** + * DOC: HXG Request + * + * The `HXG Request`_ message should be used to initiate synchronous activity + * for which confirmation or return data is expected. + * + * The recipient of this message shall use `HXG Response`_, `HXG Failure`_ + * or `HXG Retry`_ message as a definite reply, and may use `HXG Busy`_ + * message as a intermediate reply. + * + * Format of @DATA0 and all @DATAn fields depends on the @ACTION code. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_REQUEST_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **DATA0** - request data (depends on ACTION) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **ACTION** - requested action code | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **DATAn** - optional data (depends on ACTION) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_REQUEST_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_REQUEST_MSG_0_DATA0 (0xfff << 16) +#define GUC_HXG_REQUEST_MSG_0_ACTION (0xffff << 0) +#define GUC_HXG_REQUEST_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD + +/** + * DOC: HXG Event + * + * The `HXG Event`_ message should be used to initiate asynchronous activity + * that does not involves immediate confirmation nor data. + * + * Format of @DATA0 and all @DATAn fields depends on the @ACTION code. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_EVENT_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **DATA0** - event data (depends on ACTION) | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **ACTION** - event action code | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **DATAn** - optional event data (depends on ACTION) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_EVENT_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_EVENT_MSG_0_DATA0 (0xfff << 16) +#define GUC_HXG_EVENT_MSG_0_ACTION (0xffff << 0) +#define GUC_HXG_EVENT_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD + +/** + * DOC: HXG Busy + * + * The `HXG Busy`_ message may be used to acknowledge reception of the `HXG Request`_ + * message if the recipient expects that it processing will be longer than default + * timeout. + * + * The @COUNTER field may be used as a progress indicator. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_NO_RESPONSE_BUSY_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **COUNTER** - progress indicator | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_BUSY_MSG_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_BUSY_MSG_0_COUNTER GUC_HXG_MSG_0_AUX + +/** + * DOC: HXG Retry + * + * The `HXG Retry`_ message should be used by recipient to indicate that the + * `HXG Request`_ message was dropped and it should be resent again. + * + * The @REASON field may be used to provide additional information. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_NO_RESPONSE_RETRY_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **REASON** - reason for retry | + * | | | - _`GUC_HXG_RETRY_REASON_UNSPECIFIED` = 0 | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_RETRY_MSG_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_RETRY_MSG_0_REASON GUC_HXG_MSG_0_AUX +#define GUC_HXG_RETRY_REASON_UNSPECIFIED 0u + +/** + * DOC: HXG Failure + * + * The `HXG Failure`_ message shall be used as a reply to the `HXG Request`_ + * message that could not be processed due to an error. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_FAILURE_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:16 | **HINT** - additional error hint | + * | +-------+--------------------------------------------------------------+ + * | | 15:0 | **ERROR** - error/result code | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_FAILURE_MSG_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_FAILURE_MSG_0_HINT (0xfff << 16) +#define GUC_HXG_FAILURE_MSG_0_ERROR (0xffff << 0) + +/** + * DOC: HXG Response + * + * The `HXG Response`_ message shall be used as a reply to the `HXG Request`_ + * message that was successfully processed without an error. + * + * +---+-------+--------------------------------------------------------------+ + * | | Bits | Description | + * +===+=======+==============================================================+ + * | 0 | 31 | ORIGIN | + * | +-------+--------------------------------------------------------------+ + * | | 30:28 | TYPE = GUC_HXG_TYPE_RESPONSE_SUCCESS_ | + * | +-------+--------------------------------------------------------------+ + * | | 27:0 | **DATA0** - data (depends on ACTION from `HXG Request`_) | + * +---+-------+--------------------------------------------------------------+ + * | 1 | 31:0 | | + * +---+-------+ | + * |...| | **DATAn** - data (depends on ACTION from `HXG Request`_) | + * +---+-------+ | + * | n | 31:0 | | + * +---+-------+--------------------------------------------------------------+ + */ + +#define GUC_HXG_RESPONSE_MSG_MIN_LEN GUC_HXG_MSG_MIN_LEN +#define GUC_HXG_RESPONSE_MSG_0_DATA0 GUC_HXG_MSG_0_AUX +#define GUC_HXG_RESPONSE_MSG_n_DATAn GUC_HXG_MSG_n_PAYLOAD + +/* deprecated */ +#define INTEL_GUC_MSG_TYPE_SHIFT 28 +#define INTEL_GUC_MSG_TYPE_MASK (0xF << INTEL_GUC_MSG_TYPE_SHIFT) +#define INTEL_GUC_MSG_DATA_SHIFT 16 +#define INTEL_GUC_MSG_DATA_MASK (0xFFF << INTEL_GUC_MSG_DATA_SHIFT) +#define INTEL_GUC_MSG_CODE_SHIFT 0 +#define INTEL_GUC_MSG_CODE_MASK (0xFFFF << INTEL_GUC_MSG_CODE_SHIFT) + +enum intel_guc_msg_type { + INTEL_GUC_MSG_TYPE_REQUEST = 0x0, + INTEL_GUC_MSG_TYPE_RESPONSE = 0xF, +}; + +#endif diff --git a/drivers/gpu/drm/xe/tests/Makefile b/drivers/gpu/drm/xe/tests/Makefile new file mode 100644 index 000000000000..47056b6459e3 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_DRM_XE_KUNIT_TEST) += xe_bo_test.o xe_dma_buf_test.o \ + xe_migrate_test.o diff --git a/drivers/gpu/drm/xe/tests/xe_bo.c b/drivers/gpu/drm/xe/tests/xe_bo.c new file mode 100644 index 000000000000..87ac21cc8ca9 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_bo.c @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_bo_evict.h" +#include "xe_pci.h" + +static int ccs_test_migrate(struct xe_gt *gt, struct xe_bo *bo, + bool clear, u64 get_val, u64 assign_val, + struct kunit *test) +{ + struct dma_fence *fence; + struct ttm_tt *ttm; + struct page *page; + pgoff_t ccs_page; + long timeout; + u64 *cpu_map; + int ret; + u32 offset; + + /* Move bo to VRAM if not already there. */ + ret = xe_bo_validate(bo, NULL, false); + if (ret) { + KUNIT_FAIL(test, "Failed to validate bo.\n"); + return ret; + } + + /* Optionally clear bo *and* CCS data in VRAM. */ + if (clear) { + fence = xe_migrate_clear(gt->migrate, bo, bo->ttm.resource, 0); + if (IS_ERR(fence)) { + KUNIT_FAIL(test, "Failed to submit bo clear.\n"); + return PTR_ERR(fence); + } + dma_fence_put(fence); + } + + /* Evict to system. CCS data should be copied. */ + ret = xe_bo_evict(bo, true); + if (ret) { + KUNIT_FAIL(test, "Failed to evict bo.\n"); + return ret; + } + + /* Sync all migration blits */ + timeout = dma_resv_wait_timeout(bo->ttm.base.resv, + DMA_RESV_USAGE_KERNEL, + true, + 5 * HZ); + if (timeout <= 0) { + KUNIT_FAIL(test, "Failed to sync bo eviction.\n"); + return -ETIME; + } + + /* + * Bo with CCS data is now in system memory. Verify backing store + * and data integrity. Then assign for the next testing round while + * we still have a CPU map. + */ + ttm = bo->ttm.ttm; + if (!ttm || !ttm_tt_is_populated(ttm)) { + KUNIT_FAIL(test, "Bo was not in expected placement.\n"); + return -EINVAL; + } + + ccs_page = xe_bo_ccs_pages_start(bo) >> PAGE_SHIFT; + if (ccs_page >= ttm->num_pages) { + KUNIT_FAIL(test, "No TTM CCS pages present.\n"); + return -EINVAL; + } + + page = ttm->pages[ccs_page]; + cpu_map = kmap_local_page(page); + + /* Check first CCS value */ + if (cpu_map[0] != get_val) { + KUNIT_FAIL(test, + "Expected CCS readout 0x%016llx, got 0x%016llx.\n", + (unsigned long long)get_val, + (unsigned long long)cpu_map[0]); + ret = -EINVAL; + } + + /* Check last CCS value, or at least last value in page. */ + offset = xe_device_ccs_bytes(gt->xe, bo->size); + offset = min_t(u32, offset, PAGE_SIZE) / sizeof(u64) - 1; + if (cpu_map[offset] != get_val) { + KUNIT_FAIL(test, + "Expected CCS readout 0x%016llx, got 0x%016llx.\n", + (unsigned long long)get_val, + (unsigned long long)cpu_map[offset]); + ret = -EINVAL; + } + + cpu_map[0] = assign_val; + cpu_map[offset] = assign_val; + kunmap_local(cpu_map); + + return ret; +} + +static void ccs_test_run_gt(struct xe_device *xe, struct xe_gt *gt, + struct kunit *test) +{ + struct xe_bo *bo; + u32 vram_bit; + int ret; + + /* TODO: Sanity check */ + vram_bit = XE_BO_CREATE_VRAM0_BIT << gt->info.vram_id; + kunit_info(test, "Testing gt id %u vram id %u\n", gt->info.id, + gt->info.vram_id); + + bo = xe_bo_create_locked(xe, NULL, NULL, SZ_1M, ttm_bo_type_device, + vram_bit); + if (IS_ERR(bo)) { + KUNIT_FAIL(test, "Failed to create bo.\n"); + return; + } + + kunit_info(test, "Verifying that CCS data is cleared on creation.\n"); + ret = ccs_test_migrate(gt, bo, false, 0ULL, 0xdeadbeefdeadbeefULL, + test); + if (ret) + goto out_unlock; + + kunit_info(test, "Verifying that CCS data survives migration.\n"); + ret = ccs_test_migrate(gt, bo, false, 0xdeadbeefdeadbeefULL, + 0xdeadbeefdeadbeefULL, test); + if (ret) + goto out_unlock; + + kunit_info(test, "Verifying that CCS data can be properly cleared.\n"); + ret = ccs_test_migrate(gt, bo, true, 0ULL, 0ULL, test); + +out_unlock: + xe_bo_unlock_no_vm(bo); + xe_bo_put(bo); +} + +static int ccs_test_run_device(struct xe_device *xe) +{ + struct kunit *test = xe_cur_kunit(); + struct xe_gt *gt; + int id; + + if (!xe_device_has_flat_ccs(xe)) { + kunit_info(test, "Skipping non-flat-ccs device.\n"); + return 0; + } + + for_each_gt(gt, xe, id) + ccs_test_run_gt(xe, gt, test); + + return 0; +} + +void xe_ccs_migrate_kunit(struct kunit *test) +{ + xe_call_for_each_device(ccs_test_run_device); +} +EXPORT_SYMBOL(xe_ccs_migrate_kunit); + +static int evict_test_run_gt(struct xe_device *xe, struct xe_gt *gt, struct kunit *test) +{ + struct xe_bo *bo, *external; + unsigned int bo_flags = XE_BO_CREATE_USER_BIT | + XE_BO_CREATE_VRAM_IF_DGFX(gt); + struct xe_vm *vm = xe_migrate_get_vm(xe->gt[0].migrate); + struct ww_acquire_ctx ww; + int err, i; + + kunit_info(test, "Testing device %s gt id %u vram id %u\n", + dev_name(xe->drm.dev), gt->info.id, gt->info.vram_id); + + for (i = 0; i < 2; ++i) { + xe_vm_lock(vm, &ww, 0, false); + bo = xe_bo_create(xe, NULL, vm, 0x10000, ttm_bo_type_device, + bo_flags); + xe_vm_unlock(vm, &ww); + if (IS_ERR(bo)) { + KUNIT_FAIL(test, "bo create err=%pe\n", bo); + break; + } + + external = xe_bo_create(xe, NULL, NULL, 0x10000, + ttm_bo_type_device, bo_flags); + if (IS_ERR(external)) { + KUNIT_FAIL(test, "external bo create err=%pe\n", external); + goto cleanup_bo; + } + + xe_bo_lock(external, &ww, 0, false); + err = xe_bo_pin_external(external); + xe_bo_unlock(external, &ww); + if (err) { + KUNIT_FAIL(test, "external bo pin err=%pe\n", + ERR_PTR(err)); + goto cleanup_external; + } + + err = xe_bo_evict_all(xe); + if (err) { + KUNIT_FAIL(test, "evict err=%pe\n", ERR_PTR(err)); + goto cleanup_all; + } + + err = xe_bo_restore_kernel(xe); + if (err) { + KUNIT_FAIL(test, "restore kernel err=%pe\n", + ERR_PTR(err)); + goto cleanup_all; + } + + err = xe_bo_restore_user(xe); + if (err) { + KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err)); + goto cleanup_all; + } + + if (!xe_bo_is_vram(external)) { + KUNIT_FAIL(test, "external bo is not vram\n"); + err = -EPROTO; + goto cleanup_all; + } + + if (xe_bo_is_vram(bo)) { + KUNIT_FAIL(test, "bo is vram\n"); + err = -EPROTO; + goto cleanup_all; + } + + if (i) { + down_read(&vm->lock); + xe_vm_lock(vm, &ww, 0, false); + err = xe_bo_validate(bo, bo->vm, false); + xe_vm_unlock(vm, &ww); + up_read(&vm->lock); + if (err) { + KUNIT_FAIL(test, "bo valid err=%pe\n", + ERR_PTR(err)); + goto cleanup_all; + } + xe_bo_lock(external, &ww, 0, false); + err = xe_bo_validate(external, NULL, false); + xe_bo_unlock(external, &ww); + if (err) { + KUNIT_FAIL(test, "external bo valid err=%pe\n", + ERR_PTR(err)); + goto cleanup_all; + } + } + + xe_bo_lock(external, &ww, 0, false); + xe_bo_unpin_external(external); + xe_bo_unlock(external, &ww); + + xe_bo_put(external); + xe_bo_put(bo); + continue; + +cleanup_all: + xe_bo_lock(external, &ww, 0, false); + xe_bo_unpin_external(external); + xe_bo_unlock(external, &ww); +cleanup_external: + xe_bo_put(external); +cleanup_bo: + xe_bo_put(bo); + break; + } + + xe_vm_put(vm); + + return 0; +} + +static int evict_test_run_device(struct xe_device *xe) +{ + struct kunit *test = xe_cur_kunit(); + struct xe_gt *gt; + int id; + + if (!IS_DGFX(xe)) { + kunit_info(test, "Skipping non-discrete device %s.\n", + dev_name(xe->drm.dev)); + return 0; + } + + for_each_gt(gt, xe, id) + evict_test_run_gt(xe, gt, test); + + return 0; +} + +void xe_bo_evict_kunit(struct kunit *test) +{ + xe_call_for_each_device(evict_test_run_device); +} +EXPORT_SYMBOL(xe_bo_evict_kunit); diff --git a/drivers/gpu/drm/xe/tests/xe_bo_test.c b/drivers/gpu/drm/xe/tests/xe_bo_test.c new file mode 100644 index 000000000000..c8fa29b0b3b2 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_bo_test.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +void xe_ccs_migrate_kunit(struct kunit *test); +void xe_bo_evict_kunit(struct kunit *test); + +static struct kunit_case xe_bo_tests[] = { + KUNIT_CASE(xe_ccs_migrate_kunit), + KUNIT_CASE(xe_bo_evict_kunit), + {} +}; + +static struct kunit_suite xe_bo_test_suite = { + .name = "xe_bo", + .test_cases = xe_bo_tests, +}; + +kunit_test_suite(xe_bo_test_suite); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c new file mode 100644 index 000000000000..615d22e3f731 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c @@ -0,0 +1,259 @@ +// SPDX-License-Identifier: GPL-2.0 AND MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_pci.h" + +static bool p2p_enabled(struct dma_buf_test_params *params) +{ + return IS_ENABLED(CONFIG_PCI_P2PDMA) && params->attach_ops && + params->attach_ops->allow_peer2peer; +} + +static bool is_dynamic(struct dma_buf_test_params *params) +{ + return IS_ENABLED(CONFIG_DMABUF_MOVE_NOTIFY) && params->attach_ops && + params->attach_ops->move_notify; +} + +static void check_residency(struct kunit *test, struct xe_bo *exported, + struct xe_bo *imported, struct dma_buf *dmabuf) +{ + struct dma_buf_test_params *params = to_dma_buf_test_params(test->priv); + u32 mem_type; + int ret; + + xe_bo_assert_held(exported); + xe_bo_assert_held(imported); + + mem_type = XE_PL_VRAM0; + if (!(params->mem_mask & XE_BO_CREATE_VRAM0_BIT)) + /* No VRAM allowed */ + mem_type = XE_PL_TT; + else if (params->force_different_devices && !p2p_enabled(params)) + /* No P2P */ + mem_type = XE_PL_TT; + else if (params->force_different_devices && !is_dynamic(params) && + (params->mem_mask & XE_BO_CREATE_SYSTEM_BIT)) + /* Pin migrated to TT */ + mem_type = XE_PL_TT; + + if (!xe_bo_is_mem_type(exported, mem_type)) { + KUNIT_FAIL(test, "Exported bo was not in expected memory type.\n"); + return; + } + + if (xe_bo_is_pinned(exported)) + return; + + /* + * Evict exporter. Note that the gem object dma_buf member isn't + * set from xe_gem_prime_export(), and it's needed for the move_notify() + * functionality, so hack that up here. Evicting the exported bo will + * evict also the imported bo through the move_notify() functionality if + * importer is on a different device. If they're on the same device, + * the exporter and the importer should be the same bo. + */ + swap(exported->ttm.base.dma_buf, dmabuf); + ret = xe_bo_evict(exported, true); + swap(exported->ttm.base.dma_buf, dmabuf); + if (ret) { + if (ret != -EINTR && ret != -ERESTARTSYS) + KUNIT_FAIL(test, "Evicting exporter failed with err=%d.\n", + ret); + return; + } + + /* Verify that also importer has been evicted to SYSTEM */ + if (!xe_bo_is_mem_type(imported, XE_PL_SYSTEM)) { + KUNIT_FAIL(test, "Importer wasn't properly evicted.\n"); + return; + } + + /* Re-validate the importer. This should move also exporter in. */ + ret = xe_bo_validate(imported, NULL, false); + if (ret) { + if (ret != -EINTR && ret != -ERESTARTSYS) + KUNIT_FAIL(test, "Validating importer failed with err=%d.\n", + ret); + return; + } + + /* + * If on different devices, the exporter is kept in system if + * possible, saving a migration step as the transfer is just + * likely as fast from system memory. + */ + if (params->force_different_devices && + params->mem_mask & XE_BO_CREATE_SYSTEM_BIT) + KUNIT_EXPECT_TRUE(test, xe_bo_is_mem_type(exported, XE_PL_TT)); + else + KUNIT_EXPECT_TRUE(test, xe_bo_is_mem_type(exported, mem_type)); + + if (params->force_different_devices) + KUNIT_EXPECT_TRUE(test, xe_bo_is_mem_type(imported, XE_PL_TT)); + else + KUNIT_EXPECT_TRUE(test, exported == imported); +} + +static void xe_test_dmabuf_import_same_driver(struct xe_device *xe) +{ + struct kunit *test = xe_cur_kunit(); + struct dma_buf_test_params *params = to_dma_buf_test_params(test->priv); + struct drm_gem_object *import; + struct dma_buf *dmabuf; + struct xe_bo *bo; + + /* No VRAM on this device? */ + if (!ttm_manager_type(&xe->ttm, XE_PL_VRAM0) && + (params->mem_mask & XE_BO_CREATE_VRAM0_BIT)) + return; + + kunit_info(test, "running %s\n", __func__); + bo = xe_bo_create(xe, NULL, NULL, PAGE_SIZE, ttm_bo_type_device, + XE_BO_CREATE_USER_BIT | params->mem_mask); + if (IS_ERR(bo)) { + KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n", + PTR_ERR(bo)); + return; + } + + dmabuf = xe_gem_prime_export(&bo->ttm.base, 0); + if (IS_ERR(dmabuf)) { + KUNIT_FAIL(test, "xe_gem_prime_export() failed with err=%ld\n", + PTR_ERR(dmabuf)); + goto out; + } + + import = xe_gem_prime_import(&xe->drm, dmabuf); + if (!IS_ERR(import)) { + struct xe_bo *import_bo = gem_to_xe_bo(import); + + /* + * Did import succeed when it shouldn't due to lack of p2p support? + */ + if (params->force_different_devices && + !p2p_enabled(params) && + !(params->mem_mask & XE_BO_CREATE_SYSTEM_BIT)) { + KUNIT_FAIL(test, + "xe_gem_prime_import() succeeded when it shouldn't have\n"); + } else { + int err; + + /* Is everything where we expect it to be? */ + xe_bo_lock_no_vm(import_bo, NULL); + err = xe_bo_validate(import_bo, NULL, false); + if (err && err != -EINTR && err != -ERESTARTSYS) + KUNIT_FAIL(test, + "xe_bo_validate() failed with err=%d\n", err); + + check_residency(test, bo, import_bo, dmabuf); + xe_bo_unlock_no_vm(import_bo); + } + drm_gem_object_put(import); + } else if (PTR_ERR(import) != -EOPNOTSUPP) { + /* Unexpected error code. */ + KUNIT_FAIL(test, + "xe_gem_prime_import failed with the wrong err=%ld\n", + PTR_ERR(import)); + } else if (!params->force_different_devices || + p2p_enabled(params) || + (params->mem_mask & XE_BO_CREATE_SYSTEM_BIT)) { + /* Shouldn't fail if we can reuse same bo, use p2p or use system */ + KUNIT_FAIL(test, "dynamic p2p attachment failed with err=%ld\n", + PTR_ERR(import)); + } + dma_buf_put(dmabuf); +out: + drm_gem_object_put(&bo->ttm.base); +} + +static const struct dma_buf_attach_ops nop2p_attach_ops = { + .allow_peer2peer = false, + .move_notify = xe_dma_buf_move_notify +}; + +/* + * We test the implementation with bos of different residency and with + * importers with different capabilities; some lacking p2p support and some + * lacking dynamic capabilities (attach_ops == NULL). We also fake + * different devices avoiding the import shortcut that just reuses the same + * gem object. + */ +static const struct dma_buf_test_params test_params[] = { + {.mem_mask = XE_BO_CREATE_VRAM0_BIT, + .attach_ops = &xe_dma_buf_attach_ops}, + {.mem_mask = XE_BO_CREATE_VRAM0_BIT, + .attach_ops = &xe_dma_buf_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_CREATE_VRAM0_BIT, + .attach_ops = &nop2p_attach_ops}, + {.mem_mask = XE_BO_CREATE_VRAM0_BIT, + .attach_ops = &nop2p_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_CREATE_VRAM0_BIT}, + {.mem_mask = XE_BO_CREATE_VRAM0_BIT, + .force_different_devices = true}, + + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT, + .attach_ops = &xe_dma_buf_attach_ops}, + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT, + .attach_ops = &xe_dma_buf_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT, + .attach_ops = &nop2p_attach_ops}, + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT, + .attach_ops = &nop2p_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT}, + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT, + .force_different_devices = true}, + + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT, + .attach_ops = &xe_dma_buf_attach_ops}, + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT, + .attach_ops = &xe_dma_buf_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT, + .attach_ops = &nop2p_attach_ops}, + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT, + .attach_ops = &nop2p_attach_ops, + .force_different_devices = true}, + + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT}, + {.mem_mask = XE_BO_CREATE_SYSTEM_BIT | XE_BO_CREATE_VRAM0_BIT, + .force_different_devices = true}, + + {} +}; + +static int dma_buf_run_device(struct xe_device *xe) +{ + const struct dma_buf_test_params *params; + struct kunit *test = xe_cur_kunit(); + + for (params = test_params; params->mem_mask; ++params) { + struct dma_buf_test_params p = *params; + + p.base.id = XE_TEST_LIVE_DMA_BUF; + test->priv = &p; + xe_test_dmabuf_import_same_driver(xe); + } + + /* A non-zero return would halt iteration over driver devices */ + return 0; +} + +void xe_dma_buf_kunit(struct kunit *test) +{ + xe_call_for_each_device(dma_buf_run_device); +} +EXPORT_SYMBOL(xe_dma_buf_kunit); diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf_test.c b/drivers/gpu/drm/xe/tests/xe_dma_buf_test.c new file mode 100644 index 000000000000..7bb292da1193 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf_test.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +void xe_dma_buf_kunit(struct kunit *test); + +static struct kunit_case xe_dma_buf_tests[] = { + KUNIT_CASE(xe_dma_buf_kunit), + {} +}; + +static struct kunit_suite xe_dma_buf_test_suite = { + .name = "xe_dma_buf", + .test_cases = xe_dma_buf_tests, +}; + +kunit_test_suite(xe_dma_buf_test_suite); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c new file mode 100644 index 000000000000..0f3b819f0a34 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -0,0 +1,378 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020-2022 Intel Corporation + */ + +#include + +#include "xe_pci.h" + +static bool sanity_fence_failed(struct xe_device *xe, struct dma_fence *fence, + const char *str, struct kunit *test) +{ + long ret; + + if (IS_ERR(fence)) { + KUNIT_FAIL(test, "Failed to create fence for %s: %li\n", str, + PTR_ERR(fence)); + return true; + } + if (!fence) + return true; + + ret = dma_fence_wait_timeout(fence, false, 5 * HZ); + if (ret <= 0) { + KUNIT_FAIL(test, "Fence timed out for %s: %li\n", str, ret); + return true; + } + + return false; +} + +static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe, + struct xe_bb *bb, u32 second_idx, const char *str, + struct kunit *test) +{ + struct xe_sched_job *job = xe_bb_create_migration_job(m->eng, bb, + m->batch_base_ofs, + second_idx); + struct dma_fence *fence; + + if (IS_ERR(job)) { + KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", + PTR_ERR(job)); + return PTR_ERR(job); + } + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + if (sanity_fence_failed(xe, fence, str, test)) + return -ETIMEDOUT; + + dma_fence_put(fence); + kunit_info(test, "%s: Job completed\n", str); + return 0; +} + +static void +sanity_populate_cb(struct xe_migrate_pt_update *pt_update, + struct xe_gt *gt, struct iosys_map *map, void *dst, + u32 qword_ofs, u32 num_qwords, + const struct xe_vm_pgtable_update *update) +{ + int i; + u64 *ptr = dst; + + for (i = 0; i < num_qwords; i++) + ptr[i] = (qword_ofs + i - update->ofs) * 0x1111111111111111ULL; +} + +static const struct xe_migrate_pt_update_ops sanity_ops = { + .populate = sanity_populate_cb, +}; + +#define check(_retval, _expected, str, _test) \ + do { if ((_retval) != (_expected)) { \ + KUNIT_FAIL(_test, "Sanity check failed: " str \ + " expected %llx, got %llx\n", \ + (u64)(_expected), (u64)(_retval)); \ + } } while (0) + +static void test_copy(struct xe_migrate *m, struct xe_bo *bo, + struct kunit *test) +{ + struct xe_device *xe = gt_to_xe(m->gt); + u64 retval, expected = 0xc0c0c0c0c0c0c0c0ULL; + bool big = bo->size >= SZ_2M; + struct dma_fence *fence; + const char *str = big ? "Copying big bo" : "Copying small bo"; + int err; + + struct xe_bo *sysmem = xe_bo_create_locked(xe, m->gt, NULL, + bo->size, + ttm_bo_type_kernel, + XE_BO_CREATE_SYSTEM_BIT); + if (IS_ERR(sysmem)) { + KUNIT_FAIL(test, "Failed to allocate sysmem bo for %s: %li\n", + str, PTR_ERR(sysmem)); + return; + } + + err = xe_bo_validate(sysmem, NULL, false); + if (err) { + KUNIT_FAIL(test, "Failed to validate system bo for %s: %li\n", + str, err); + goto out_unlock; + } + + err = xe_bo_vmap(sysmem); + if (err) { + KUNIT_FAIL(test, "Failed to vmap system bo for %s: %li\n", + str, err); + goto out_unlock; + } + + xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size); + fence = xe_migrate_clear(m, sysmem, sysmem->ttm.resource, 0xc0c0c0c0); + if (!sanity_fence_failed(xe, fence, big ? "Clearing sysmem big bo" : + "Clearing sysmem small bo", test)) { + retval = xe_map_rd(xe, &sysmem->vmap, 0, u64); + check(retval, expected, "sysmem first offset should be cleared", + test); + retval = xe_map_rd(xe, &sysmem->vmap, sysmem->size - 8, u64); + check(retval, expected, "sysmem last offset should be cleared", + test); + } + dma_fence_put(fence); + + /* Try to copy 0xc0 from sysmem to lmem with 2MB or 64KiB/4KiB pages */ + xe_map_memset(xe, &sysmem->vmap, 0, 0xc0, sysmem->size); + xe_map_memset(xe, &bo->vmap, 0, 0xd0, bo->size); + + fence = xe_migrate_copy(m, sysmem, sysmem->ttm.resource, + bo->ttm.resource); + if (!sanity_fence_failed(xe, fence, big ? "Copying big bo sysmem -> vram" : + "Copying small bo sysmem -> vram", test)) { + retval = xe_map_rd(xe, &bo->vmap, 0, u64); + check(retval, expected, + "sysmem -> vram bo first offset should be copied", test); + retval = xe_map_rd(xe, &bo->vmap, bo->size - 8, u64); + check(retval, expected, + "sysmem -> vram bo offset should be copied", test); + } + dma_fence_put(fence); + + /* And other way around.. slightly hacky.. */ + xe_map_memset(xe, &sysmem->vmap, 0, 0xd0, sysmem->size); + xe_map_memset(xe, &bo->vmap, 0, 0xc0, bo->size); + + fence = xe_migrate_copy(m, sysmem, bo->ttm.resource, + sysmem->ttm.resource); + if (!sanity_fence_failed(xe, fence, big ? "Copying big bo vram -> sysmem" : + "Copying small bo vram -> sysmem", test)) { + retval = xe_map_rd(xe, &sysmem->vmap, 0, u64); + check(retval, expected, + "vram -> sysmem bo first offset should be copied", test); + retval = xe_map_rd(xe, &sysmem->vmap, bo->size - 8, u64); + check(retval, expected, + "vram -> sysmem bo last offset should be copied", test); + } + dma_fence_put(fence); + + xe_bo_vunmap(sysmem); +out_unlock: + xe_bo_unlock_no_vm(sysmem); + xe_bo_put(sysmem); +} + +static void test_pt_update(struct xe_migrate *m, struct xe_bo *pt, + struct kunit *test) +{ + struct xe_device *xe = gt_to_xe(m->gt); + struct dma_fence *fence; + u64 retval, expected; + int i; + + struct xe_vm_pgtable_update update = { + .ofs = 1, + .qwords = 0x10, + .pt_bo = pt, + }; + struct xe_migrate_pt_update pt_update = { + .ops = &sanity_ops, + }; + + /* Test xe_migrate_update_pgtables() updates the pagetable as expected */ + expected = 0xf0f0f0f0f0f0f0f0ULL; + xe_map_memset(xe, &pt->vmap, 0, (u8)expected, pt->size); + + fence = xe_migrate_update_pgtables(m, NULL, NULL, m->eng, &update, 1, + NULL, 0, &pt_update); + if (sanity_fence_failed(xe, fence, "Migration pagetable update", test)) + return; + + dma_fence_put(fence); + retval = xe_map_rd(xe, &pt->vmap, 0, u64); + check(retval, expected, "PTE[0] must stay untouched", test); + + for (i = 0; i < update.qwords; i++) { + retval = xe_map_rd(xe, &pt->vmap, (update.ofs + i) * 8, u64); + check(retval, i * 0x1111111111111111ULL, "PTE update", test); + } + + retval = xe_map_rd(xe, &pt->vmap, 8 * (update.ofs + update.qwords), + u64); + check(retval, expected, "PTE[0x11] must stay untouched", test); +} + +static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) +{ + struct xe_gt *gt = m->gt; + struct xe_device *xe = gt_to_xe(gt); + struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny; + struct xe_res_cursor src_it; + struct dma_fence *fence; + u64 retval, expected; + struct xe_bb *bb; + int err; + u8 id = gt->info.id; + + err = xe_bo_vmap(bo); + if (err) { + KUNIT_FAIL(test, "Failed to vmap our pagetables: %li\n", + PTR_ERR(bo)); + return; + } + + big = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, SZ_4M, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(m->gt) | + XE_BO_CREATE_PINNED_BIT); + if (IS_ERR(big)) { + KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big)); + goto vunmap; + } + + pt = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, GEN8_PAGE_SIZE, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(m->gt) | + XE_BO_CREATE_PINNED_BIT); + if (IS_ERR(pt)) { + KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", + PTR_ERR(pt)); + goto free_big; + } + + tiny = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, + 2 * SZ_4K, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(m->gt) | + XE_BO_CREATE_PINNED_BIT); + if (IS_ERR(tiny)) { + KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", + PTR_ERR(pt)); + goto free_pt; + } + + bb = xe_bb_new(m->gt, 32, xe->info.supports_usm); + if (IS_ERR(bb)) { + KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n", + PTR_ERR(bb)); + goto free_tiny; + } + + kunit_info(test, "Starting tests, top level PT addr: %llx, special pagetable base addr: %llx\n", + xe_bo_main_addr(m->eng->vm->pt_root[id]->bo, GEN8_PAGE_SIZE), + xe_bo_main_addr(m->pt_bo, GEN8_PAGE_SIZE)); + + /* First part of the test, are we updating our pagetable bo with a new entry? */ + xe_map_wr(xe, &bo->vmap, GEN8_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64, 0xdeaddeadbeefbeef); + expected = gen8_pte_encode(NULL, pt, 0, XE_CACHE_WB, 0, 0); + if (m->eng->vm->flags & XE_VM_FLAGS_64K) + expected |= GEN12_PTE_PS64; + xe_res_first(pt->ttm.resource, 0, pt->size, &src_it); + emit_pte(m, bb, NUM_KERNEL_PDE - 1, xe_bo_is_vram(pt), + &src_it, GEN8_PAGE_SIZE, pt); + run_sanity_job(m, xe, bb, bb->len, "Writing PTE for our fake PT", test); + + retval = xe_map_rd(xe, &bo->vmap, GEN8_PAGE_SIZE * (NUM_KERNEL_PDE - 1), + u64); + check(retval, expected, "PTE entry write", test); + + /* Now try to write data to our newly mapped 'pagetable', see if it succeeds */ + bb->len = 0; + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + xe_map_wr(xe, &pt->vmap, 0, u32, 0xdeaddead); + expected = 0x12345678U; + + emit_clear(m->gt, bb, xe_migrate_vm_addr(NUM_KERNEL_PDE - 1, 0), 4, 4, + expected, IS_DGFX(xe)); + run_sanity_job(m, xe, bb, 1, "Writing to our newly mapped pagetable", + test); + + retval = xe_map_rd(xe, &pt->vmap, 0, u32); + check(retval, expected, "Write to PT after adding PTE", test); + + /* Sanity checks passed, try the full ones! */ + + /* Clear a small bo */ + kunit_info(test, "Clearing small buffer object\n"); + xe_map_memset(xe, &tiny->vmap, 0, 0x22, tiny->size); + expected = 0x224488ff; + fence = xe_migrate_clear(m, tiny, tiny->ttm.resource, expected); + if (sanity_fence_failed(xe, fence, "Clearing small bo", test)) + goto out; + + dma_fence_put(fence); + retval = xe_map_rd(xe, &tiny->vmap, 0, u32); + check(retval, expected, "Command clear small first value", test); + retval = xe_map_rd(xe, &tiny->vmap, tiny->size - 4, u32); + check(retval, expected, "Command clear small last value", test); + + if (IS_DGFX(xe)) { + kunit_info(test, "Copying small buffer object to system\n"); + test_copy(m, tiny, test); + } + + /* Clear a big bo with a fixed value */ + kunit_info(test, "Clearing big buffer object\n"); + xe_map_memset(xe, &big->vmap, 0, 0x11, big->size); + expected = 0x11223344U; + fence = xe_migrate_clear(m, big, big->ttm.resource, expected); + if (sanity_fence_failed(xe, fence, "Clearing big bo", test)) + goto out; + + dma_fence_put(fence); + retval = xe_map_rd(xe, &big->vmap, 0, u32); + check(retval, expected, "Command clear big first value", test); + retval = xe_map_rd(xe, &big->vmap, big->size - 4, u32); + check(retval, expected, "Command clear big last value", test); + + if (IS_DGFX(xe)) { + kunit_info(test, "Copying big buffer object to system\n"); + test_copy(m, big, test); + } + + test_pt_update(m, pt, test); + +out: + xe_bb_free(bb, NULL); +free_tiny: + xe_bo_unpin(tiny); + xe_bo_put(tiny); +free_pt: + xe_bo_unpin(pt); + xe_bo_put(pt); +free_big: + xe_bo_unpin(big); + xe_bo_put(big); +vunmap: + xe_bo_vunmap(m->pt_bo); +} + +static int migrate_test_run_device(struct xe_device *xe) +{ + struct kunit *test = xe_cur_kunit(); + struct xe_gt *gt; + int id; + + for_each_gt(gt, xe, id) { + struct xe_migrate *m = gt->migrate; + struct ww_acquire_ctx ww; + + kunit_info(test, "Testing gt id %d.\n", id); + xe_vm_lock(m->eng->vm, &ww, 0, true); + xe_migrate_sanity_test(m, test); + xe_vm_unlock(m->eng->vm, &ww); + } + + return 0; +} + +void xe_migrate_sanity_kunit(struct kunit *test) +{ + xe_call_for_each_device(migrate_test_run_device); +} +EXPORT_SYMBOL(xe_migrate_sanity_kunit); diff --git a/drivers/gpu/drm/xe/tests/xe_migrate_test.c b/drivers/gpu/drm/xe/tests/xe_migrate_test.c new file mode 100644 index 000000000000..ad779e2bd071 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_migrate_test.c @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +void xe_migrate_sanity_kunit(struct kunit *test); + +static struct kunit_case xe_migrate_tests[] = { + KUNIT_CASE(xe_migrate_sanity_kunit), + {} +}; + +static struct kunit_suite xe_migrate_test_suite = { + .name = "xe_migrate", + .test_cases = xe_migrate_tests, +}; + +kunit_test_suite(xe_migrate_test_suite); + +MODULE_AUTHOR("Intel Corporation"); +MODULE_LICENSE("GPL"); diff --git a/drivers/gpu/drm/xe/tests/xe_test.h b/drivers/gpu/drm/xe/tests/xe_test.h new file mode 100644 index 000000000000..1ec502b5acf3 --- /dev/null +++ b/drivers/gpu/drm/xe/tests/xe_test.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0 AND MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __XE_TEST_H__ +#define __XE_TEST_H__ + +#include + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include +#include + +/* + * Each test that provides a kunit private test structure, place a test id + * here and point the kunit->priv to an embedded struct xe_test_priv. + */ +enum xe_test_priv_id { + XE_TEST_LIVE_DMA_BUF, +}; + +/** + * struct xe_test_priv - Base class for test private info + * @id: enum xe_test_priv_id to identify the subclass. + */ +struct xe_test_priv { + enum xe_test_priv_id id; +}; + +#define XE_TEST_DECLARE(x) x +#define XE_TEST_ONLY(x) unlikely(x) +#define XE_TEST_EXPORT +#define xe_cur_kunit() current->kunit_test + +/** + * xe_cur_kunit_priv - Obtain the struct xe_test_priv pointed to by + * current->kunit->priv if it exists and is embedded in the expected subclass. + * @id: Id of the expected subclass. + * + * Return: NULL if the process is not a kunit test, and NULL if the + * current kunit->priv pointer is not pointing to an object of the expected + * subclass. A pointer to the embedded struct xe_test_priv otherwise. + */ +static inline struct xe_test_priv * +xe_cur_kunit_priv(enum xe_test_priv_id id) +{ + struct xe_test_priv *priv; + + if (!xe_cur_kunit()) + return NULL; + + priv = xe_cur_kunit()->priv; + return priv->id == id ? priv : NULL; +} + +#else /* if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) */ + +#define XE_TEST_DECLARE(x) +#define XE_TEST_ONLY(x) 0 +#define XE_TEST_EXPORT static +#define xe_cur_kunit() NULL +#define xe_cur_kunit_priv(_id) NULL + +#endif +#endif diff --git a/drivers/gpu/drm/xe/xe_bb.c b/drivers/gpu/drm/xe/xe_bb.c new file mode 100644 index 000000000000..8b9209571fd0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bb.c @@ -0,0 +1,97 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_bb.h" +#include "xe_sa.h" +#include "xe_device.h" +#include "xe_engine_types.h" +#include "xe_hw_fence.h" +#include "xe_sched_job.h" +#include "xe_vm_types.h" + +#include "gt/intel_gpu_commands.h" + +struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 dwords, bool usm) +{ + struct xe_bb *bb = kmalloc(sizeof(*bb), GFP_KERNEL); + int err; + + if (!bb) + return ERR_PTR(-ENOMEM); + + bb->bo = xe_sa_bo_new(!usm ? >->kernel_bb_pool : + >->usm.bb_pool, 4 * dwords + 4); + if (IS_ERR(bb->bo)) { + err = PTR_ERR(bb->bo); + goto err; + } + + bb->cs = xe_sa_bo_cpu_addr(bb->bo); + bb->len = 0; + + return bb; +err: + kfree(bb); + return ERR_PTR(err); +} + +static struct xe_sched_job * +__xe_bb_create_job(struct xe_engine *kernel_eng, struct xe_bb *bb, u64 *addr) +{ + u32 size = drm_suballoc_size(bb->bo); + + XE_BUG_ON((bb->len * 4 + 1) > size); + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + + xe_sa_bo_flush_write(bb->bo); + + return xe_sched_job_create(kernel_eng, addr); +} + +struct xe_sched_job *xe_bb_create_wa_job(struct xe_engine *wa_eng, + struct xe_bb *bb, u64 batch_base_ofs) +{ + u64 addr = batch_base_ofs + drm_suballoc_soffset(bb->bo); + + XE_BUG_ON(!(wa_eng->vm->flags & XE_VM_FLAG_MIGRATION)); + + return __xe_bb_create_job(wa_eng, bb, &addr); +} + +struct xe_sched_job *xe_bb_create_migration_job(struct xe_engine *kernel_eng, + struct xe_bb *bb, + u64 batch_base_ofs, + u32 second_idx) +{ + u64 addr[2] = { + batch_base_ofs + drm_suballoc_soffset(bb->bo), + batch_base_ofs + drm_suballoc_soffset(bb->bo) + + 4 * second_idx, + }; + + BUG_ON(second_idx > bb->len); + BUG_ON(!(kernel_eng->vm->flags & XE_VM_FLAG_MIGRATION)); + + return __xe_bb_create_job(kernel_eng, bb, addr); +} + +struct xe_sched_job *xe_bb_create_job(struct xe_engine *kernel_eng, + struct xe_bb *bb) +{ + u64 addr = xe_sa_bo_gpu_addr(bb->bo); + + BUG_ON(kernel_eng->vm && kernel_eng->vm->flags & XE_VM_FLAG_MIGRATION); + return __xe_bb_create_job(kernel_eng, bb, &addr); +} + +void xe_bb_free(struct xe_bb *bb, struct dma_fence *fence) +{ + if (!bb) + return; + + xe_sa_bo_free(bb->bo, fence); + kfree(bb); +} diff --git a/drivers/gpu/drm/xe/xe_bb.h b/drivers/gpu/drm/xe/xe_bb.h new file mode 100644 index 000000000000..0cc9260c9634 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bb.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BB_H_ +#define _XE_BB_H_ + +#include "xe_bb_types.h" + +struct dma_fence; + +struct xe_gt; +struct xe_engine; +struct xe_sched_job; + +struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 size, bool usm); +struct xe_sched_job *xe_bb_create_job(struct xe_engine *kernel_eng, + struct xe_bb *bb); +struct xe_sched_job *xe_bb_create_migration_job(struct xe_engine *kernel_eng, + struct xe_bb *bb, u64 batch_ofs, + u32 second_idx); +struct xe_sched_job *xe_bb_create_wa_job(struct xe_engine *wa_eng, + struct xe_bb *bb, u64 batch_ofs); +void xe_bb_free(struct xe_bb *bb, struct dma_fence *fence); + +#endif diff --git a/drivers/gpu/drm/xe/xe_bb_types.h b/drivers/gpu/drm/xe/xe_bb_types.h new file mode 100644 index 000000000000..b7d30308cf90 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bb_types.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BB_TYPES_H_ +#define _XE_BB_TYPES_H_ + +#include + +struct drm_suballoc; + +struct xe_bb { + struct drm_suballoc *bo; + + u32 *cs; + u32 len; /* in dwords */ +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c new file mode 100644 index 000000000000..ef2c9196c113 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -0,0 +1,1698 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + + +#include "xe_bo.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "xe_device.h" +#include "xe_dma_buf.h" +#include "xe_ggtt.h" +#include "xe_gt.h" +#include "xe_map.h" +#include "xe_migrate.h" +#include "xe_preempt_fence.h" +#include "xe_res_cursor.h" +#include "xe_trace.h" +#include "xe_vm.h" + +static const struct ttm_place sys_placement_flags = { + .fpfn = 0, + .lpfn = 0, + .mem_type = XE_PL_SYSTEM, + .flags = 0, +}; + +static struct ttm_placement sys_placement = { + .num_placement = 1, + .placement = &sys_placement_flags, + .num_busy_placement = 1, + .busy_placement = &sys_placement_flags, +}; + +bool mem_type_is_vram(u32 mem_type) +{ + return mem_type >= XE_PL_VRAM0; +} + +static bool resource_is_vram(struct ttm_resource *res) +{ + return mem_type_is_vram(res->mem_type); +} + +bool xe_bo_is_vram(struct xe_bo *bo) +{ + return resource_is_vram(bo->ttm.resource); +} + +static bool xe_bo_is_user(struct xe_bo *bo) +{ + return bo->flags & XE_BO_CREATE_USER_BIT; +} + +static struct xe_gt * +mem_type_to_gt(struct xe_device *xe, u32 mem_type) +{ + XE_BUG_ON(!mem_type_is_vram(mem_type)); + + return xe_device_get_gt(xe, mem_type - XE_PL_VRAM0); +} + +static void try_add_system(struct xe_bo *bo, struct ttm_place *places, + u32 bo_flags, u32 *c) +{ + if (bo_flags & XE_BO_CREATE_SYSTEM_BIT) { + places[*c] = (struct ttm_place) { + .mem_type = XE_PL_TT, + }; + *c += 1; + + if (bo->props.preferred_mem_type == XE_BO_PROPS_INVALID) + bo->props.preferred_mem_type = XE_PL_TT; + } +} + +static void try_add_vram0(struct xe_device *xe, struct xe_bo *bo, + struct ttm_place *places, u32 bo_flags, u32 *c) +{ + struct xe_gt *gt; + + if (bo_flags & XE_BO_CREATE_VRAM0_BIT) { + gt = mem_type_to_gt(xe, XE_PL_VRAM0); + XE_BUG_ON(!gt->mem.vram.size); + + places[*c] = (struct ttm_place) { + .mem_type = XE_PL_VRAM0, + /* + * For eviction / restore on suspend / resume objects + * pinned in VRAM must be contiguous + */ + .flags = bo_flags & (XE_BO_CREATE_PINNED_BIT | + XE_BO_CREATE_GGTT_BIT) ? + TTM_PL_FLAG_CONTIGUOUS : 0, + }; + *c += 1; + + if (bo->props.preferred_mem_type == XE_BO_PROPS_INVALID) + bo->props.preferred_mem_type = XE_PL_VRAM0; + } +} + +static void try_add_vram1(struct xe_device *xe, struct xe_bo *bo, + struct ttm_place *places, u32 bo_flags, u32 *c) +{ + struct xe_gt *gt; + + if (bo_flags & XE_BO_CREATE_VRAM1_BIT) { + gt = mem_type_to_gt(xe, XE_PL_VRAM1); + XE_BUG_ON(!gt->mem.vram.size); + + places[*c] = (struct ttm_place) { + .mem_type = XE_PL_VRAM1, + /* + * For eviction / restore on suspend / resume objects + * pinned in VRAM must be contiguous + */ + .flags = bo_flags & (XE_BO_CREATE_PINNED_BIT | + XE_BO_CREATE_GGTT_BIT) ? + TTM_PL_FLAG_CONTIGUOUS : 0, + }; + *c += 1; + + if (bo->props.preferred_mem_type == XE_BO_PROPS_INVALID) + bo->props.preferred_mem_type = XE_PL_VRAM1; + } +} + +static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, + u32 bo_flags) +{ + struct ttm_place *places = bo->placements; + u32 c = 0; + + bo->props.preferred_mem_type = XE_BO_PROPS_INVALID; + + /* The order of placements should indicate preferred location */ + + if (bo->props.preferred_mem_class == XE_MEM_REGION_CLASS_SYSMEM) { + try_add_system(bo, places, bo_flags, &c); + if (bo->props.preferred_gt == XE_GT1) { + try_add_vram1(xe, bo, places, bo_flags, &c); + try_add_vram0(xe, bo, places, bo_flags, &c); + } else { + try_add_vram0(xe, bo, places, bo_flags, &c); + try_add_vram1(xe, bo, places, bo_flags, &c); + } + } else if (bo->props.preferred_gt == XE_GT1) { + try_add_vram1(xe, bo, places, bo_flags, &c); + try_add_vram0(xe, bo, places, bo_flags, &c); + try_add_system(bo, places, bo_flags, &c); + } else { + try_add_vram0(xe, bo, places, bo_flags, &c); + try_add_vram1(xe, bo, places, bo_flags, &c); + try_add_system(bo, places, bo_flags, &c); + } + + if (!c) + return -EINVAL; + + bo->placement = (struct ttm_placement) { + .num_placement = c, + .placement = places, + .num_busy_placement = c, + .busy_placement = places, + }; + + return 0; +} + +int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, + u32 bo_flags) +{ + xe_bo_assert_held(bo); + return __xe_bo_placement_for_flags(xe, bo, bo_flags); +} + +static void xe_evict_flags(struct ttm_buffer_object *tbo, + struct ttm_placement *placement) +{ + struct xe_bo *bo; + + if (!xe_bo_is_xe_bo(tbo)) { + /* Don't handle scatter gather BOs */ + if (tbo->type == ttm_bo_type_sg) { + placement->num_placement = 0; + placement->num_busy_placement = 0; + return; + } + + *placement = sys_placement; + return; + } + + /* + * For xe, sg bos that are evicted to system just triggers a + * rebind of the sg list upon subsequent validation to XE_PL_TT. + */ + + bo = ttm_to_xe_bo(tbo); + switch (tbo->resource->mem_type) { + case XE_PL_VRAM0: + case XE_PL_VRAM1: + case XE_PL_TT: + default: + /* for now kick out to system */ + *placement = sys_placement; + break; + } +} + +struct xe_ttm_tt { + struct ttm_tt ttm; + struct device *dev; + struct sg_table sgt; + struct sg_table *sg; +}; + +static int xe_tt_map_sg(struct ttm_tt *tt) +{ + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + unsigned long num_pages = tt->num_pages; + int ret; + + XE_BUG_ON(tt->page_flags & TTM_TT_FLAG_EXTERNAL); + + if (xe_tt->sg) + return 0; + + ret = sg_alloc_table_from_pages(&xe_tt->sgt, tt->pages, num_pages, + 0, (u64)num_pages << PAGE_SHIFT, + GFP_KERNEL); + if (ret) + return ret; + + xe_tt->sg = &xe_tt->sgt; + ret = dma_map_sgtable(xe_tt->dev, xe_tt->sg, DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC); + if (ret) { + sg_free_table(xe_tt->sg); + xe_tt->sg = NULL; + return ret; + } + + return 0; +} + +struct sg_table *xe_bo_get_sg(struct xe_bo *bo) +{ + struct ttm_tt *tt = bo->ttm.ttm; + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + + return xe_tt->sg; +} + +static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo, + u32 page_flags) +{ + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + struct xe_device *xe = xe_bo_device(bo); + struct xe_ttm_tt *tt; + int err; + + tt = kzalloc(sizeof(*tt), GFP_KERNEL); + if (!tt) + return NULL; + + tt->dev = xe->drm.dev; + + /* TODO: Select caching mode */ + err = ttm_tt_init(&tt->ttm, &bo->ttm, page_flags, + bo->flags & XE_BO_SCANOUT_BIT ? ttm_write_combined : ttm_cached, + DIV_ROUND_UP(xe_device_ccs_bytes(xe_bo_device(bo), + bo->ttm.base.size), + PAGE_SIZE)); + if (err) { + kfree(tt); + return NULL; + } + + return &tt->ttm; +} + +static int xe_ttm_tt_populate(struct ttm_device *ttm_dev, struct ttm_tt *tt, + struct ttm_operation_ctx *ctx) +{ + int err; + + /* + * dma-bufs are not populated with pages, and the dma- + * addresses are set up when moved to XE_PL_TT. + */ + if (tt->page_flags & TTM_TT_FLAG_EXTERNAL) + return 0; + + err = ttm_pool_alloc(&ttm_dev->pool, tt, ctx); + if (err) + return err; + + /* A follow up may move this xe_bo_move when BO is moved to XE_PL_TT */ + err = xe_tt_map_sg(tt); + if (err) + ttm_pool_free(&ttm_dev->pool, tt); + + return err; +} + +static void xe_ttm_tt_unpopulate(struct ttm_device *ttm_dev, struct ttm_tt *tt) +{ + struct xe_ttm_tt *xe_tt = container_of(tt, struct xe_ttm_tt, ttm); + + if (tt->page_flags & TTM_TT_FLAG_EXTERNAL) + return; + + if (xe_tt->sg) { + dma_unmap_sgtable(xe_tt->dev, xe_tt->sg, + DMA_BIDIRECTIONAL, 0); + sg_free_table(xe_tt->sg); + xe_tt->sg = NULL; + } + + return ttm_pool_free(&ttm_dev->pool, tt); +} + +static void xe_ttm_tt_destroy(struct ttm_device *ttm_dev, struct ttm_tt *tt) +{ + ttm_tt_fini(tt); + kfree(tt); +} + +static int xe_ttm_io_mem_reserve(struct ttm_device *bdev, + struct ttm_resource *mem) +{ + struct xe_device *xe = ttm_to_xe_device(bdev); + struct xe_gt *gt; + + switch (mem->mem_type) { + case XE_PL_SYSTEM: + case XE_PL_TT: + return 0; + case XE_PL_VRAM0: + case XE_PL_VRAM1: + gt = mem_type_to_gt(xe, mem->mem_type); + mem->bus.offset = mem->start << PAGE_SHIFT; + + if (gt->mem.vram.mapping && + mem->placement & TTM_PL_FLAG_CONTIGUOUS) + mem->bus.addr = (u8 *)gt->mem.vram.mapping + + mem->bus.offset; + + mem->bus.offset += gt->mem.vram.io_start; + mem->bus.is_iomem = true; + +#if !defined(CONFIG_X86) + mem->bus.caching = ttm_write_combined; +#endif + break; + default: + return -EINVAL; + } + return 0; +} + +static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo, + const struct ttm_operation_ctx *ctx) +{ + struct dma_resv_iter cursor; + struct dma_fence *fence; + struct xe_vma *vma; + int ret = 0; + + dma_resv_assert_held(bo->ttm.base.resv); + + if (!xe_device_in_fault_mode(xe) && !list_empty(&bo->vmas)) { + dma_resv_iter_begin(&cursor, bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) + dma_fence_enable_sw_signaling(fence); + dma_resv_iter_end(&cursor); + } + + list_for_each_entry(vma, &bo->vmas, bo_link) { + struct xe_vm *vm = vma->vm; + + trace_xe_vma_evict(vma); + + if (xe_vm_in_fault_mode(vm)) { + /* Wait for pending binds / unbinds. */ + long timeout; + + if (ctx->no_wait_gpu && + !dma_resv_test_signaled(bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP)) + return -EBUSY; + + timeout = dma_resv_wait_timeout(bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP, + ctx->interruptible, + MAX_SCHEDULE_TIMEOUT); + if (timeout > 0) { + ret = xe_vm_invalidate_vma(vma); + XE_WARN_ON(ret); + } else if (!timeout) { + ret = -ETIME; + } else { + ret = timeout; + } + + } else { + bool vm_resv_locked = false; + struct xe_vm *vm = vma->vm; + + /* + * We need to put the vma on the vm's rebind_list, + * but need the vm resv to do so. If we can't verify + * that we indeed have it locked, put the vma an the + * vm's notifier.rebind_list instead and scoop later. + */ + if (dma_resv_trylock(&vm->resv)) + vm_resv_locked = true; + else if (ctx->resv != &vm->resv) { + spin_lock(&vm->notifier.list_lock); + list_move_tail(&vma->notifier.rebind_link, + &vm->notifier.rebind_list); + spin_unlock(&vm->notifier.list_lock); + continue; + } + + xe_vm_assert_held(vm); + if (list_empty(&vma->rebind_link) && vma->gt_present) + list_add_tail(&vma->rebind_link, &vm->rebind_list); + + if (vm_resv_locked) + dma_resv_unlock(&vm->resv); + } + } + + return ret; +} + +/* + * The dma-buf map_attachment() / unmap_attachment() is hooked up here. + * Note that unmapping the attachment is deferred to the next + * map_attachment time, or to bo destroy (after idling) whichever comes first. + * This is to avoid syncing before unmap_attachment(), assuming that the + * caller relies on idling the reservation object before moving the + * backing store out. Should that assumption not hold, then we will be able + * to unconditionally call unmap_attachment() when moving out to system. + */ +static int xe_bo_move_dmabuf(struct ttm_buffer_object *ttm_bo, + struct ttm_resource *old_res, + struct ttm_resource *new_res) +{ + struct dma_buf_attachment *attach = ttm_bo->base.import_attach; + struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, struct xe_ttm_tt, + ttm); + struct sg_table *sg; + + XE_BUG_ON(!attach); + XE_BUG_ON(!ttm_bo->ttm); + + if (new_res->mem_type == XE_PL_SYSTEM) + goto out; + + if (ttm_bo->sg) { + dma_buf_unmap_attachment(attach, ttm_bo->sg, DMA_BIDIRECTIONAL); + ttm_bo->sg = NULL; + } + + sg = dma_buf_map_attachment(attach, DMA_BIDIRECTIONAL); + if (IS_ERR(sg)) + return PTR_ERR(sg); + + ttm_bo->sg = sg; + xe_tt->sg = sg; + +out: + ttm_bo_move_null(ttm_bo, new_res); + + return 0; +} + +/** + * xe_bo_move_notify - Notify subsystems of a pending move + * @bo: The buffer object + * @ctx: The struct ttm_operation_ctx controlling locking and waits. + * + * This function notifies subsystems of an upcoming buffer move. + * Upon receiving such a notification, subsystems should schedule + * halting access to the underlying pages and optionally add a fence + * to the buffer object's dma_resv object, that signals when access is + * stopped. The caller will wait on all dma_resv fences before + * starting the move. + * + * A subsystem may commence access to the object after obtaining + * bindings to the new backing memory under the object lock. + * + * Return: 0 on success, -EINTR or -ERESTARTSYS if interrupted in fault mode, + * negative error code on error. + */ +static int xe_bo_move_notify(struct xe_bo *bo, + const struct ttm_operation_ctx *ctx) +{ + struct ttm_buffer_object *ttm_bo = &bo->ttm; + struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); + int ret; + + /* + * If this starts to call into many components, consider + * using a notification chain here. + */ + + if (xe_bo_is_pinned(bo)) + return -EINVAL; + + xe_bo_vunmap(bo); + ret = xe_bo_trigger_rebind(xe, bo, ctx); + if (ret) + return ret; + + /* Don't call move_notify() for imported dma-bufs. */ + if (ttm_bo->base.dma_buf && !ttm_bo->base.import_attach) + dma_buf_move_notify(ttm_bo->base.dma_buf); + + return 0; +} + +static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, + struct ttm_operation_ctx *ctx, + struct ttm_resource *new_mem, + struct ttm_place *hop) +{ + struct xe_device *xe = ttm_to_xe_device(ttm_bo->bdev); + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + struct ttm_resource *old_mem = ttm_bo->resource; + struct ttm_tt *ttm = ttm_bo->ttm; + struct xe_gt *gt = NULL; + struct dma_fence *fence; + bool move_lacks_source; + bool needs_clear; + int ret = 0; + + if (!old_mem) { + if (new_mem->mem_type != TTM_PL_SYSTEM) { + hop->mem_type = TTM_PL_SYSTEM; + hop->flags = TTM_PL_FLAG_TEMPORARY; + ret = -EMULTIHOP; + goto out; + } + + ttm_bo_move_null(ttm_bo, new_mem); + goto out; + } + + if (ttm_bo->type == ttm_bo_type_sg) { + ret = xe_bo_move_notify(bo, ctx); + if (!ret) + ret = xe_bo_move_dmabuf(ttm_bo, old_mem, new_mem); + goto out; + } + + move_lacks_source = !resource_is_vram(old_mem) && + (!ttm || !ttm_tt_is_populated(ttm)); + + needs_clear = (ttm && ttm->page_flags & TTM_TT_FLAG_ZERO_ALLOC) || + (!ttm && ttm_bo->type == ttm_bo_type_device); + + if ((move_lacks_source && !needs_clear) || + (old_mem->mem_type == XE_PL_SYSTEM && + new_mem->mem_type == XE_PL_TT)) { + ttm_bo_move_null(ttm_bo, new_mem); + goto out; + } + + if (!move_lacks_source && !xe_bo_is_pinned(bo)) { + ret = xe_bo_move_notify(bo, ctx); + if (ret) + goto out; + } + + if (old_mem->mem_type == XE_PL_TT && + new_mem->mem_type == XE_PL_SYSTEM) { + long timeout = dma_resv_wait_timeout(ttm_bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP, + true, + MAX_SCHEDULE_TIMEOUT); + if (timeout < 0) { + ret = timeout; + goto out; + } + ttm_bo_move_null(ttm_bo, new_mem); + goto out; + } + + if (!move_lacks_source && + ((old_mem->mem_type == XE_PL_SYSTEM && resource_is_vram(new_mem)) || + (resource_is_vram(old_mem) && + new_mem->mem_type == XE_PL_SYSTEM))) { + hop->fpfn = 0; + hop->lpfn = 0; + hop->mem_type = XE_PL_TT; + hop->flags = TTM_PL_FLAG_TEMPORARY; + ret = -EMULTIHOP; + goto out; + } + + if (bo->gt) + gt = bo->gt; + else if (resource_is_vram(new_mem)) + gt = mem_type_to_gt(xe, new_mem->mem_type); + else if (resource_is_vram(old_mem)) + gt = mem_type_to_gt(xe, old_mem->mem_type); + + XE_BUG_ON(!gt); + XE_BUG_ON(!gt->migrate); + + trace_xe_bo_move(bo); + xe_device_mem_access_get(xe); + + if (xe_bo_is_pinned(bo) && !xe_bo_is_user(bo)) { + /* + * Kernel memory that is pinned should only be moved on suspend + * / resume, some of the pinned memory is required for the + * device to resume / use the GPU to move other evicted memory + * (user memory) around. This likely could be optimized a bit + * futher where we find the minimum set of pinned memory + * required for resume but for simplity doing a memcpy for all + * pinned memory. + */ + ret = xe_bo_vmap(bo); + if (!ret) { + ret = ttm_bo_move_memcpy(ttm_bo, ctx, new_mem); + + /* Create a new VMAP once kernel BO back in VRAM */ + if (!ret && resource_is_vram(new_mem)) { + void *new_addr = gt->mem.vram.mapping + + (new_mem->start << PAGE_SHIFT); + + XE_BUG_ON(new_mem->start != + bo->placements->fpfn); + + iosys_map_set_vaddr_iomem(&bo->vmap, new_addr); + } + } + } else { + if (move_lacks_source) + fence = xe_migrate_clear(gt->migrate, bo, new_mem, 0); + else + fence = xe_migrate_copy(gt->migrate, bo, old_mem, new_mem); + if (IS_ERR(fence)) { + ret = PTR_ERR(fence); + xe_device_mem_access_put(xe); + goto out; + } + ret = ttm_bo_move_accel_cleanup(ttm_bo, fence, evict, true, + new_mem); + dma_fence_put(fence); + } + + xe_device_mem_access_put(xe); + trace_printk("new_mem->mem_type=%d\n", new_mem->mem_type); + +out: + return ret; + +} + +static unsigned long xe_ttm_io_mem_pfn(struct ttm_buffer_object *bo, + unsigned long page_offset) +{ + struct xe_device *xe = ttm_to_xe_device(bo->bdev); + struct xe_gt *gt = mem_type_to_gt(xe, bo->resource->mem_type); + struct xe_res_cursor cursor; + + xe_res_first(bo->resource, (u64)page_offset << PAGE_SHIFT, 0, &cursor); + return (gt->mem.vram.io_start + cursor.start) >> PAGE_SHIFT; +} + +static void __xe_bo_vunmap(struct xe_bo *bo); + +/* + * TODO: Move this function to TTM so we don't rely on how TTM does its + * locking, thereby abusing TTM internals. + */ +static bool xe_ttm_bo_lock_in_destructor(struct ttm_buffer_object *ttm_bo) +{ + bool locked; + + XE_WARN_ON(kref_read(&ttm_bo->kref)); + + /* + * We can typically only race with TTM trylocking under the + * lru_lock, which will immediately be unlocked again since + * the ttm_bo refcount is zero at this point. So trylocking *should* + * always succeed here, as long as we hold the lru lock. + */ + spin_lock(&ttm_bo->bdev->lru_lock); + locked = dma_resv_trylock(ttm_bo->base.resv); + spin_unlock(&ttm_bo->bdev->lru_lock); + XE_WARN_ON(!locked); + + return locked; +} + +static void xe_ttm_bo_release_notify(struct ttm_buffer_object *ttm_bo) +{ + struct dma_resv_iter cursor; + struct dma_fence *fence; + struct dma_fence *replacement = NULL; + struct xe_bo *bo; + + if (!xe_bo_is_xe_bo(ttm_bo)) + return; + + bo = ttm_to_xe_bo(ttm_bo); + XE_WARN_ON(bo->created && kref_read(&ttm_bo->base.refcount)); + + /* + * Corner case where TTM fails to allocate memory and this BOs resv + * still points the VMs resv + */ + if (ttm_bo->base.resv != &ttm_bo->base._resv) + return; + + if (!xe_ttm_bo_lock_in_destructor(ttm_bo)) + return; + + /* + * Scrub the preempt fences if any. The unbind fence is already + * attached to the resv. + * TODO: Don't do this for external bos once we scrub them after + * unbind. + */ + dma_resv_for_each_fence(&cursor, ttm_bo->base.resv, + DMA_RESV_USAGE_BOOKKEEP, fence) { + if (xe_fence_is_xe_preempt(fence) && + !dma_fence_is_signaled(fence)) { + if (!replacement) + replacement = dma_fence_get_stub(); + + dma_resv_replace_fences(ttm_bo->base.resv, + fence->context, + replacement, + DMA_RESV_USAGE_BOOKKEEP); + } + } + dma_fence_put(replacement); + + dma_resv_unlock(ttm_bo->base.resv); +} + +static void xe_ttm_bo_delete_mem_notify(struct ttm_buffer_object *ttm_bo) +{ + if (!xe_bo_is_xe_bo(ttm_bo)) + return; + + /* + * Object is idle and about to be destroyed. Release the + * dma-buf attachment. + */ + if (ttm_bo->type == ttm_bo_type_sg && ttm_bo->sg) { + struct xe_ttm_tt *xe_tt = container_of(ttm_bo->ttm, + struct xe_ttm_tt, ttm); + + dma_buf_unmap_attachment(ttm_bo->base.import_attach, ttm_bo->sg, + DMA_BIDIRECTIONAL); + ttm_bo->sg = NULL; + xe_tt->sg = NULL; + } +} + +struct ttm_device_funcs xe_ttm_funcs = { + .ttm_tt_create = xe_ttm_tt_create, + .ttm_tt_populate = xe_ttm_tt_populate, + .ttm_tt_unpopulate = xe_ttm_tt_unpopulate, + .ttm_tt_destroy = xe_ttm_tt_destroy, + .evict_flags = xe_evict_flags, + .move = xe_bo_move, + .io_mem_reserve = xe_ttm_io_mem_reserve, + .io_mem_pfn = xe_ttm_io_mem_pfn, + .release_notify = xe_ttm_bo_release_notify, + .eviction_valuable = ttm_bo_eviction_valuable, + .delete_mem_notify = xe_ttm_bo_delete_mem_notify, +}; + +static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo) +{ + struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); + + if (bo->ttm.base.import_attach) + drm_prime_gem_destroy(&bo->ttm.base, NULL); + drm_gem_object_release(&bo->ttm.base); + + WARN_ON(!list_empty(&bo->vmas)); + + if (bo->ggtt_node.size) + xe_ggtt_remove_bo(bo->gt->mem.ggtt, bo); + + if (bo->vm && xe_bo_is_user(bo)) + xe_vm_put(bo->vm); + + kfree(bo); +} + +static void xe_gem_object_free(struct drm_gem_object *obj) +{ + /* Our BO reference counting scheme works as follows: + * + * The gem object kref is typically used throughout the driver, + * and the gem object holds a ttm_buffer_object refcount, so + * that when the last gem object reference is put, which is when + * we end up in this function, we put also that ttm_buffer_object + * refcount. Anything using gem interfaces is then no longer + * allowed to access the object in a way that requires a gem + * refcount, including locking the object. + * + * driver ttm callbacks is allowed to use the ttm_buffer_object + * refcount directly if needed. + */ + __xe_bo_vunmap(gem_to_xe_bo(obj)); + ttm_bo_put(container_of(obj, struct ttm_buffer_object, base)); +} + +static bool should_migrate_to_system(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + + return xe_device_in_fault_mode(xe) && bo->props.cpu_atomic; +} + +static vm_fault_t xe_gem_fault(struct vm_fault *vmf) +{ + struct ttm_buffer_object *tbo = vmf->vma->vm_private_data; + struct drm_device *ddev = tbo->base.dev; + vm_fault_t ret; + int idx, r = 0; + + ret = ttm_bo_vm_reserve(tbo, vmf); + if (ret) + return ret; + + if (drm_dev_enter(ddev, &idx)) { + struct xe_bo *bo = ttm_to_xe_bo(tbo); + + trace_xe_bo_cpu_fault(bo); + + if (should_migrate_to_system(bo)) { + r = xe_bo_migrate(bo, XE_PL_TT); + if (r == -EBUSY || r == -ERESTARTSYS || r == -EINTR) + ret = VM_FAULT_NOPAGE; + else if (r) + ret = VM_FAULT_SIGBUS; + } + if (!ret) + ret = ttm_bo_vm_fault_reserved(vmf, + vmf->vma->vm_page_prot, + TTM_BO_VM_NUM_PREFAULT); + + drm_dev_exit(idx); + } else { + ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); + } + if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) + return ret; + + dma_resv_unlock(tbo->base.resv); + return ret; +} + +static const struct vm_operations_struct xe_gem_vm_ops = { + .fault = xe_gem_fault, + .open = ttm_bo_vm_open, + .close = ttm_bo_vm_close, + .access = ttm_bo_vm_access +}; + +static const struct drm_gem_object_funcs xe_gem_object_funcs = { + .free = xe_gem_object_free, + .mmap = drm_gem_ttm_mmap, + .export = xe_gem_prime_export, + .vm_ops = &xe_gem_vm_ops, +}; + +/** + * xe_bo_alloc - Allocate storage for a struct xe_bo + * + * This funcition is intended to allocate storage to be used for input + * to __xe_bo_create_locked(), in the case a pointer to the bo to be + * created is needed before the call to __xe_bo_create_locked(). + * If __xe_bo_create_locked ends up never to be called, then the + * storage allocated with this function needs to be freed using + * xe_bo_free(). + * + * Return: A pointer to an uninitialized struct xe_bo on success, + * ERR_PTR(-ENOMEM) on error. + */ +struct xe_bo *xe_bo_alloc(void) +{ + struct xe_bo *bo = kzalloc(sizeof(*bo), GFP_KERNEL); + + if (!bo) + return ERR_PTR(-ENOMEM); + + return bo; +} + +/** + * xe_bo_free - Free storage allocated using xe_bo_alloc() + * @bo: The buffer object storage. + * + * Refer to xe_bo_alloc() documentation for valid use-cases. + */ +void xe_bo_free(struct xe_bo *bo) +{ + kfree(bo); +} + +struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, + struct xe_gt *gt, struct dma_resv *resv, + size_t size, enum ttm_bo_type type, + u32 flags) +{ + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + }; + struct ttm_placement *placement; + uint32_t alignment; + int err; + + /* Only kernel objects should set GT */ + XE_BUG_ON(gt && type != ttm_bo_type_kernel); + + if (!bo) { + bo = xe_bo_alloc(); + if (IS_ERR(bo)) + return bo; + } + + if (flags & (XE_BO_CREATE_VRAM0_BIT | XE_BO_CREATE_VRAM1_BIT) && + !(flags & XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT) && + xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) { + size = ALIGN(size, SZ_64K); + flags |= XE_BO_INTERNAL_64K; + alignment = SZ_64K >> PAGE_SHIFT; + } else { + alignment = SZ_4K >> PAGE_SHIFT; + } + + bo->gt = gt; + bo->size = size; + bo->flags = flags; + bo->ttm.base.funcs = &xe_gem_object_funcs; + bo->props.preferred_mem_class = XE_BO_PROPS_INVALID; + bo->props.preferred_gt = XE_BO_PROPS_INVALID; + bo->props.preferred_mem_type = XE_BO_PROPS_INVALID; + bo->ttm.priority = DRM_XE_VMA_PRIORITY_NORMAL; + INIT_LIST_HEAD(&bo->vmas); + INIT_LIST_HEAD(&bo->pinned_link); + + drm_gem_private_object_init(&xe->drm, &bo->ttm.base, size); + + if (resv) { + ctx.allow_res_evict = true; + ctx.resv = resv; + } + + err = __xe_bo_placement_for_flags(xe, bo, bo->flags); + if (WARN_ON(err)) + return ERR_PTR(err); + + /* Defer populating type_sg bos */ + placement = (type == ttm_bo_type_sg || + bo->flags & XE_BO_DEFER_BACKING) ? &sys_placement : + &bo->placement; + err = ttm_bo_init_reserved(&xe->ttm, &bo->ttm, type, + placement, alignment, + &ctx, NULL, resv, xe_ttm_bo_destroy); + if (err) + return ERR_PTR(err); + + bo->created = true; + ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); + + return bo; +} + +struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags) +{ + struct xe_bo *bo; + int err; + + if (vm) + xe_vm_assert_held(vm); + bo = __xe_bo_create_locked(xe, NULL, gt, vm ? &vm->resv : NULL, size, + type, flags); + if (IS_ERR(bo)) + return bo; + + if (vm && xe_bo_is_user(bo)) + xe_vm_get(vm); + bo->vm = vm; + + if (flags & XE_BO_CREATE_GGTT_BIT) { + XE_BUG_ON(!gt); + + err = xe_ggtt_insert_bo(gt->mem.ggtt, bo); + if (err) + goto err_unlock_put_bo; + } + + return bo; + +err_unlock_put_bo: + xe_bo_unlock_vm_held(bo); + xe_bo_put(bo); + return ERR_PTR(err); +} + +struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags) +{ + struct xe_bo *bo = xe_bo_create_locked(xe, gt, vm, size, type, flags); + + if (!IS_ERR(bo)) + xe_bo_unlock_vm_held(bo); + + return bo; +} + +struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags) +{ + struct xe_bo *bo = xe_bo_create_locked(xe, gt, vm, size, type, flags); + int err; + + if (IS_ERR(bo)) + return bo; + + err = xe_bo_pin(bo); + if (err) + goto err_put; + + err = xe_bo_vmap(bo); + if (err) + goto err_unpin; + + xe_bo_unlock_vm_held(bo); + + return bo; + +err_unpin: + xe_bo_unpin(bo); +err_put: + xe_bo_unlock_vm_held(bo); + xe_bo_put(bo); + return ERR_PTR(err); +} + +struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_gt *gt, + const void *data, size_t size, + enum ttm_bo_type type, u32 flags) +{ + struct xe_bo *bo = xe_bo_create_pin_map(xe, gt, NULL, + ALIGN(size, PAGE_SIZE), + type, flags); + if (IS_ERR(bo)) + return bo; + + xe_map_memcpy_to(xe, &bo->vmap, 0, data, size); + + return bo; +} + +/* + * XXX: This is in the VM bind data path, likely should calculate this once and + * store, with a recalculation if the BO is moved. + */ +static uint64_t vram_region_io_offset(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + struct xe_gt *gt = mem_type_to_gt(xe, bo->ttm.resource->mem_type); + + return gt->mem.vram.io_start - xe->mem.vram.io_start; +} + +/** + * xe_bo_pin_external - pin an external BO + * @bo: buffer object to be pinned + * + * Pin an external (not tied to a VM, can be exported via dma-buf / prime FD) + * BO. Unique call compared to xe_bo_pin as this function has it own set of + * asserts and code to ensure evict / restore on suspend / resume. + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_bo_pin_external(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + int err; + + XE_BUG_ON(bo->vm); + XE_BUG_ON(!xe_bo_is_user(bo)); + + if (!xe_bo_is_pinned(bo)) { + err = xe_bo_validate(bo, NULL, false); + if (err) + return err; + + if (xe_bo_is_vram(bo)) { + spin_lock(&xe->pinned.lock); + list_add_tail(&bo->pinned_link, + &xe->pinned.external_vram); + spin_unlock(&xe->pinned.lock); + } + } + + ttm_bo_pin(&bo->ttm); + + /* + * FIXME: If we always use the reserve / unreserve functions for locking + * we do not need this. + */ + ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); + + return 0; +} + +int xe_bo_pin(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + int err; + + /* We currently don't expect user BO to be pinned */ + XE_BUG_ON(xe_bo_is_user(bo)); + + /* Pinned object must be in GGTT or have pinned flag */ + XE_BUG_ON(!(bo->flags & (XE_BO_CREATE_PINNED_BIT | + XE_BO_CREATE_GGTT_BIT))); + + /* + * No reason we can't support pinning imported dma-bufs we just don't + * expect to pin an imported dma-buf. + */ + XE_BUG_ON(bo->ttm.base.import_attach); + + /* We only expect at most 1 pin */ + XE_BUG_ON(xe_bo_is_pinned(bo)); + + err = xe_bo_validate(bo, NULL, false); + if (err) + return err; + + /* + * For pinned objects in on DGFX, we expect these objects to be in + * contiguous VRAM memory. Required eviction / restore during suspend / + * resume (force restore to same physical address). + */ + if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) && + bo->flags & XE_BO_INTERNAL_TEST)) { + struct ttm_place *place = &(bo->placements[0]); + bool lmem; + + XE_BUG_ON(!(place->flags & TTM_PL_FLAG_CONTIGUOUS)); + XE_BUG_ON(!mem_type_is_vram(place->mem_type)); + + place->fpfn = (xe_bo_addr(bo, 0, PAGE_SIZE, &lmem) - + vram_region_io_offset(bo)) >> PAGE_SHIFT; + place->lpfn = place->fpfn + (bo->size >> PAGE_SHIFT); + + spin_lock(&xe->pinned.lock); + list_add_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present); + spin_unlock(&xe->pinned.lock); + } + + ttm_bo_pin(&bo->ttm); + + /* + * FIXME: If we always use the reserve / unreserve functions for locking + * we do not need this. + */ + ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); + + return 0; +} + +/** + * xe_bo_unpin_external - unpin an external BO + * @bo: buffer object to be unpinned + * + * Unpin an external (not tied to a VM, can be exported via dma-buf / prime FD) + * BO. Unique call compared to xe_bo_unpin as this function has it own set of + * asserts and code to ensure evict / restore on suspend / resume. + * + * Returns 0 for success, negative error code otherwise. + */ +void xe_bo_unpin_external(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + + XE_BUG_ON(bo->vm); + XE_BUG_ON(!xe_bo_is_pinned(bo)); + XE_BUG_ON(!xe_bo_is_user(bo)); + + if (bo->ttm.pin_count == 1 && !list_empty(&bo->pinned_link)) { + spin_lock(&xe->pinned.lock); + list_del_init(&bo->pinned_link); + spin_unlock(&xe->pinned.lock); + } + + ttm_bo_unpin(&bo->ttm); + + /* + * FIXME: If we always use the reserve / unreserve functions for locking + * we do not need this. + */ + ttm_bo_move_to_lru_tail_unlocked(&bo->ttm); +} + +void xe_bo_unpin(struct xe_bo *bo) +{ + struct xe_device *xe = xe_bo_device(bo); + + XE_BUG_ON(bo->ttm.base.import_attach); + XE_BUG_ON(!xe_bo_is_pinned(bo)); + + if (IS_DGFX(xe) && !(IS_ENABLED(CONFIG_DRM_XE_DEBUG) && + bo->flags & XE_BO_INTERNAL_TEST)) { + XE_BUG_ON(list_empty(&bo->pinned_link)); + + spin_lock(&xe->pinned.lock); + list_del_init(&bo->pinned_link); + spin_unlock(&xe->pinned.lock); + } + + ttm_bo_unpin(&bo->ttm); +} + +/** + * xe_bo_validate() - Make sure the bo is in an allowed placement + * @bo: The bo, + * @vm: Pointer to a the vm the bo shares a locked dma_resv object with, or + * NULL. Used together with @allow_res_evict. + * @allow_res_evict: Whether it's allowed to evict bos sharing @vm's + * reservation object. + * + * Make sure the bo is in allowed placement, migrating it if necessary. If + * needed, other bos will be evicted. If bos selected for eviction shares + * the @vm's reservation object, they can be evicted iff @allow_res_evict is + * set to true, otherwise they will be bypassed. + * + * Return: 0 on success, negative error code on failure. May return + * -EINTR or -ERESTARTSYS if internal waits are interrupted by a signal. + */ +int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict) +{ + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + }; + + if (vm) { + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + + ctx.allow_res_evict = allow_res_evict; + ctx.resv = &vm->resv; + } + + return ttm_bo_validate(&bo->ttm, &bo->placement, &ctx); +} + +bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo) +{ + if (bo->destroy == &xe_ttm_bo_destroy) + return true; + + return false; +} + +dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, + size_t page_size, bool *is_lmem) +{ + struct xe_res_cursor cur; + u64 page; + + if (!READ_ONCE(bo->ttm.pin_count)) + xe_bo_assert_held(bo); + + XE_BUG_ON(page_size > PAGE_SIZE); + page = offset >> PAGE_SHIFT; + offset &= (PAGE_SIZE - 1); + + *is_lmem = xe_bo_is_vram(bo); + + if (!*is_lmem) { + XE_BUG_ON(!bo->ttm.ttm); + + xe_res_first_sg(xe_bo_get_sg(bo), page << PAGE_SHIFT, + page_size, &cur); + return xe_res_dma(&cur) + offset; + } else { + struct xe_res_cursor cur; + + xe_res_first(bo->ttm.resource, page << PAGE_SHIFT, + page_size, &cur); + return cur.start + offset + vram_region_io_offset(bo); + } +} + +int xe_bo_vmap(struct xe_bo *bo) +{ + void *virtual; + bool is_iomem; + int ret; + + xe_bo_assert_held(bo); + + if (!iosys_map_is_null(&bo->vmap)) + return 0; + + /* + * We use this more or less deprecated interface for now since + * ttm_bo_vmap() doesn't offer the optimization of kmapping + * single page bos, which is done here. + * TODO: Fix up ttm_bo_vmap to do that, or fix up ttm_bo_kmap + * to use struct iosys_map. + */ + ret = ttm_bo_kmap(&bo->ttm, 0, bo->size >> PAGE_SHIFT, &bo->kmap); + if (ret) + return ret; + + virtual = ttm_kmap_obj_virtual(&bo->kmap, &is_iomem); + if (is_iomem) + iosys_map_set_vaddr_iomem(&bo->vmap, (void __iomem *)virtual); + else + iosys_map_set_vaddr(&bo->vmap, virtual); + + return 0; +} + +static void __xe_bo_vunmap(struct xe_bo *bo) +{ + if (!iosys_map_is_null(&bo->vmap)) { + iosys_map_clear(&bo->vmap); + ttm_bo_kunmap(&bo->kmap); + } +} + +void xe_bo_vunmap(struct xe_bo *bo) +{ + xe_bo_assert_held(bo); + __xe_bo_vunmap(bo); +} + +int xe_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_gem_create *args = data; + struct ww_acquire_ctx ww; + struct xe_vm *vm = NULL; + struct xe_bo *bo; + unsigned bo_flags = XE_BO_CREATE_USER_BIT; + u32 handle; + int err; + + if (XE_IOCTL_ERR(xe, args->extensions)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->flags & + ~(XE_GEM_CREATE_FLAG_DEFER_BACKING | + XE_GEM_CREATE_FLAG_SCANOUT | + xe->info.mem_region_mask))) + return -EINVAL; + + /* at least one memory type must be specified */ + if (XE_IOCTL_ERR(xe, !(args->flags & xe->info.mem_region_mask))) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->handle)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->size > SIZE_MAX)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->size & ~PAGE_MASK)) + return -EINVAL; + + if (args->vm_id) { + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_ERR(xe, !vm)) + return -ENOENT; + err = xe_vm_lock(vm, &ww, 0, true); + if (err) { + xe_vm_put(vm); + return err; + } + } + + if (args->flags & XE_GEM_CREATE_FLAG_DEFER_BACKING) + bo_flags |= XE_BO_DEFER_BACKING; + + if (args->flags & XE_GEM_CREATE_FLAG_SCANOUT) + bo_flags |= XE_BO_SCANOUT_BIT; + + bo_flags |= args->flags << (ffs(XE_BO_CREATE_SYSTEM_BIT) - 1); + bo = xe_bo_create(xe, NULL, vm, args->size, ttm_bo_type_device, + bo_flags); + if (vm) { + xe_vm_unlock(vm, &ww); + xe_vm_put(vm); + } + + if (IS_ERR(bo)) + return PTR_ERR(bo); + + err = drm_gem_handle_create(file, &bo->ttm.base, &handle); + xe_bo_put(bo); + if (err) + return err; + + args->handle = handle; + + return 0; +} + +int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct drm_xe_gem_mmap_offset *args = data; + struct drm_gem_object *gem_obj; + + if (XE_IOCTL_ERR(xe, args->extensions)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->flags)) + return -EINVAL; + + gem_obj = drm_gem_object_lookup(file, args->handle); + if (XE_IOCTL_ERR(xe, !gem_obj)) + return -ENOENT; + + /* The mmap offset was set up at BO allocation time. */ + args->offset = drm_vma_node_offset_addr(&gem_obj->vma_node); + + xe_bo_put(gem_to_xe_bo(gem_obj)); + return 0; +} + +int xe_bo_lock(struct xe_bo *bo, struct ww_acquire_ctx *ww, + int num_resv, bool intr) +{ + struct ttm_validate_buffer tv_bo; + LIST_HEAD(objs); + LIST_HEAD(dups); + + XE_BUG_ON(!ww); + + tv_bo.num_shared = num_resv; + tv_bo.bo = &bo->ttm;; + list_add_tail(&tv_bo.head, &objs); + + return ttm_eu_reserve_buffers(ww, &objs, intr, &dups); +} + +void xe_bo_unlock(struct xe_bo *bo, struct ww_acquire_ctx *ww) +{ + dma_resv_unlock(bo->ttm.base.resv); + ww_acquire_fini(ww); +} + +/** + * xe_bo_can_migrate - Whether a buffer object likely can be migrated + * @bo: The buffer object to migrate + * @mem_type: The TTM memory type intended to migrate to + * + * Check whether the buffer object supports migration to the + * given memory type. Note that pinning may affect the ability to migrate as + * returned by this function. + * + * This function is primarily intended as a helper for checking the + * possibility to migrate buffer objects and can be called without + * the object lock held. + * + * Return: true if migration is possible, false otherwise. + */ +bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type) +{ + unsigned int cur_place; + + if (bo->ttm.type == ttm_bo_type_kernel) + return true; + + if (bo->ttm.type == ttm_bo_type_sg) + return false; + + for (cur_place = 0; cur_place < bo->placement.num_placement; + cur_place++) { + if (bo->placements[cur_place].mem_type == mem_type) + return true; + } + + return false; +} + +static void xe_place_from_ttm_type(u32 mem_type, struct ttm_place *place) +{ + memset(place, 0, sizeof(*place)); + place->mem_type = mem_type; +} + +/** + * xe_bo_migrate - Migrate an object to the desired region id + * @bo: The buffer object to migrate. + * @mem_type: The TTM region type to migrate to. + * + * Attempt to migrate the buffer object to the desired memory region. The + * buffer object may not be pinned, and must be locked. + * On successful completion, the object memory type will be updated, + * but an async migration task may not have completed yet, and to + * accomplish that, the object's kernel fences must be signaled with + * the object lock held. + * + * Return: 0 on success. Negative error code on failure. In particular may + * return -EINTR or -ERESTARTSYS if signal pending. + */ +int xe_bo_migrate(struct xe_bo *bo, u32 mem_type) +{ + struct ttm_operation_ctx ctx = { + .interruptible = true, + .no_wait_gpu = false, + }; + struct ttm_placement placement; + struct ttm_place requested; + + xe_bo_assert_held(bo); + + if (bo->ttm.resource->mem_type == mem_type) + return 0; + + if (xe_bo_is_pinned(bo)) + return -EBUSY; + + if (!xe_bo_can_migrate(bo, mem_type)) + return -EINVAL; + + xe_place_from_ttm_type(mem_type, &requested); + placement.num_placement = 1; + placement.num_busy_placement = 1; + placement.placement = &requested; + placement.busy_placement = &requested; + + return ttm_bo_validate(&bo->ttm, &placement, &ctx); +} + +/** + * xe_bo_evict - Evict an object to evict placement + * @bo: The buffer object to migrate. + * @force_alloc: Set force_alloc in ttm_operation_ctx + * + * On successful completion, the object memory will be moved to evict + * placement. Ths function blocks until the object has been fully moved. + * + * Return: 0 on success. Negative error code on failure. + */ +int xe_bo_evict(struct xe_bo *bo, bool force_alloc) +{ + struct ttm_operation_ctx ctx = { + .interruptible = false, + .no_wait_gpu = false, + .force_alloc = force_alloc, + }; + struct ttm_placement placement; + int ret; + + xe_evict_flags(&bo->ttm, &placement); + ret = ttm_bo_validate(&bo->ttm, &placement, &ctx); + if (ret) + return ret; + + dma_resv_wait_timeout(bo->ttm.base.resv, DMA_RESV_USAGE_KERNEL, + false, MAX_SCHEDULE_TIMEOUT); + + return 0; +} + +/** + * xe_bo_needs_ccs_pages - Whether a bo needs to back up CCS pages when + * placed in system memory. + * @bo: The xe_bo + * + * If a bo has an allowable placement in XE_PL_TT memory, it can't use + * flat CCS compression, because the GPU then has no way to access the + * CCS metadata using relevant commands. For the opposite case, we need to + * allocate storage for the CCS metadata when the BO is not resident in + * VRAM memory. + * + * Return: true if extra pages need to be allocated, false otherwise. + */ +bool xe_bo_needs_ccs_pages(struct xe_bo *bo) +{ + return bo->ttm.type == ttm_bo_type_device && + !(bo->flags & XE_BO_CREATE_SYSTEM_BIT) && + (bo->flags & (XE_BO_CREATE_VRAM0_BIT | XE_BO_CREATE_VRAM1_BIT)); +} + +/** + * __xe_bo_release_dummy() - Dummy kref release function + * @kref: The embedded struct kref. + * + * Dummy release function for xe_bo_put_deferred(). Keep off. + */ +void __xe_bo_release_dummy(struct kref *kref) +{ +} + +/** + * xe_bo_put_commit() - Put bos whose put was deferred by xe_bo_put_deferred(). + * @deferred: The lockless list used for the call to xe_bo_put_deferred(). + * + * Puts all bos whose put was deferred by xe_bo_put_deferred(). + * The @deferred list can be either an onstack local list or a global + * shared list used by a workqueue. + */ +void xe_bo_put_commit(struct llist_head *deferred) +{ + struct llist_node *freed; + struct xe_bo *bo, *next; + + if (!deferred) + return; + + freed = llist_del_all(deferred); + if (!freed) + return; + + llist_for_each_entry_safe(bo, next, freed, freed) + drm_gem_object_free(&bo->ttm.base.refcount); +} + +/** + * xe_bo_dumb_create - Create a dumb bo as backing for a fb + * @file_priv: ... + * @dev: ... + * @args: ... + * + * See dumb_create() hook in include/drm/drm_drv.h + * + * Return: ... + */ +int xe_bo_dumb_create(struct drm_file *file_priv, + struct drm_device *dev, + struct drm_mode_create_dumb *args) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_bo *bo; + uint32_t handle; + int cpp = DIV_ROUND_UP(args->bpp, 8); + int err; + u32 page_size = max_t(u32, PAGE_SIZE, + xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K); + + args->pitch = ALIGN(args->width * cpp, 64); + args->size = ALIGN(mul_u32_u32(args->pitch, args->height), + page_size); + + bo = xe_bo_create(xe, NULL, NULL, args->size, ttm_bo_type_device, + XE_BO_CREATE_VRAM_IF_DGFX(to_gt(xe)) | + XE_BO_CREATE_USER_BIT | XE_BO_SCANOUT_BIT); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + err = drm_gem_handle_create(file_priv, &bo->ttm.base, &handle); + /* drop reference from allocate - handle holds it now */ + drm_gem_object_put(&bo->ttm.base); + if (!err) + args->handle = handle; + return err; +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_bo.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h new file mode 100644 index 000000000000..1a49c0a3c4c6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -0,0 +1,290 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_BO_H_ +#define _XE_BO_H_ + +#include "xe_bo_types.h" +#include "xe_macros.h" +#include "xe_vm_types.h" + +#define XE_DEFAULT_GTT_SIZE_MB 3072ULL /* 3GB by default */ + +#define XE_BO_CREATE_USER_BIT BIT(1) +#define XE_BO_CREATE_SYSTEM_BIT BIT(2) +#define XE_BO_CREATE_VRAM0_BIT BIT(3) +#define XE_BO_CREATE_VRAM1_BIT BIT(4) +#define XE_BO_CREATE_VRAM_IF_DGFX(gt) \ + (IS_DGFX(gt_to_xe(gt)) ? XE_BO_CREATE_VRAM0_BIT << gt->info.vram_id : \ + XE_BO_CREATE_SYSTEM_BIT) +#define XE_BO_CREATE_GGTT_BIT BIT(5) +#define XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT BIT(6) +#define XE_BO_CREATE_PINNED_BIT BIT(7) +#define XE_BO_DEFER_BACKING BIT(8) +#define XE_BO_SCANOUT_BIT BIT(9) +/* this one is trigger internally only */ +#define XE_BO_INTERNAL_TEST BIT(30) +#define XE_BO_INTERNAL_64K BIT(31) + +#define PPAT_UNCACHED GENMASK_ULL(4, 3) +#define PPAT_CACHED_PDE 0 +#define PPAT_CACHED BIT_ULL(7) +#define PPAT_DISPLAY_ELLC BIT_ULL(4) + +#define GEN8_PTE_SHIFT 12 +#define GEN8_PAGE_SIZE (1 << GEN8_PTE_SHIFT) +#define GEN8_PTE_MASK (GEN8_PAGE_SIZE - 1) +#define GEN8_PDE_SHIFT (GEN8_PTE_SHIFT - 3) +#define GEN8_PDES (1 << GEN8_PDE_SHIFT) +#define GEN8_PDE_MASK (GEN8_PDES - 1) + +#define GEN8_64K_PTE_SHIFT 16 +#define GEN8_64K_PAGE_SIZE (1 << GEN8_64K_PTE_SHIFT) +#define GEN8_64K_PTE_MASK (GEN8_64K_PAGE_SIZE - 1) +#define GEN8_64K_PDE_MASK (GEN8_PDE_MASK >> 4) + +#define GEN8_PDE_PS_2M BIT_ULL(7) +#define GEN8_PDPE_PS_1G BIT_ULL(7) +#define GEN8_PDE_IPS_64K BIT_ULL(11) + +#define GEN12_GGTT_PTE_LM BIT_ULL(1) +#define GEN12_USM_PPGTT_PTE_AE BIT_ULL(10) +#define GEN12_PPGTT_PTE_LM BIT_ULL(11) +#define GEN12_PDE_64K BIT_ULL(6) +#define GEN12_PTE_PS64 BIT_ULL(8) + +#define GEN8_PAGE_PRESENT BIT_ULL(0) +#define GEN8_PAGE_RW BIT_ULL(1) + +#define PTE_READ_ONLY BIT(0) + +#define XE_PL_SYSTEM TTM_PL_SYSTEM +#define XE_PL_TT TTM_PL_TT +#define XE_PL_VRAM0 TTM_PL_VRAM +#define XE_PL_VRAM1 (XE_PL_VRAM0 + 1) + +#define XE_BO_PROPS_INVALID (-1) + +struct sg_table; + +struct xe_bo *xe_bo_alloc(void); +void xe_bo_free(struct xe_bo *bo); + +struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, + struct xe_gt *gt, struct dma_resv *resv, + size_t size, enum ttm_bo_type type, + u32 flags); +struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags); +struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags); +struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, size_t size, + enum ttm_bo_type type, u32 flags); +struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_gt *gt, + const void *data, size_t size, + enum ttm_bo_type type, u32 flags); + +int xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, + u32 bo_flags); + +static inline struct xe_bo *ttm_to_xe_bo(const struct ttm_buffer_object *bo) +{ + return container_of(bo, struct xe_bo, ttm); +} + +static inline struct xe_bo *gem_to_xe_bo(const struct drm_gem_object *obj) +{ + return container_of(obj, struct xe_bo, ttm.base); +} + +#define xe_bo_device(bo) ttm_to_xe_device((bo)->ttm.bdev) + +static inline struct xe_bo *xe_bo_get(struct xe_bo *bo) +{ + if (bo) + drm_gem_object_get(&bo->ttm.base); + + return bo; +} + +static inline void xe_bo_put(struct xe_bo *bo) +{ + if (bo) + drm_gem_object_put(&bo->ttm.base); +} + +static inline void xe_bo_assert_held(struct xe_bo *bo) +{ + if (bo) + dma_resv_assert_held((bo)->ttm.base.resv); +} + +int xe_bo_lock(struct xe_bo *bo, struct ww_acquire_ctx *ww, + int num_resv, bool intr); + +void xe_bo_unlock(struct xe_bo *bo, struct ww_acquire_ctx *ww); + +static inline void xe_bo_unlock_vm_held(struct xe_bo *bo) +{ + if (bo) { + XE_BUG_ON(bo->vm && bo->ttm.base.resv != &bo->vm->resv); + if (bo->vm) + xe_vm_assert_held(bo->vm); + else + dma_resv_unlock(bo->ttm.base.resv); + } +} + +static inline void xe_bo_lock_no_vm(struct xe_bo *bo, + struct ww_acquire_ctx *ctx) +{ + if (bo) { + XE_BUG_ON(bo->vm || (bo->ttm.type != ttm_bo_type_sg && + bo->ttm.base.resv != &bo->ttm.base._resv)); + dma_resv_lock(bo->ttm.base.resv, ctx); + } +} + +static inline void xe_bo_unlock_no_vm(struct xe_bo *bo) +{ + if (bo) { + XE_BUG_ON(bo->vm || (bo->ttm.type != ttm_bo_type_sg && + bo->ttm.base.resv != &bo->ttm.base._resv)); + dma_resv_unlock(bo->ttm.base.resv); + } +} + +int xe_bo_pin_external(struct xe_bo *bo); +int xe_bo_pin(struct xe_bo *bo); +void xe_bo_unpin_external(struct xe_bo *bo); +void xe_bo_unpin(struct xe_bo *bo); +int xe_bo_validate(struct xe_bo *bo, struct xe_vm *vm, bool allow_res_evict); + +static inline bool xe_bo_is_pinned(struct xe_bo *bo) +{ + return bo->ttm.pin_count; +} + +static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo) +{ + if (likely(bo)) { + xe_bo_lock_no_vm(bo, NULL); + xe_bo_unpin(bo); + xe_bo_unlock_no_vm(bo); + + xe_bo_put(bo); + } +} + +bool xe_bo_is_xe_bo(struct ttm_buffer_object *bo); +dma_addr_t xe_bo_addr(struct xe_bo *bo, u64 offset, + size_t page_size, bool *is_lmem); + +static inline dma_addr_t +xe_bo_main_addr(struct xe_bo *bo, size_t page_size) +{ + bool is_lmem; + + return xe_bo_addr(bo, 0, page_size, &is_lmem); +} + +static inline u32 +xe_bo_ggtt_addr(struct xe_bo *bo) +{ + XE_BUG_ON(bo->ggtt_node.size > bo->size); + XE_BUG_ON(bo->ggtt_node.start + bo->ggtt_node.size > (1ull << 32)); + return bo->ggtt_node.start; +} + +int xe_bo_vmap(struct xe_bo *bo); +void xe_bo_vunmap(struct xe_bo *bo); + +bool mem_type_is_vram(u32 mem_type); +bool xe_bo_is_vram(struct xe_bo *bo); + +bool xe_bo_can_migrate(struct xe_bo *bo, u32 mem_type); + +int xe_bo_migrate(struct xe_bo *bo, u32 mem_type); +int xe_bo_evict(struct xe_bo *bo, bool force_alloc); + +extern struct ttm_device_funcs xe_ttm_funcs; + +int xe_gem_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_bo_dumb_create(struct drm_file *file_priv, + struct drm_device *dev, + struct drm_mode_create_dumb *args); + +bool xe_bo_needs_ccs_pages(struct xe_bo *bo); + +static inline size_t xe_bo_ccs_pages_start(struct xe_bo *bo) +{ + return PAGE_ALIGN(bo->ttm.base.size); +} + +void __xe_bo_release_dummy(struct kref *kref); + +/** + * xe_bo_put_deferred() - Put a buffer object with delayed final freeing + * @bo: The bo to put. + * @deferred: List to which to add the buffer object if we cannot put, or + * NULL if the function is to put unconditionally. + * + * Since the final freeing of an object includes both sleeping and (!) + * memory allocation in the dma_resv individualization, it's not ok + * to put an object from atomic context nor from within a held lock + * tainted by reclaim. In such situations we want to defer the final + * freeing until we've exited the restricting context, or in the worst + * case to a workqueue. + * This function either puts the object if possible without the refcount + * reaching zero, or adds it to the @deferred list if that was not possible. + * The caller needs to follow up with a call to xe_bo_put_commit() to actually + * put the bo iff this function returns true. It's safe to always + * follow up with a call to xe_bo_put_commit(). + * TODO: It's TTM that is the villain here. Perhaps TTM should add an + * interface like this. + * + * Return: true if @bo was the first object put on the @freed list, + * false otherwise. + */ +static inline bool +xe_bo_put_deferred(struct xe_bo *bo, struct llist_head *deferred) +{ + if (!deferred) { + xe_bo_put(bo); + return false; + } + + if (!kref_put(&bo->ttm.base.refcount, __xe_bo_release_dummy)) + return false; + + return llist_add(&bo->freed, deferred); +} + +void xe_bo_put_commit(struct llist_head *deferred); + +struct sg_table *xe_bo_get_sg(struct xe_bo *bo); + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +/** + * xe_bo_is_mem_type - Whether the bo currently resides in the given + * TTM memory type + * @bo: The bo to check. + * @mem_type: The TTM memory type. + * + * Return: true iff the bo resides in @mem_type, false otherwise. + */ +static inline bool xe_bo_is_mem_type(struct xe_bo *bo, u32 mem_type) +{ + xe_bo_assert_held(bo); + return bo->ttm.resource->mem_type == mem_type; +} +#endif +#endif diff --git a/drivers/gpu/drm/xe/xe_bo_doc.h b/drivers/gpu/drm/xe/xe_bo_doc.h new file mode 100644 index 000000000000..f57d440cc95a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo_doc.h @@ -0,0 +1,179 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BO_DOC_H_ +#define _XE_BO_DOC_H_ + +/** + * DOC: Buffer Objects (BO) + * + * BO management + * ============= + * + * TTM manages (placement, eviction, etc...) all BOs in XE. + * + * BO creation + * =========== + * + * Create a chunk of memory which can be used by the GPU. Placement rules + * (sysmem or vram region) passed in upon creation. TTM handles placement of BO + * and can trigger eviction of other BOs to make space for the new BO. + * + * Kernel BOs + * ---------- + * + * A kernel BO is created as part of driver load (e.g. uC firmware images, GuC + * ADS, etc...) or a BO created as part of a user operation which requires + * a kernel BO (e.g. engine state, memory for page tables, etc...). These BOs + * are typically mapped in the GGTT (any kernel BOs aside memory for page tables + * are in the GGTT), are pinned (can't move or be evicted at runtime), have a + * vmap (XE can access the memory via xe_map layer) and have contiguous physical + * memory. + * + * More details of why kernel BOs are pinned and contiguous below. + * + * User BOs + * -------- + * + * A user BO is created via the DRM_IOCTL_XE_GEM_CREATE IOCTL. Once it is + * created the BO can be mmap'd (via DRM_IOCTL_XE_GEM_MMAP_OFFSET) for user + * access and it can be bound for GPU access (via DRM_IOCTL_XE_VM_BIND). All + * user BOs are evictable and user BOs are never pinned by XE. The allocation of + * the backing store can be defered from creation time until first use which is + * either mmap, bind, or pagefault. + * + * Private BOs + * ~~~~~~~~~~~ + * + * A private BO is a user BO created with a valid VM argument passed into the + * create IOCTL. If a BO is private it cannot be exported via prime FD and + * mappings can only be created for the BO within the VM it is tied to. Lastly, + * the BO dma-resv slots / lock point to the VM's dma-resv slots / lock (all + * private BOs to a VM share common dma-resv slots / lock). + * + * External BOs + * ~~~~~~~~~~~~ + * + * An external BO is a user BO created with a NULL VM argument passed into the + * create IOCTL. An external BO can be shared with different UMDs / devices via + * prime FD and the BO can be mapped into multiple VMs. An external BO has its + * own unique dma-resv slots / lock. An external BO will be in an array of all + * VMs which has a mapping of the BO. This allows VMs to lookup and lock all + * external BOs mapped in the VM as needed. + * + * BO placement + * ~~~~~~~~~~~~ + * + * When a user BO is created, a mask of valid placements is passed indicating + * which memory regions are considered valid. + * + * The memory region information is available via query uAPI (TODO: add link). + * + * BO validation + * ============= + * + * BO validation (ttm_bo_validate) refers to ensuring a BO has a valid + * placement. If a BO was swapped to temporary storage, a validation call will + * trigger a move back to a valid (location where GPU can access BO) placement. + * Validation of a BO may evict other BOs to make room for the BO being + * validated. + * + * BO eviction / moving + * ==================== + * + * All eviction (or in other words, moving a BO from one memory location to + * another) is routed through TTM with a callback into XE. + * + * Runtime eviction + * ---------------- + * + * Runtime evictions refers to during normal operations where TTM decides it + * needs to move a BO. Typically this is because TTM needs to make room for + * another BO and the evicted BO is first BO on LRU list that is not locked. + * + * An example of this is a new BO which can only be placed in VRAM but there is + * not space in VRAM. There could be multiple BOs which have sysmem and VRAM + * placement rules which currently reside in VRAM, TTM trigger a will move of + * one (or multiple) of these BO(s) until there is room in VRAM to place the new + * BO. The evicted BO(s) are valid but still need new bindings before the BO + * used again (exec or compute mode rebind worker). + * + * Another example would be, TTM can't find a BO to evict which has another + * valid placement. In this case TTM will evict one (or multiple) unlocked BO(s) + * to a temporary unreachable (invalid) placement. The evicted BO(s) are invalid + * and before next use need to be moved to a valid placement and rebound. + * + * In both cases, moves of these BOs are scheduled behind the fences in the BO's + * dma-resv slots. + * + * WW locking tries to ensures if 2 VMs use 51% of the memory forward progress + * is made on both VMs. + * + * Runtime eviction uses per a GT migration engine (TODO: link to migration + * engine doc) to do a GPU memcpy from one location to another. + * + * Rebinds after runtime eviction + * ------------------------------ + * + * When BOs are moved, every mapping (VMA) of the BO needs to rebound before + * the BO is used again. Every VMA is added to an evicted list of its VM when + * the BO is moved. This is safe because of the VM locking structure (TODO: link + * to VM locking doc). On the next use of a VM (exec or compute mode rebind + * worker) the evicted VMA list is checked and rebinds are triggered. In the + * case of faulting VM, the rebind is done in the page fault handler. + * + * Suspend / resume eviction of VRAM + * --------------------------------- + * + * During device suspend / resume VRAM may lose power which means the contents + * of VRAM's memory is blown away. Thus BOs present in VRAM at the time of + * suspend must be moved to sysmem in order for their contents to be saved. + * + * A simple TTM call (ttm_resource_manager_evict_all) can move all non-pinned + * (user) BOs to sysmem. External BOs that are pinned need to be manually + * evicted with a simple loop + xe_bo_evict call. It gets a little trickier + * with kernel BOs. + * + * Some kernel BOs are used by the GT migration engine to do moves, thus we + * can't move all of the BOs via the GT migration engine. For simplity, use a + * TTM memcpy (CPU) to move any kernel (pinned) BO on either suspend or resume. + * + * Some kernel BOs need to be restored to the exact same physical location. TTM + * makes this rather easy but the caveat is the memory must be contiguous. Again + * for simplity, we enforce that all kernel (pinned) BOs are contiguous and + * restored to the same physical location. + * + * Pinned external BOs in VRAM are restored on resume via the GPU. + * + * Rebinds after suspend / resume + * ------------------------------ + * + * Most kernel BOs have GGTT mappings which must be restored during the resume + * process. All user BOs are rebound after validation on their next use. + * + * Future work + * =========== + * + * Trim the list of BOs which is saved / restored via TTM memcpy on suspend / + * resume. All we really need to save / restore via TTM memcpy is the memory + * required for the GuC to load and the memory for the GT migrate engine to + * operate. + * + * Do not require kernel BOs to be contiguous in physical memory / restored to + * the same physical address on resume. In all likelihood the only memory that + * needs to be restored to the same physical address is memory used for page + * tables. All of that memory is allocated 1 page at time so the contiguous + * requirement isn't needed. Some work on the vmap code would need to be done if + * kernel BOs are not contiguous too. + * + * Make some kernel BO evictable rather than pinned. An example of this would be + * engine state, in all likelihood if the dma-slots of these BOs where properly + * used rather than pinning we could safely evict + rebind these BOs as needed. + * + * Some kernel BOs do not need to be restored on resume (e.g. GuC ADS as that is + * repopulated on resume), add flag to mark such objects as no save / restore. + */ + +#endif diff --git a/drivers/gpu/drm/xe/xe_bo_evict.c b/drivers/gpu/drm/xe/xe_bo_evict.c new file mode 100644 index 000000000000..7046dc203138 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo_evict.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_bo.h" +#include "xe_bo_evict.h" +#include "xe_device.h" +#include "xe_ggtt.h" +#include "xe_gt.h" + +/** + * xe_bo_evict_all - evict all BOs from VRAM + * + * @xe: xe device + * + * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next + * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU. + * All eviction magic done via TTM calls. + * + * Evict == move VRAM BOs to temporary (typically system) memory. + * + * This function should be called before the device goes into a suspend state + * where the VRAM loses power. + */ +int xe_bo_evict_all(struct xe_device *xe) +{ + struct ttm_device *bdev = &xe->ttm; + struct ww_acquire_ctx ww; + struct xe_bo *bo; + struct xe_gt *gt; + struct list_head still_in_list; + u32 mem_type; + u8 id; + int ret; + + if (!IS_DGFX(xe)) + return 0; + + /* User memory */ + for (mem_type = XE_PL_VRAM0; mem_type <= XE_PL_VRAM1; ++mem_type) { + struct ttm_resource_manager *man = + ttm_manager_type(bdev, mem_type); + + if (man) { + ret = ttm_resource_manager_evict_all(bdev, man); + if (ret) + return ret; + } + } + + /* Pinned user memory in VRAM */ + INIT_LIST_HEAD(&still_in_list); + spin_lock(&xe->pinned.lock); + for (;;) { + bo = list_first_entry_or_null(&xe->pinned.external_vram, + typeof(*bo), pinned_link); + if (!bo) + break; + xe_bo_get(bo); + list_move_tail(&bo->pinned_link, &still_in_list); + spin_unlock(&xe->pinned.lock); + + xe_bo_lock(bo, &ww, 0, false); + ret = xe_bo_evict(bo, true); + xe_bo_unlock(bo, &ww); + xe_bo_put(bo); + if (ret) { + spin_lock(&xe->pinned.lock); + list_splice_tail(&still_in_list, + &xe->pinned.external_vram); + spin_unlock(&xe->pinned.lock); + return ret; + } + + spin_lock(&xe->pinned.lock); + } + list_splice_tail(&still_in_list, &xe->pinned.external_vram); + spin_unlock(&xe->pinned.lock); + + /* + * Wait for all user BO to be evicted as those evictions depend on the + * memory moved below. + */ + for_each_gt(gt, xe, id) + xe_gt_migrate_wait(gt); + + spin_lock(&xe->pinned.lock); + for (;;) { + bo = list_first_entry_or_null(&xe->pinned.kernel_bo_present, + typeof(*bo), pinned_link); + if (!bo) + break; + xe_bo_get(bo); + list_move_tail(&bo->pinned_link, &xe->pinned.evicted); + spin_unlock(&xe->pinned.lock); + + xe_bo_lock(bo, &ww, 0, false); + ret = xe_bo_evict(bo, true); + xe_bo_unlock(bo, &ww); + xe_bo_put(bo); + if (ret) + return ret; + + spin_lock(&xe->pinned.lock); + } + spin_unlock(&xe->pinned.lock); + + return 0; +} + +/** + * xe_bo_restore_kernel - restore kernel BOs to VRAM + * + * @xe: xe device + * + * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All + * moves done via TTM calls. + * + * This function should be called early, before trying to init the GT, on device + * resume. + */ +int xe_bo_restore_kernel(struct xe_device *xe) +{ + struct ww_acquire_ctx ww; + struct xe_bo *bo; + int ret; + + if (!IS_DGFX(xe)) + return 0; + + spin_lock(&xe->pinned.lock); + for (;;) { + bo = list_first_entry_or_null(&xe->pinned.evicted, + typeof(*bo), pinned_link); + if (!bo) + break; + xe_bo_get(bo); + list_move_tail(&bo->pinned_link, &xe->pinned.kernel_bo_present); + spin_unlock(&xe->pinned.lock); + + xe_bo_lock(bo, &ww, 0, false); + ret = xe_bo_validate(bo, NULL, false); + xe_bo_unlock(bo, &ww); + if (ret) { + xe_bo_put(bo); + return ret; + } + + if (bo->flags & XE_BO_CREATE_GGTT_BIT) + xe_ggtt_map_bo(bo->gt->mem.ggtt, bo); + + /* + * We expect validate to trigger a move VRAM and our move code + * should setup the iosys map. + */ + XE_BUG_ON(iosys_map_is_null(&bo->vmap)); + XE_BUG_ON(!xe_bo_is_vram(bo)); + + xe_bo_put(bo); + + spin_lock(&xe->pinned.lock); + } + spin_unlock(&xe->pinned.lock); + + return 0; +} + +/** + * xe_bo_restore_user - restore pinned user BOs to VRAM + * + * @xe: xe device + * + * Move pinned user BOs from temporary (typically system) memory to VRAM via + * CPU. All moves done via TTM calls. + * + * This function should be called late, after GT init, on device resume. + */ +int xe_bo_restore_user(struct xe_device *xe) +{ + struct ww_acquire_ctx ww; + struct xe_bo *bo; + struct xe_gt *gt; + struct list_head still_in_list; + u8 id; + int ret; + + if (!IS_DGFX(xe)) + return 0; + + /* Pinned user memory in VRAM should be validated on resume */ + INIT_LIST_HEAD(&still_in_list); + spin_lock(&xe->pinned.lock); + for (;;) { + bo = list_first_entry_or_null(&xe->pinned.external_vram, + typeof(*bo), pinned_link); + if (!bo) + break; + list_move_tail(&bo->pinned_link, &still_in_list); + xe_bo_get(bo); + spin_unlock(&xe->pinned.lock); + + xe_bo_lock(bo, &ww, 0, false); + ret = xe_bo_validate(bo, NULL, false); + xe_bo_unlock(bo, &ww); + xe_bo_put(bo); + if (ret) { + spin_lock(&xe->pinned.lock); + list_splice_tail(&still_in_list, + &xe->pinned.external_vram); + spin_unlock(&xe->pinned.lock); + return ret; + } + + spin_lock(&xe->pinned.lock); + } + list_splice_tail(&still_in_list, &xe->pinned.external_vram); + spin_unlock(&xe->pinned.lock); + + /* Wait for validate to complete */ + for_each_gt(gt, xe, id) + xe_gt_migrate_wait(gt); + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_bo_evict.h b/drivers/gpu/drm/xe/xe_bo_evict.h new file mode 100644 index 000000000000..746894798852 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo_evict.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BO_EVICT_H_ +#define _XE_BO_EVICT_H_ + +struct xe_device; + +int xe_bo_evict_all(struct xe_device *xe); +int xe_bo_restore_kernel(struct xe_device *xe); +int xe_bo_restore_user(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h new file mode 100644 index 000000000000..06de3330211d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_BO_TYPES_H_ +#define _XE_BO_TYPES_H_ + +#include + +#include +#include +#include +#include +#include + +struct xe_device; +struct xe_vm; + +#define XE_BO_MAX_PLACEMENTS 3 + +/** @xe_bo: XE buffer object */ +struct xe_bo { + /** @ttm: TTM base buffer object */ + struct ttm_buffer_object ttm; + /** @size: Size of this buffer object */ + size_t size; + /** @flags: flags for this buffer object */ + u32 flags; + /** @vm: VM this BO is attached to, for extobj this will be NULL */ + struct xe_vm *vm; + /** @gt: GT this BO is attached to (kernel BO only) */ + struct xe_gt *gt; + /** @vmas: List of VMAs for this BO */ + struct list_head vmas; + /** @placements: valid placements for this BO */ + struct ttm_place placements[XE_BO_MAX_PLACEMENTS]; + /** @placement: current placement for this BO */ + struct ttm_placement placement; + /** @ggtt_node: GGTT node if this BO is mapped in the GGTT */ + struct drm_mm_node ggtt_node; + /** @vmap: iosys map of this buffer */ + struct iosys_map vmap; + /** @ttm_kmap: TTM bo kmap object for internal use only. Keep off. */ + struct ttm_bo_kmap_obj kmap; + /** @pinned_link: link to present / evicted list of pinned BO */ + struct list_head pinned_link; + /** @props: BO user controlled properties */ + struct { + /** @preferred_mem: preferred memory class for this BO */ + s16 preferred_mem_class; + /** @prefered_gt: preferred GT for this BO */ + s16 preferred_gt; + /** @preferred_mem_type: preferred memory type */ + s32 preferred_mem_type; + /** + * @cpu_atomic: the CPU expects to do atomics operations to + * this BO + */ + bool cpu_atomic; + /** + * @device_atomic: the device expects to do atomics operations + * to this BO + */ + bool device_atomic; + } props; + /** @freed: List node for delayed put. */ + struct llist_node freed; + /** @created: Whether the bo has passed initial creation */ + bool created; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_debugfs.c b/drivers/gpu/drm/xe/xe_debugfs.c new file mode 100644 index 000000000000..84db7b3f501e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_debugfs.c @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_debugfs.h" +#include "xe_gt_debugfs.h" +#include "xe_step.h" + +#ifdef CONFIG_DRM_XE_DEBUG +#include "xe_bo_evict.h" +#include "xe_migrate.h" +#include "xe_vm.h" +#endif + +static struct xe_device *node_to_xe(struct drm_info_node *node) +{ + return to_xe_device(node->minor->dev); +} + +static int info(struct seq_file *m, void *data) +{ + struct xe_device *xe = node_to_xe(m->private); + struct drm_printer p = drm_seq_file_printer(m); + struct xe_gt *gt; + u8 id; + + drm_printf(&p, "graphics_verx100 %d\n", xe->info.graphics_verx100); + drm_printf(&p, "media_verx100 %d\n", xe->info.media_verx100); + drm_printf(&p, "stepping G:%s M:%s D:%s B:%s\n", + xe_step_name(xe->info.step.graphics), + xe_step_name(xe->info.step.media), + xe_step_name(xe->info.step.display), + xe_step_name(xe->info.step.basedie)); + drm_printf(&p, "is_dgfx %s\n", str_yes_no(xe->info.is_dgfx)); + drm_printf(&p, "platform %d\n", xe->info.platform); + drm_printf(&p, "subplatform %d\n", + xe->info.subplatform > XE_SUBPLATFORM_NONE ? xe->info.subplatform : 0); + drm_printf(&p, "devid 0x%x\n", xe->info.devid); + drm_printf(&p, "revid %d\n", xe->info.revid); + drm_printf(&p, "tile_count %d\n", xe->info.tile_count); + drm_printf(&p, "vm_max_level %d\n", xe->info.vm_max_level); + drm_printf(&p, "enable_guc %s\n", str_yes_no(xe->info.enable_guc)); + drm_printf(&p, "supports_usm %s\n", str_yes_no(xe->info.supports_usm)); + drm_printf(&p, "has_flat_ccs %s\n", str_yes_no(xe->info.has_flat_ccs)); + for_each_gt(gt, xe, id) { + drm_printf(&p, "gt%d force wake %d\n", id, + xe_force_wake_ref(gt_to_fw(gt), XE_FW_GT)); + drm_printf(&p, "gt%d engine_mask 0x%llx\n", id, + gt->info.engine_mask); + } + + return 0; +} + +static const struct drm_info_list debugfs_list[] = { + {"info", info, 0}, +}; + +static int forcewake_open(struct inode *inode, struct file *file) +{ + struct xe_device *xe = inode->i_private; + struct xe_gt *gt; + u8 id; + + for_each_gt(gt, xe, id) + XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + + return 0; +} + +static int forcewake_release(struct inode *inode, struct file *file) +{ + struct xe_device *xe = inode->i_private; + struct xe_gt *gt; + u8 id; + + for_each_gt(gt, xe, id) + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + + return 0; +} + +static const struct file_operations forcewake_all_fops = { + .owner = THIS_MODULE, + .open = forcewake_open, + .release = forcewake_release, +}; + +void xe_debugfs_register(struct xe_device *xe) +{ + struct ttm_device *bdev = &xe->ttm; + struct drm_minor *minor = xe->drm.primary; + struct dentry *root = minor->debugfs_root; + struct ttm_resource_manager *man; + struct xe_gt *gt; + u32 mem_type; + u8 id; + + drm_debugfs_create_files(debugfs_list, + ARRAY_SIZE(debugfs_list), + root, minor); + + debugfs_create_file("forcewake_all", 0400, root, xe, + &forcewake_all_fops); + + for (mem_type = XE_PL_VRAM0; mem_type <= XE_PL_VRAM1; ++mem_type) { + man = ttm_manager_type(bdev, mem_type); + + if (man) { + char name[16]; + + sprintf(name, "vram%d_mm", mem_type - XE_PL_VRAM0); + ttm_resource_manager_create_debugfs(man, root, name); + } + } + + man = ttm_manager_type(bdev, XE_PL_TT); + ttm_resource_manager_create_debugfs(man, root, "gtt_mm"); + + for_each_gt(gt, xe, id) + xe_gt_debugfs_register(gt); +} diff --git a/drivers/gpu/drm/xe/xe_debugfs.h b/drivers/gpu/drm/xe/xe_debugfs.h new file mode 100644 index 000000000000..715b8e2e0bd9 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_debugfs.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_DEBUGFS_H_ +#define _XE_DEBUGFS_H_ + +struct xe_device; + +void xe_debugfs_register(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c new file mode 100644 index 000000000000..93dea2b9c464 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_device.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_device.h" + +#include +#include +#include +#include +#include +#include + +#include "xe_bo.h" +#include "xe_debugfs.h" +#include "xe_dma_buf.h" +#include "xe_drv.h" +#include "xe_engine.h" +#include "xe_exec.h" +#include "xe_gt.h" +#include "xe_irq.h" +#include "xe_module.h" +#include "xe_mmio.h" +#include "xe_pcode.h" +#include "xe_pm.h" +#include "xe_query.h" +#include "xe_vm.h" +#include "xe_vm_madvise.h" +#include "xe_wait_user_fence.h" + +static int xe_file_open(struct drm_device *dev, struct drm_file *file) +{ + struct xe_file *xef; + + xef = kzalloc(sizeof(*xef), GFP_KERNEL); + if (!xef) + return -ENOMEM; + + xef->drm = file; + + mutex_init(&xef->vm.lock); + xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); + + mutex_init(&xef->engine.lock); + xa_init_flags(&xef->engine.xa, XA_FLAGS_ALLOC1); + + file->driver_priv = xef; + return 0; +} + +static void device_kill_persitent_engines(struct xe_device *xe, + struct xe_file *xef); + +static void xe_file_close(struct drm_device *dev, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = file->driver_priv; + struct xe_vm *vm; + struct xe_engine *e; + unsigned long idx; + + mutex_lock(&xef->engine.lock); + xa_for_each(&xef->engine.xa, idx, e) { + xe_engine_kill(e); + xe_engine_put(e); + } + mutex_unlock(&xef->engine.lock); + mutex_destroy(&xef->engine.lock); + device_kill_persitent_engines(xe, xef); + + mutex_lock(&xef->vm.lock); + xa_for_each(&xef->vm.xa, idx, vm) + xe_vm_close_and_put(vm); + mutex_unlock(&xef->vm.lock); + mutex_destroy(&xef->vm.lock); + + kfree(xef); +} + +static const struct drm_ioctl_desc xe_ioctls[] = { + DRM_IOCTL_DEF_DRV(XE_DEVICE_QUERY, xe_query_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_GEM_CREATE, xe_gem_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_GEM_MMAP_OFFSET, xe_gem_mmap_offset_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_ENGINE_CREATE, xe_engine_create_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_ENGINE_DESTROY, xe_engine_destroy_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_MMIO, xe_mmio_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_ENGINE_SET_PROPERTY, xe_engine_set_property_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, + DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_VM_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), +}; + +static const struct file_operations xe_driver_fops = { + .owner = THIS_MODULE, + .open = drm_open, + .release = drm_release_noglobal, + .unlocked_ioctl = drm_ioctl, + .mmap = drm_gem_mmap, + .poll = drm_poll, + .read = drm_read, +// .compat_ioctl = i915_ioc32_compat_ioctl, + .llseek = noop_llseek, +}; + +static void xe_driver_release(struct drm_device *dev) +{ + struct xe_device *xe = to_xe_device(dev); + + pci_set_drvdata(to_pci_dev(xe->drm.dev), NULL); +} + +static struct drm_driver driver = { + /* Don't use MTRRs here; the Xserver or userspace app should + * deal with them for Intel hardware. + */ + .driver_features = + DRIVER_GEM | + DRIVER_RENDER | DRIVER_SYNCOBJ | + DRIVER_SYNCOBJ_TIMELINE, + .open = xe_file_open, + .postclose = xe_file_close, + + .gem_prime_import = xe_gem_prime_import, + + .dumb_create = xe_bo_dumb_create, + .dumb_map_offset = drm_gem_ttm_dumb_map_offset, + .release = &xe_driver_release, + + .ioctls = xe_ioctls, + .num_ioctls = ARRAY_SIZE(xe_ioctls), + .fops = &xe_driver_fops, + .name = DRIVER_NAME, + .desc = DRIVER_DESC, + .date = DRIVER_DATE, + .major = DRIVER_MAJOR, + .minor = DRIVER_MINOR, + .patchlevel = DRIVER_PATCHLEVEL, +}; + +static void xe_device_destroy(struct drm_device *dev, void *dummy) +{ + struct xe_device *xe = to_xe_device(dev); + + destroy_workqueue(xe->ordered_wq); + mutex_destroy(&xe->persitent_engines.lock); + ttm_device_fini(&xe->ttm); +} + +struct xe_device *xe_device_create(struct pci_dev *pdev, + const struct pci_device_id *ent) +{ + struct xe_device *xe; + int err; + + err = drm_aperture_remove_conflicting_pci_framebuffers(pdev, &driver); + if (err) + return ERR_PTR(err); + + xe = devm_drm_dev_alloc(&pdev->dev, &driver, struct xe_device, drm); + if (IS_ERR(xe)) + return xe; + + err = ttm_device_init(&xe->ttm, &xe_ttm_funcs, xe->drm.dev, + xe->drm.anon_inode->i_mapping, + xe->drm.vma_offset_manager, false, false); + if (WARN_ON(err)) + goto err_put; + + xe->info.devid = pdev->device; + xe->info.revid = pdev->revision; + xe->info.enable_guc = enable_guc; + + spin_lock_init(&xe->irq.lock); + + init_waitqueue_head(&xe->ufence_wq); + + mutex_init(&xe->usm.lock); + xa_init_flags(&xe->usm.asid_to_vm, XA_FLAGS_ALLOC1); + + mutex_init(&xe->persitent_engines.lock); + INIT_LIST_HEAD(&xe->persitent_engines.list); + + spin_lock_init(&xe->pinned.lock); + INIT_LIST_HEAD(&xe->pinned.kernel_bo_present); + INIT_LIST_HEAD(&xe->pinned.external_vram); + INIT_LIST_HEAD(&xe->pinned.evicted); + + xe->ordered_wq = alloc_ordered_workqueue("xe-ordered-wq", 0); + + mutex_init(&xe->sb_lock); + xe->enabled_irq_mask = ~0; + + err = drmm_add_action_or_reset(&xe->drm, xe_device_destroy, NULL); + if (err) + goto err_put; + + mutex_init(&xe->mem_access.lock); + return xe; + +err_put: + drm_dev_put(&xe->drm); + + return ERR_PTR(err); +} + +int xe_device_probe(struct xe_device *xe) +{ + struct xe_gt *gt; + int err; + u8 id; + + xe->info.mem_region_mask = 1; + + for_each_gt(gt, xe, id) { + err = xe_gt_alloc(xe, gt); + if (err) + return err; + } + + err = xe_mmio_init(xe); + if (err) + return err; + + for_each_gt(gt, xe, id) { + err = xe_pcode_probe(gt); + if (err) + return err; + } + + err = xe_irq_install(xe); + if (err) + return err; + + for_each_gt(gt, xe, id) { + err = xe_gt_init_early(gt); + if (err) + goto err_irq_shutdown; + } + + err = xe_mmio_probe_vram(xe); + if (err) + goto err_irq_shutdown; + + for_each_gt(gt, xe, id) { + err = xe_gt_init_noalloc(gt); + if (err) + goto err_irq_shutdown; + } + + for_each_gt(gt, xe, id) { + err = xe_gt_init(gt); + if (err) + goto err_irq_shutdown; + } + + err = drm_dev_register(&xe->drm, 0); + if (err) + goto err_irq_shutdown; + + xe_debugfs_register(xe); + + return 0; + +err_irq_shutdown: + xe_irq_shutdown(xe); + return err; +} + +void xe_device_remove(struct xe_device *xe) +{ + xe_irq_shutdown(xe); +} + +void xe_device_shutdown(struct xe_device *xe) +{ +} + +void xe_device_add_persitent_engines(struct xe_device *xe, struct xe_engine *e) +{ + mutex_lock(&xe->persitent_engines.lock); + list_add_tail(&e->persitent.link, &xe->persitent_engines.list); + mutex_unlock(&xe->persitent_engines.lock); +} + +void xe_device_remove_persitent_engines(struct xe_device *xe, + struct xe_engine *e) +{ + mutex_lock(&xe->persitent_engines.lock); + if (!list_empty(&e->persitent.link)) + list_del(&e->persitent.link); + mutex_unlock(&xe->persitent_engines.lock); +} + +static void device_kill_persitent_engines(struct xe_device *xe, + struct xe_file *xef) +{ + struct xe_engine *e, *next; + + mutex_lock(&xe->persitent_engines.lock); + list_for_each_entry_safe(e, next, &xe->persitent_engines.list, + persitent.link) + if (e->persitent.xef == xef) { + xe_engine_kill(e); + list_del_init(&e->persitent.link); + } + mutex_unlock(&xe->persitent_engines.lock); +} + +#define SOFTWARE_FLAGS_SPR33 _MMIO(0x4F084) + +void xe_device_wmb(struct xe_device *xe) +{ + struct xe_gt *gt = xe_device_get_gt(xe, 0); + + wmb(); + if (IS_DGFX(xe)) + xe_mmio_write32(gt, SOFTWARE_FLAGS_SPR33.reg, 0); +} + +u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size) +{ + return xe_device_has_flat_ccs(xe) ? + DIV_ROUND_UP(size, NUM_BYTES_PER_CCS_BYTE) : 0; +} + +void xe_device_mem_access_get(struct xe_device *xe) +{ + bool resumed = xe_pm_runtime_resume_if_suspended(xe); + + mutex_lock(&xe->mem_access.lock); + if (xe->mem_access.ref++ == 0) + xe->mem_access.hold_rpm = xe_pm_runtime_get_if_active(xe); + mutex_unlock(&xe->mem_access.lock); + + /* The usage counter increased if device was immediately resumed */ + if (resumed) + xe_pm_runtime_put(xe); + + XE_WARN_ON(xe->mem_access.ref == U32_MAX); +} + +void xe_device_mem_access_put(struct xe_device *xe) +{ + mutex_lock(&xe->mem_access.lock); + if (--xe->mem_access.ref == 0 && xe->mem_access.hold_rpm) + xe_pm_runtime_put(xe); + mutex_unlock(&xe->mem_access.lock); + + XE_WARN_ON(xe->mem_access.ref < 0); +} diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h new file mode 100644 index 000000000000..88d55671b068 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_device.h @@ -0,0 +1,126 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_DEVICE_H_ +#define _XE_DEVICE_H_ + +struct xe_engine; +struct xe_file; + +#include + +#include "xe_device_types.h" +#include "xe_macros.h" +#include "xe_force_wake.h" + +#include "gt/intel_gpu_commands.h" + +static inline struct xe_device *to_xe_device(const struct drm_device *dev) +{ + return container_of(dev, struct xe_device, drm); +} + +static inline struct xe_device *pdev_to_xe_device(struct pci_dev *pdev) +{ + return pci_get_drvdata(pdev); +} + +static inline struct xe_device *ttm_to_xe_device(struct ttm_device *ttm) +{ + return container_of(ttm, struct xe_device, ttm); +} + +struct xe_device *xe_device_create(struct pci_dev *pdev, + const struct pci_device_id *ent); +int xe_device_probe(struct xe_device *xe); +void xe_device_remove(struct xe_device *xe); +void xe_device_shutdown(struct xe_device *xe); + +void xe_device_add_persitent_engines(struct xe_device *xe, struct xe_engine *e); +void xe_device_remove_persitent_engines(struct xe_device *xe, + struct xe_engine *e); + +void xe_device_wmb(struct xe_device *xe); + +static inline struct xe_file *to_xe_file(const struct drm_file *file) +{ + return file->driver_priv; +} + +static inline struct xe_gt *xe_device_get_gt(struct xe_device *xe, u8 gt_id) +{ + struct xe_gt *gt; + + XE_BUG_ON(gt_id > XE_MAX_GT); + gt = xe->gt + gt_id; + XE_BUG_ON(gt->info.id != gt_id); + XE_BUG_ON(gt->info.type == XE_GT_TYPE_UNINITIALIZED); + + return gt; +} + +/* + * FIXME: Placeholder until multi-gt lands. Once that lands, kill this function. + */ +static inline struct xe_gt *to_gt(struct xe_device *xe) +{ + return xe->gt; +} + +static inline bool xe_device_guc_submission_enabled(struct xe_device *xe) +{ + return xe->info.enable_guc; +} + +static inline void xe_device_guc_submission_disable(struct xe_device *xe) +{ + xe->info.enable_guc = false; +} + +#define for_each_gt(gt__, xe__, id__) \ + for ((id__) = 0; (id__) < (xe__)->info.tile_count; (id__++)) \ + for_each_if ((gt__) = xe_device_get_gt((xe__), (id__))) + +static inline struct xe_force_wake * gt_to_fw(struct xe_gt *gt) +{ + return >->mmio.fw; +} + +void xe_device_mem_access_get(struct xe_device *xe); +void xe_device_mem_access_put(struct xe_device *xe); + +static inline void xe_device_assert_mem_access(struct xe_device *xe) +{ + XE_WARN_ON(!xe->mem_access.ref); +} + +static inline bool xe_device_mem_access_ongoing(struct xe_device *xe) +{ + bool ret; + + mutex_lock(&xe->mem_access.lock); + ret = xe->mem_access.ref; + mutex_unlock(&xe->mem_access.lock); + + return ret; +} + +static inline bool xe_device_in_fault_mode(struct xe_device *xe) +{ + return xe->usm.num_vm_in_fault_mode != 0; +} + +static inline bool xe_device_in_non_fault_mode(struct xe_device *xe) +{ + return xe->usm.num_vm_in_non_fault_mode != 0; +} + +static inline bool xe_device_has_flat_ccs(struct xe_device *xe) +{ + return xe->info.has_flat_ccs; +} + +u32 xe_device_ccs_bytes(struct xe_device *xe, u64 size); +#endif diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h new file mode 100644 index 000000000000..d62ee85bfcbe --- /dev/null +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -0,0 +1,214 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_DEVICE_TYPES_H_ +#define _XE_DEVICE_TYPES_H_ + +#include + +#include +#include +#include + +#include "xe_gt_types.h" +#include "xe_platform_types.h" +#include "xe_step_types.h" + +#define XE_BO_INVALID_OFFSET LONG_MAX + +#define GRAPHICS_VER(xe) ((xe)->info.graphics_verx100 / 100) +#define MEDIA_VER(xe) ((xe)->info.media_verx100 / 100) +#define GRAPHICS_VERx100(xe) ((xe)->info.graphics_verx100) +#define MEDIA_VERx100(xe) ((xe)->info.media_verx100) +#define IS_DGFX(xe) ((xe)->info.is_dgfx) + +#define XE_VRAM_FLAGS_NEED64K BIT(0) + +#define XE_GT0 0 +#define XE_GT1 1 +#define XE_MAX_GT (XE_GT1 + 1) + +#define XE_MAX_ASID (BIT(20)) + +#define IS_PLATFORM_STEP(_xe, _platform, min_step, max_step) \ + ((_xe)->info.platform == (_platform) && \ + (_xe)->info.step.graphics >= (min_step) && \ + (_xe)->info.step.graphics < (max_step)) +#define IS_SUBPLATFORM_STEP(_xe, _platform, sub, min_step, max_step) \ + ((_xe)->info.platform == (_platform) && \ + (_xe)->info.subplatform == (sub) && \ + (_xe)->info.step.graphics >= (min_step) && \ + (_xe)->info.step.graphics < (max_step)) + +/** + * struct xe_device - Top level struct of XE device + */ +struct xe_device { + /** @drm: drm device */ + struct drm_device drm; + + /** @info: device info */ + struct intel_device_info { + /** @graphics_verx100: graphics IP version */ + u32 graphics_verx100; + /** @media_verx100: media IP version */ + u32 media_verx100; + /** @mem_region_mask: mask of valid memory regions */ + u32 mem_region_mask; + /** @is_dgfx: is discrete device */ + bool is_dgfx; + /** @platform: XE platform enum */ + enum xe_platform platform; + /** @subplatform: XE subplatform enum */ + enum xe_subplatform subplatform; + /** @devid: device ID */ + u16 devid; + /** @revid: device revision */ + u8 revid; + /** @step: stepping information for each IP */ + struct xe_step_info step; + /** @dma_mask_size: DMA address bits */ + u8 dma_mask_size; + /** @vram_flags: Vram flags */ + u8 vram_flags; + /** @tile_count: Number of tiles */ + u8 tile_count; + /** @vm_max_level: Max VM level */ + u8 vm_max_level; + /** @media_ver: Media version */ + u8 media_ver; + /** @supports_usm: Supports unified shared memory */ + bool supports_usm; + /** @enable_guc: GuC submission enabled */ + bool enable_guc; + /** @has_flat_ccs: Whether flat CCS metadata is used */ + bool has_flat_ccs; + /** @has_4tile: Whether tile-4 tiling is supported */ + bool has_4tile; + } info; + + /** @irq: device interrupt state */ + struct { + /** @lock: lock for processing irq's on this device */ + spinlock_t lock; + + /** @enabled: interrupts enabled on this device */ + bool enabled; + } irq; + + /** @ttm: ttm device */ + struct ttm_device ttm; + + /** @mmio: mmio info for device */ + struct { + /** @size: size of MMIO space for device */ + size_t size; + /** @regs: pointer to MMIO space for device */ + void *regs; + } mmio; + + /** @mem: memory info for device */ + struct { + /** @vram: VRAM info for device */ + struct { + /** @io_start: start address of VRAM */ + resource_size_t io_start; + /** @size: size of VRAM */ + resource_size_t size; + /** @mapping: pointer to VRAM mappable space */ + void *__iomem mapping; + } vram; + } mem; + + /** @usm: unified memory state */ + struct { + /** @asid: convert a ASID to VM */ + struct xarray asid_to_vm; + /** @next_asid: next ASID, used to cyclical alloc asids */ + u32 next_asid; + /** @num_vm_in_fault_mode: number of VM in fault mode */ + u32 num_vm_in_fault_mode; + /** @num_vm_in_non_fault_mode: number of VM in non-fault mode */ + u32 num_vm_in_non_fault_mode; + /** @lock: protects UM state */ + struct mutex lock; + } usm; + + /** @persitent_engines: engines that are closed but still running */ + struct { + /** @lock: protects persitent engines */ + struct mutex lock; + /** @list: list of persitent engines */ + struct list_head list; + } persitent_engines; + + /** @pinned: pinned BO state */ + struct { + /** @lock: protected pinned BO list state */ + spinlock_t lock; + /** @evicted: pinned kernel BO that are present */ + struct list_head kernel_bo_present; + /** @evicted: pinned BO that have been evicted */ + struct list_head evicted; + /** @external_vram: pinned external BO in vram*/ + struct list_head external_vram; + } pinned; + + /** @ufence_wq: user fence wait queue */ + wait_queue_head_t ufence_wq; + + /** @ordered_wq: used to serialize compute mode resume */ + struct workqueue_struct *ordered_wq; + + /** @gt: graphics tile */ + struct xe_gt gt[XE_MAX_GT]; + + /** + * @mem_access: keep track of memory access in the device, possibly + * triggering additional actions when they occur. + */ + struct { + /** @lock: protect the ref count */ + struct mutex lock; + /** @ref: ref count of memory accesses */ + u32 ref; + /** @hold_rpm: need to put rpm ref back at the end */ + bool hold_rpm; + } mem_access; + + /** @d3cold_allowed: Indicates if d3cold is a valid device state */ + bool d3cold_allowed; + + /* For pcode */ + struct mutex sb_lock; + + u32 enabled_irq_mask; +}; + +/** + * struct xe_file - file handle for XE driver + */ +struct xe_file { + /** @drm: base DRM file */ + struct drm_file *drm; + + /** @vm: VM state for file */ + struct { + /** @xe: xarray to store VMs */ + struct xarray xa; + /** @lock: protects file VM state */ + struct mutex lock; + } vm; + + /** @engine: Submission engine state for file */ + struct { + /** @xe: xarray to store engines */ + struct xarray xa; + /** @lock: protects file engine state */ + struct mutex lock; + } engine; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c new file mode 100644 index 000000000000..d09ff25bd940 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include +#include + +#include + +#include +#include + +#include "tests/xe_test.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_dma_buf.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_vm.h" + +MODULE_IMPORT_NS(DMA_BUF); + +static int xe_dma_buf_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->dmabuf->priv; + + if (attach->peer2peer && + pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0) + attach->peer2peer = false; + + if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT)) + return -EOPNOTSUPP; + + xe_device_mem_access_get(to_xe_device(obj->dev)); + return 0; +} + +static void xe_dma_buf_detach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->dmabuf->priv; + + xe_device_mem_access_put(to_xe_device(obj->dev)); +} + +static int xe_dma_buf_pin(struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->dmabuf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + + /* + * Migrate to TT first to increase the chance of non-p2p clients + * can attach. + */ + (void)xe_bo_migrate(bo, XE_PL_TT); + xe_bo_pin_external(bo); + + return 0; +} + +static void xe_dma_buf_unpin(struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->dmabuf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + + xe_bo_unpin_external(bo); +} + +static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach, + enum dma_data_direction dir) +{ + struct dma_buf *dma_buf = attach->dmabuf; + struct drm_gem_object *obj = dma_buf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + struct sg_table *sgt; + int r = 0; + + if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT)) + return ERR_PTR(-EOPNOTSUPP); + + if (!xe_bo_is_pinned(bo)) { + if (!attach->peer2peer || + bo->ttm.resource->mem_type == XE_PL_SYSTEM) { + if (xe_bo_can_migrate(bo, XE_PL_TT)) + r = xe_bo_migrate(bo, XE_PL_TT); + else + r = xe_bo_validate(bo, NULL, false); + } + if (r) + return ERR_PTR(r); + } + + switch (bo->ttm.resource->mem_type) { + case XE_PL_TT: + sgt = drm_prime_pages_to_sg(obj->dev, + bo->ttm.ttm->pages, + bo->ttm.ttm->num_pages); + if (IS_ERR(sgt)) + return sgt; + + if (dma_map_sgtable(attach->dev, sgt, dir, + DMA_ATTR_SKIP_CPU_SYNC)) + goto error_free; + break; + + case XE_PL_VRAM0: + case XE_PL_VRAM1: + r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo), + bo->ttm.resource, 0, + bo->ttm.base.size, attach->dev, + dir, &sgt); + if (r) + return ERR_PTR(r); + break; + default: + return ERR_PTR(-EINVAL); + } + + return sgt; + +error_free: + sg_free_table(sgt); + kfree(sgt); + return ERR_PTR(-EBUSY); +} + +static void xe_dma_buf_unmap(struct dma_buf_attachment *attach, + struct sg_table *sgt, + enum dma_data_direction dir) +{ + struct dma_buf *dma_buf = attach->dmabuf; + struct xe_bo *bo = gem_to_xe_bo(dma_buf->priv); + + if (!xe_bo_is_vram(bo)) { + dma_unmap_sgtable(attach->dev, sgt, dir, 0); + sg_free_table(sgt); + kfree(sgt); + } else { + xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt); + } +} + +static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf, + enum dma_data_direction direction) +{ + struct drm_gem_object *obj = dma_buf->priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + bool reads = (direction == DMA_BIDIRECTIONAL || + direction == DMA_FROM_DEVICE); + + if (!reads) + return 0; + + xe_bo_lock_no_vm(bo, NULL); + (void)xe_bo_migrate(bo, XE_PL_TT); + xe_bo_unlock_no_vm(bo); + + return 0; +} + +const struct dma_buf_ops xe_dmabuf_ops = { + .attach = xe_dma_buf_attach, + .detach = xe_dma_buf_detach, + .pin = xe_dma_buf_pin, + .unpin = xe_dma_buf_unpin, + .map_dma_buf = xe_dma_buf_map, + .unmap_dma_buf = xe_dma_buf_unmap, + .release = drm_gem_dmabuf_release, + .begin_cpu_access = xe_dma_buf_begin_cpu_access, + .mmap = drm_gem_dmabuf_mmap, + .vmap = drm_gem_dmabuf_vmap, + .vunmap = drm_gem_dmabuf_vunmap, +}; + +struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags) +{ + struct xe_bo *bo = gem_to_xe_bo(obj); + struct dma_buf *buf; + + if (bo->vm) + return ERR_PTR(-EPERM); + + buf = drm_gem_prime_export(obj, flags); + if (!IS_ERR(buf)) + buf->ops = &xe_dmabuf_ops; + + return buf; +} + +static struct drm_gem_object * +xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, + struct dma_buf *dma_buf) +{ + struct dma_resv *resv = dma_buf->resv; + struct xe_device *xe = to_xe_device(dev); + struct xe_bo *bo; + int ret; + + dma_resv_lock(resv, NULL); + bo = __xe_bo_create_locked(xe, storage, NULL, resv, dma_buf->size, + ttm_bo_type_sg, XE_BO_CREATE_SYSTEM_BIT); + if (IS_ERR(bo)) { + ret = PTR_ERR(bo); + goto error; + } + dma_resv_unlock(resv); + + return &bo->ttm.base; + +error: + dma_resv_unlock(resv); + return ERR_PTR(ret); +} + +static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach) +{ + struct drm_gem_object *obj = attach->importer_priv; + struct xe_bo *bo = gem_to_xe_bo(obj); + + XE_WARN_ON(xe_bo_evict(bo, false)); +} + +static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = { + .allow_peer2peer = true, + .move_notify = xe_dma_buf_move_notify +}; + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) + +struct dma_buf_test_params { + struct xe_test_priv base; + const struct dma_buf_attach_ops *attach_ops; + bool force_different_devices; + u32 mem_mask; +}; + +#define to_dma_buf_test_params(_priv) \ + container_of(_priv, struct dma_buf_test_params, base) +#endif + +struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf) +{ + XE_TEST_DECLARE(struct dma_buf_test_params *test = + to_dma_buf_test_params + (xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));) + const struct dma_buf_attach_ops *attach_ops; + struct dma_buf_attachment *attach; + struct drm_gem_object *obj; + struct xe_bo *bo; + + if (dma_buf->ops == &xe_dmabuf_ops) { + obj = dma_buf->priv; + if (obj->dev == dev && + !XE_TEST_ONLY(test && test->force_different_devices)) { + /* + * Importing dmabuf exported from out own gem increases + * refcount on gem itself instead of f_count of dmabuf. + */ + drm_gem_object_get(obj); + return obj; + } + } + + /* + * Don't publish the bo until we have a valid attachment, and a + * valid attachment needs the bo address. So pre-create a bo before + * creating the attachment and publish. + */ + bo = xe_bo_alloc(); + if (IS_ERR(bo)) + return ERR_CAST(bo); + + attach_ops = &xe_dma_buf_attach_ops; +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) + if (test) + attach_ops = test->attach_ops; +#endif + + attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base); + if (IS_ERR(attach)) { + obj = ERR_CAST(attach); + goto out_err; + } + + /* Errors here will take care of freeing the bo. */ + obj = xe_dma_buf_init_obj(dev, bo, dma_buf); + if (IS_ERR(obj)) + return obj; + + + get_dma_buf(dma_buf); + obj->import_attach = attach; + return obj; + +out_err: + xe_bo_free(bo); + + return obj; +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_dma_buf.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_dma_buf.h b/drivers/gpu/drm/xe/xe_dma_buf.h new file mode 100644 index 000000000000..861dd28a862c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_dma_buf.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_DMA_BUF_H_ +#define _XE_DMA_BUF_H_ + +#include + +struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags); +struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev, + struct dma_buf *dma_buf); + +#endif diff --git a/drivers/gpu/drm/xe/xe_drv.h b/drivers/gpu/drm/xe/xe_drv.h new file mode 100644 index 000000000000..0377e5e4e35f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_drv.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_DRV_H_ +#define _XE_DRV_H_ + +#include + +#define DRIVER_NAME "xe" +#define DRIVER_DESC "Intel Xe Graphics" +#define DRIVER_DATE "20201103" +#define DRIVER_TIMESTAMP 1604406085 + +/* Interface history: + * + * 1.1: Original. + */ +#define DRIVER_MAJOR 1 +#define DRIVER_MINOR 1 +#define DRIVER_PATCHLEVEL 0 + +#endif diff --git a/drivers/gpu/drm/xe/xe_engine.c b/drivers/gpu/drm/xe/xe_engine.c new file mode 100644 index 000000000000..63219bd98be7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_engine.c @@ -0,0 +1,734 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_engine.h" + +#include +#include +#include +#include + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_migrate.h" +#include "xe_pm.h" +#include "xe_trace.h" +#include "xe_vm.h" + +static struct xe_engine *__xe_engine_create(struct xe_device *xe, + struct xe_vm *vm, + u32 logical_mask, + u16 width, struct xe_hw_engine *hwe, + u32 flags) +{ + struct xe_engine *e; + struct xe_gt *gt = hwe->gt; + int err; + int i; + + e = kzalloc(sizeof(*e) + sizeof(struct xe_lrc) * width, GFP_KERNEL); + if (!e) + return ERR_PTR(-ENOMEM); + + kref_init(&e->refcount); + e->flags = flags; + e->hwe = hwe; + e->gt = gt; + if (vm) + e->vm = xe_vm_get(vm); + e->class = hwe->class; + e->width = width; + e->logical_mask = logical_mask; + e->fence_irq = >->fence_irq[hwe->class]; + e->ring_ops = gt->ring_ops[hwe->class]; + e->ops = gt->engine_ops; + INIT_LIST_HEAD(&e->persitent.link); + INIT_LIST_HEAD(&e->compute.link); + INIT_LIST_HEAD(&e->multi_gt_link); + + /* FIXME: Wire up to configurable default value */ + e->sched_props.timeslice_us = 1 * 1000; + e->sched_props.preempt_timeout_us = 640 * 1000; + + if (xe_engine_is_parallel(e)) { + e->parallel.composite_fence_ctx = dma_fence_context_alloc(1); + e->parallel.composite_fence_seqno = 1; + } + if (e->flags & ENGINE_FLAG_VM) { + e->bind.fence_ctx = dma_fence_context_alloc(1); + e->bind.fence_seqno = 1; + } + + for (i = 0; i < width; ++i) { + err = xe_lrc_init(e->lrc + i, hwe, e, vm, SZ_16K); + if (err) + goto err_lrc; + } + + err = e->ops->init(e); + if (err) + goto err_lrc; + + return e; + +err_lrc: + for (i = i - 1; i >= 0; --i) + xe_lrc_finish(e->lrc + i); + kfree(e); + return ERR_PTR(err); +} + +struct xe_engine *xe_engine_create(struct xe_device *xe, struct xe_vm *vm, + u32 logical_mask, u16 width, + struct xe_hw_engine *hwe, u32 flags) +{ + struct ww_acquire_ctx ww; + struct xe_engine *e; + int err; + + if (vm) { + err = xe_vm_lock(vm, &ww, 0, true); + if (err) + return ERR_PTR(err); + } + e = __xe_engine_create(xe, vm, logical_mask, width, hwe, flags); + if (vm) + xe_vm_unlock(vm, &ww); + + return e; +} + +struct xe_engine *xe_engine_create_class(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, + enum xe_engine_class class, u32 flags) +{ + struct xe_hw_engine *hwe, *hwe0 = NULL; + enum xe_hw_engine_id id; + u32 logical_mask = 0; + + for_each_hw_engine(hwe, gt, id) { + if (xe_hw_engine_is_reserved(hwe)) + continue; + + if (hwe->class == class) { + logical_mask |= BIT(hwe->logical_instance); + if (!hwe0) + hwe0 = hwe; + } + } + + if (!logical_mask) + return ERR_PTR(-ENODEV); + + return xe_engine_create(xe, vm, logical_mask, 1, hwe0, flags); +} + +void xe_engine_destroy(struct kref *ref) +{ + struct xe_engine *e = container_of(ref, struct xe_engine, refcount); + struct xe_engine *engine, *next; + + if (!(e->flags & ENGINE_FLAG_BIND_ENGINE_CHILD)) { + list_for_each_entry_safe(engine, next, &e->multi_gt_list, + multi_gt_link) + xe_engine_put(engine); + } + + e->ops->fini(e); +} + +void xe_engine_fini(struct xe_engine *e) +{ + int i; + + for (i = 0; i < e->width; ++i) + xe_lrc_finish(e->lrc + i); + if (e->vm) + xe_vm_put(e->vm); + + kfree(e); +} + +struct xe_engine *xe_engine_lookup(struct xe_file *xef, u32 id) +{ + struct xe_engine *e; + + mutex_lock(&xef->engine.lock); + e = xa_load(&xef->engine.xa, id); + mutex_unlock(&xef->engine.lock); + + if (e) + xe_engine_get(e); + + return e; +} + +static int engine_set_priority(struct xe_device *xe, struct xe_engine *e, + u64 value, bool create) +{ + if (XE_IOCTL_ERR(xe, value > XE_ENGINE_PRIORITY_HIGH)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, value == XE_ENGINE_PRIORITY_HIGH && + !capable(CAP_SYS_NICE))) + return -EPERM; + + return e->ops->set_priority(e, value); +} + +static int engine_set_timeslice(struct xe_device *xe, struct xe_engine *e, + u64 value, bool create) +{ + if (!capable(CAP_SYS_NICE)) + return -EPERM; + + return e->ops->set_timeslice(e, value); +} + +static int engine_set_preemption_timeout(struct xe_device *xe, + struct xe_engine *e, u64 value, + bool create) +{ + if (!capable(CAP_SYS_NICE)) + return -EPERM; + + return e->ops->set_preempt_timeout(e, value); +} + +static int engine_set_compute_mode(struct xe_device *xe, struct xe_engine *e, + u64 value, bool create) +{ + if (XE_IOCTL_ERR(xe, !create)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, e->flags & ENGINE_FLAG_COMPUTE_MODE)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, e->flags & ENGINE_FLAG_VM)) + return -EINVAL; + + if (value) { + struct xe_vm *vm = e->vm; + int err; + + if (XE_IOCTL_ERR(xe, xe_vm_in_fault_mode(vm))) + return -EOPNOTSUPP; + + if (XE_IOCTL_ERR(xe, !xe_vm_in_compute_mode(vm))) + return -EOPNOTSUPP; + + if (XE_IOCTL_ERR(xe, e->width != 1)) + return -EINVAL; + + e->compute.context = dma_fence_context_alloc(1); + spin_lock_init(&e->compute.lock); + + err = xe_vm_add_compute_engine(vm, e); + if (XE_IOCTL_ERR(xe, err)) + return err; + + e->flags |= ENGINE_FLAG_COMPUTE_MODE; + e->flags &= ~ENGINE_FLAG_PERSISTENT; + } + + return 0; +} + +static int engine_set_persistence(struct xe_device *xe, struct xe_engine *e, + u64 value, bool create) +{ + if (XE_IOCTL_ERR(xe, !create)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, e->flags & ENGINE_FLAG_COMPUTE_MODE)) + return -EINVAL; + + if (value) + e->flags |= ENGINE_FLAG_PERSISTENT; + else + e->flags &= ~ENGINE_FLAG_PERSISTENT; + + return 0; +} + +static int engine_set_job_timeout(struct xe_device *xe, struct xe_engine *e, + u64 value, bool create) +{ + if (XE_IOCTL_ERR(xe, !create)) + return -EINVAL; + + if (!capable(CAP_SYS_NICE)) + return -EPERM; + + return e->ops->set_job_timeout(e, value); +} + +static int engine_set_acc_trigger(struct xe_device *xe, struct xe_engine *e, + u64 value, bool create) +{ + if (XE_IOCTL_ERR(xe, !create)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, !xe->info.supports_usm)) + return -EINVAL; + + e->usm.acc_trigger = value; + + return 0; +} + +static int engine_set_acc_notify(struct xe_device *xe, struct xe_engine *e, + u64 value, bool create) +{ + if (XE_IOCTL_ERR(xe, !create)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, !xe->info.supports_usm)) + return -EINVAL; + + e->usm.acc_notify = value; + + return 0; +} + +static int engine_set_acc_granularity(struct xe_device *xe, struct xe_engine *e, + u64 value, bool create) +{ + if (XE_IOCTL_ERR(xe, !create)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, !xe->info.supports_usm)) + return -EINVAL; + + e->usm.acc_granularity = value; + + return 0; +} + +typedef int (*xe_engine_set_property_fn)(struct xe_device *xe, + struct xe_engine *e, + u64 value, bool create); + +static const xe_engine_set_property_fn engine_set_property_funcs[] = { + [XE_ENGINE_PROPERTY_PRIORITY] = engine_set_priority, + [XE_ENGINE_PROPERTY_TIMESLICE] = engine_set_timeslice, + [XE_ENGINE_PROPERTY_PREEMPTION_TIMEOUT] = engine_set_preemption_timeout, + [XE_ENGINE_PROPERTY_COMPUTE_MODE] = engine_set_compute_mode, + [XE_ENGINE_PROPERTY_PERSISTENCE] = engine_set_persistence, + [XE_ENGINE_PROPERTY_JOB_TIMEOUT] = engine_set_job_timeout, + [XE_ENGINE_PROPERTY_ACC_TRIGGER] = engine_set_acc_trigger, + [XE_ENGINE_PROPERTY_ACC_NOTIFY] = engine_set_acc_notify, + [XE_ENGINE_PROPERTY_ACC_GRANULARITY] = engine_set_acc_granularity, +}; + +static int engine_user_ext_set_property(struct xe_device *xe, + struct xe_engine *e, + u64 extension, + bool create) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_ext_engine_set_property ext; + int err; + u32 idx; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_ERR(xe, err)) + return -EFAULT; + + if (XE_IOCTL_ERR(xe, ext.property >= + ARRAY_SIZE(engine_set_property_funcs))) + return -EINVAL; + + idx = array_index_nospec(ext.property, ARRAY_SIZE(engine_set_property_funcs)); + return engine_set_property_funcs[idx](xe, e, ext.value, create); +} + +typedef int (*xe_engine_user_extension_fn)(struct xe_device *xe, + struct xe_engine *e, + u64 extension, + bool create); + +static const xe_engine_set_property_fn engine_user_extension_funcs[] = { + [XE_ENGINE_EXTENSION_SET_PROPERTY] = engine_user_ext_set_property, +}; + +#define MAX_USER_EXTENSIONS 16 +static int engine_user_extensions(struct xe_device *xe, struct xe_engine *e, + u64 extensions, int ext_number, bool create) +{ + u64 __user *address = u64_to_user_ptr(extensions); + struct xe_user_extension ext; + int err; + u32 idx; + + if (XE_IOCTL_ERR(xe, ext_number >= MAX_USER_EXTENSIONS)) + return -E2BIG; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_ERR(xe, err)) + return -EFAULT; + + if (XE_IOCTL_ERR(xe, ext.name >= + ARRAY_SIZE(engine_user_extension_funcs))) + return -EINVAL; + + idx = array_index_nospec(ext.name, + ARRAY_SIZE(engine_user_extension_funcs)); + err = engine_user_extension_funcs[idx](xe, e, extensions, create); + if (XE_IOCTL_ERR(xe, err)) + return err; + + if (ext.next_extension) + return engine_user_extensions(xe, e, ext.next_extension, + ++ext_number, create); + + return 0; +} + +static const enum xe_engine_class user_to_xe_engine_class[] = { + [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, + [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, + [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, + [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, + [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, +}; + +static struct xe_hw_engine * +find_hw_engine(struct xe_device *xe, + struct drm_xe_engine_class_instance eci) +{ + u32 idx; + + if (eci.engine_class > ARRAY_SIZE(user_to_xe_engine_class)) + return NULL; + + if (eci.gt_id >= xe->info.tile_count) + return NULL; + + idx = array_index_nospec(eci.engine_class, + ARRAY_SIZE(user_to_xe_engine_class)); + + return xe_gt_hw_engine(xe_device_get_gt(xe, eci.gt_id), + user_to_xe_engine_class[idx], + eci.engine_instance, true); +} + +static u32 bind_engine_logical_mask(struct xe_device *xe, struct xe_gt *gt, + struct drm_xe_engine_class_instance *eci, + u16 width, u16 num_placements) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + u32 logical_mask = 0; + + if (XE_IOCTL_ERR(xe, width != 1)) + return 0; + if (XE_IOCTL_ERR(xe, num_placements != 1)) + return 0; + if (XE_IOCTL_ERR(xe, eci[0].engine_instance != 0)) + return 0; + + eci[0].engine_class = DRM_XE_ENGINE_CLASS_COPY; + + for_each_hw_engine(hwe, gt, id) { + if (xe_hw_engine_is_reserved(hwe)) + continue; + + if (hwe->class == + user_to_xe_engine_class[DRM_XE_ENGINE_CLASS_COPY]) + logical_mask |= BIT(hwe->logical_instance); + } + + return logical_mask; +} + +static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, + struct drm_xe_engine_class_instance *eci, + u16 width, u16 num_placements) +{ + int len = width * num_placements; + int i, j, n; + u16 class; + u16 gt_id; + u32 return_mask = 0, prev_mask; + + if (XE_IOCTL_ERR(xe, !xe_device_guc_submission_enabled(xe) && + len > 1)) + return 0; + + for (i = 0; i < width; ++i) { + u32 current_mask = 0; + + for (j = 0; j < num_placements; ++j) { + struct xe_hw_engine *hwe; + + n = j * width + i; + + hwe = find_hw_engine(xe, eci[n]); + if (XE_IOCTL_ERR(xe, !hwe)) + return 0; + + if (XE_IOCTL_ERR(xe, xe_hw_engine_is_reserved(hwe))) + return 0; + + if (XE_IOCTL_ERR(xe, n && eci[n].gt_id != gt_id) || + XE_IOCTL_ERR(xe, n && eci[n].engine_class != class)) + return 0; + + class = eci[n].engine_class; + gt_id = eci[n].gt_id; + + if (width == 1 || !i) + return_mask |= BIT(eci[n].engine_instance); + current_mask |= BIT(eci[n].engine_instance); + } + + /* Parallel submissions must be logically contiguous */ + if (i && XE_IOCTL_ERR(xe, current_mask != prev_mask << 1)) + return 0; + + prev_mask = current_mask; + } + + return return_mask; +} + +int xe_engine_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_engine_create *args = data; + struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; + struct drm_xe_engine_class_instance __user *user_eci = + u64_to_user_ptr(args->instances); + struct xe_hw_engine *hwe; + struct xe_vm *vm, *migrate_vm; + struct xe_gt *gt; + struct xe_engine *e = NULL; + u32 logical_mask; + u32 id; + int len; + int err; + + if (XE_IOCTL_ERR(xe, args->flags)) + return -EINVAL; + + len = args->width * args->num_placements; + if (XE_IOCTL_ERR(xe, !len || len > XE_HW_ENGINE_MAX_INSTANCE)) + return -EINVAL; + + err = __copy_from_user(eci, user_eci, + sizeof(struct drm_xe_engine_class_instance) * + len); + if (XE_IOCTL_ERR(xe, err)) + return -EFAULT; + + if (XE_IOCTL_ERR(xe, eci[0].gt_id >= xe->info.tile_count)) + return -EINVAL; + + xe_pm_runtime_get(xe); + + if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { + for_each_gt(gt, xe, id) { + struct xe_engine *new; + + if (xe_gt_is_media_type(gt)) + continue; + + eci[0].gt_id = gt->info.id; + logical_mask = bind_engine_logical_mask(xe, gt, eci, + args->width, + args->num_placements); + if (XE_IOCTL_ERR(xe, !logical_mask)) { + err = -EINVAL; + goto put_rpm; + } + + hwe = find_hw_engine(xe, eci[0]); + if (XE_IOCTL_ERR(xe, !hwe)) { + err = -EINVAL; + goto put_rpm; + } + + migrate_vm = xe_migrate_get_vm(gt->migrate); + new = xe_engine_create(xe, migrate_vm, logical_mask, + args->width, hwe, + ENGINE_FLAG_PERSISTENT | + ENGINE_FLAG_VM | + (id ? + ENGINE_FLAG_BIND_ENGINE_CHILD : + 0)); + xe_vm_put(migrate_vm); + if (IS_ERR(new)) { + err = PTR_ERR(new); + if (e) + goto put_engine; + goto put_rpm; + } + if (id == 0) + e = new; + else + list_add_tail(&new->multi_gt_list, + &e->multi_gt_link); + } + } else { + gt = xe_device_get_gt(xe, eci[0].gt_id); + logical_mask = calc_validate_logical_mask(xe, gt, eci, + args->width, + args->num_placements); + if (XE_IOCTL_ERR(xe, !logical_mask)) { + err = -EINVAL; + goto put_rpm; + } + + hwe = find_hw_engine(xe, eci[0]); + if (XE_IOCTL_ERR(xe, !hwe)) { + err = -EINVAL; + goto put_rpm; + } + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_ERR(xe, !vm)) { + err = -ENOENT; + goto put_rpm; + } + + e = xe_engine_create(xe, vm, logical_mask, + args->width, hwe, ENGINE_FLAG_PERSISTENT); + xe_vm_put(vm); + if (IS_ERR(e)) { + err = PTR_ERR(e); + goto put_rpm; + } + } + + if (args->extensions) { + err = engine_user_extensions(xe, e, args->extensions, 0, true); + if (XE_IOCTL_ERR(xe, err)) + goto put_engine; + } + + if (XE_IOCTL_ERR(xe, e->vm && xe_vm_in_compute_mode(e->vm) != + !!(e->flags & ENGINE_FLAG_COMPUTE_MODE))) { + err = -ENOTSUPP; + goto put_engine; + } + + e->persitent.xef = xef; + + mutex_lock(&xef->engine.lock); + err = xa_alloc(&xef->engine.xa, &id, e, xa_limit_32b, GFP_KERNEL); + mutex_unlock(&xef->engine.lock); + if (err) + goto put_engine; + + args->engine_id = id; + + return 0; + +put_engine: + xe_engine_kill(e); + xe_engine_put(e); +put_rpm: + xe_pm_runtime_put(xe); + return err; +} + +static void engine_kill_compute(struct xe_engine *e) +{ + if (!xe_vm_in_compute_mode(e->vm)) + return; + + down_write(&e->vm->lock); + list_del(&e->compute.link); + --e->vm->preempt.num_engines; + if (e->compute.pfence) { + dma_fence_enable_sw_signaling(e->compute.pfence); + dma_fence_put(e->compute.pfence); + e->compute.pfence = NULL; + } + up_write(&e->vm->lock); +} + +void xe_engine_kill(struct xe_engine *e) +{ + struct xe_engine *engine = e, *next; + + list_for_each_entry_safe(engine, next, &engine->multi_gt_list, + multi_gt_link) { + e->ops->kill(engine); + engine_kill_compute(engine); + } + + e->ops->kill(e); + engine_kill_compute(e); +} + +int xe_engine_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_engine_destroy *args = data; + struct xe_engine *e; + + if (XE_IOCTL_ERR(xe, args->pad)) + return -EINVAL; + + mutex_lock(&xef->engine.lock); + e = xa_erase(&xef->engine.xa, args->engine_id); + mutex_unlock(&xef->engine.lock); + if (XE_IOCTL_ERR(xe, !e)) + return -ENOENT; + + if (!(e->flags & ENGINE_FLAG_PERSISTENT)) + xe_engine_kill(e); + else + xe_device_add_persitent_engines(xe, e); + + trace_xe_engine_close(e); + xe_engine_put(e); + xe_pm_runtime_put(xe); + + return 0; +} + +int xe_engine_set_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_engine_set_property *args = data; + struct xe_engine *e; + int ret; + u32 idx; + + e = xe_engine_lookup(xef, args->engine_id); + if (XE_IOCTL_ERR(xe, !e)) + return -ENOENT; + + if (XE_IOCTL_ERR(xe, args->property >= + ARRAY_SIZE(engine_set_property_funcs))) { + ret = -EINVAL; + goto out; + } + + idx = array_index_nospec(args->property, + ARRAY_SIZE(engine_set_property_funcs)); + ret = engine_set_property_funcs[idx](xe, e, args->value, false); + if (XE_IOCTL_ERR(xe, ret)) + goto out; + + if (args->extensions) + ret = engine_user_extensions(xe, e, args->extensions, 0, + false); +out: + xe_engine_put(e); + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_engine.h b/drivers/gpu/drm/xe/xe_engine.h new file mode 100644 index 000000000000..4d1b609fea7e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_engine.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_ENGINE_H_ +#define _XE_ENGINE_H_ + +#include "xe_engine_types.h" +#include "xe_vm_types.h" + +struct drm_device; +struct drm_file; +struct xe_device; +struct xe_file; + +struct xe_engine *xe_engine_create(struct xe_device *xe, struct xe_vm *vm, + u32 logical_mask, u16 width, + struct xe_hw_engine *hw_engine, u32 flags); +struct xe_engine *xe_engine_create_class(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, + enum xe_engine_class class, u32 flags); + +void xe_engine_fini(struct xe_engine *e); +void xe_engine_destroy(struct kref *ref); + +struct xe_engine *xe_engine_lookup(struct xe_file *xef, u32 id); + +static inline struct xe_engine *xe_engine_get(struct xe_engine *engine) +{ + kref_get(&engine->refcount); + return engine; +} + +static inline void xe_engine_put(struct xe_engine *engine) +{ + kref_put(&engine->refcount, xe_engine_destroy); +} + +static inline bool xe_engine_is_parallel(struct xe_engine *engine) +{ + return engine->width > 1; +} + +void xe_engine_kill(struct xe_engine *e); + +int xe_engine_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_engine_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_engine_set_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +#endif diff --git a/drivers/gpu/drm/xe/xe_engine_types.h b/drivers/gpu/drm/xe/xe_engine_types.h new file mode 100644 index 000000000000..3dfa1c14e181 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_engine_types.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_ENGINE_TYPES_H_ +#define _XE_ENGINE_TYPES_H_ + +#include + +#include + +#include "xe_gpu_scheduler_types.h" +#include "xe_hw_engine_types.h" +#include "xe_hw_fence_types.h" +#include "xe_lrc_types.h" + +struct xe_execlist_engine; +struct xe_gt; +struct xe_guc_engine; +struct xe_hw_engine; +struct xe_vm; + +enum xe_engine_priority { + XE_ENGINE_PRIORITY_UNSET = -2, /* For execlist usage only */ + XE_ENGINE_PRIORITY_LOW = 0, + XE_ENGINE_PRIORITY_NORMAL, + XE_ENGINE_PRIORITY_HIGH, + XE_ENGINE_PRIORITY_KERNEL, + + XE_ENGINE_PRIORITY_COUNT +}; + +/** + * struct xe_engine - Submission engine + * + * Contains all state necessary for submissions. Can either be a user object or + * a kernel object. + */ +struct xe_engine { + /** @gt: graphics tile this engine can submit to */ + struct xe_gt *gt; + /** + * @hwe: A hardware of the same class. May (physical engine) or may not + * (virtual engine) be where jobs actual engine up running. Should never + * really be used for submissions. + */ + struct xe_hw_engine *hwe; + /** @refcount: ref count of this engine */ + struct kref refcount; + /** @vm: VM (address space) for this engine */ + struct xe_vm *vm; + /** @class: class of this engine */ + enum xe_engine_class class; + /** @priority: priority of this exec queue */ + enum xe_engine_priority priority; + /** + * @logical_mask: logical mask of where job submitted to engine can run + */ + u32 logical_mask; + /** @name: name of this engine */ + char name[MAX_FENCE_NAME_LEN]; + /** @width: width (number BB submitted per exec) of this engine */ + u16 width; + /** @fence_irq: fence IRQ used to signal job completion */ + struct xe_hw_fence_irq *fence_irq; + +#define ENGINE_FLAG_BANNED BIT(0) +#define ENGINE_FLAG_KERNEL BIT(1) +#define ENGINE_FLAG_PERSISTENT BIT(2) +#define ENGINE_FLAG_COMPUTE_MODE BIT(3) +#define ENGINE_FLAG_VM BIT(4) +#define ENGINE_FLAG_BIND_ENGINE_CHILD BIT(5) +#define ENGINE_FLAG_WA BIT(6) + + /** + * @flags: flags for this engine, should statically setup aside from ban + * bit + */ + unsigned long flags; + + union { + /** @multi_gt_list: list head for VM bind engines if multi-GT */ + struct list_head multi_gt_list; + /** @multi_gt_link: link for VM bind engines if multi-GT */ + struct list_head multi_gt_link; + }; + + union { + /** @execlist: execlist backend specific state for engine */ + struct xe_execlist_engine *execlist; + /** @guc: GuC backend specific state for engine */ + struct xe_guc_engine *guc; + }; + + /** + * @persitent: persitent engine state + */ + struct { + /** @xef: file which this engine belongs to */ + struct xe_file *xef; + /** @link: link in list of persitent engines */ + struct list_head link; + } persitent; + + union { + /** + * @parallel: parallel submission state + */ + struct { + /** @composite_fence_ctx: context composite fence */ + u64 composite_fence_ctx; + /** @composite_fence_seqno: seqno for composite fence */ + u32 composite_fence_seqno; + } parallel; + /** + * @bind: bind submission state + */ + struct { + /** @fence_ctx: context bind fence */ + u64 fence_ctx; + /** @fence_seqno: seqno for bind fence */ + u32 fence_seqno; + } bind; + }; + + /** @sched_props: scheduling properties */ + struct { + /** @timeslice_us: timeslice period in micro-seconds */ + u32 timeslice_us; + /** @preempt_timeout_us: preemption timeout in micro-seconds */ + u32 preempt_timeout_us; + } sched_props; + + /** @compute: compute engine state */ + struct { + /** @pfence: preemption fence */ + struct dma_fence *pfence; + /** @context: preemption fence context */ + u64 context; + /** @seqno: preemption fence seqno */ + u32 seqno; + /** @link: link into VM's list of engines */ + struct list_head link; + /** @lock: preemption fences lock */ + spinlock_t lock; + } compute; + + /** @usm: unified shared memory state */ + struct { + /** @acc_trigger: access counter trigger */ + u32 acc_trigger; + /** @acc_notify: access counter notify */ + u32 acc_notify; + /** @acc_granularity: access counter granularity */ + u32 acc_granularity; + } usm; + + /** @ops: submission backend engine operations */ + const struct xe_engine_ops *ops; + + /** @ring_ops: ring operations for this engine */ + const struct xe_ring_ops *ring_ops; + /** @entity: DRM sched entity for this engine (1 to 1 relationship) */ + struct drm_sched_entity *entity; + /** @lrc: logical ring context for this engine */ + struct xe_lrc lrc[0]; +}; + +/** + * struct xe_engine_ops - Submission backend engine operations + */ +struct xe_engine_ops { + /** @init: Initialize engine for submission backend */ + int (*init)(struct xe_engine *e); + /** @kill: Kill inflight submissions for backend */ + void (*kill)(struct xe_engine *e); + /** @fini: Fini engine for submission backend */ + void (*fini)(struct xe_engine *e); + /** @set_priority: Set priority for engine */ + int (*set_priority)(struct xe_engine *e, + enum xe_engine_priority priority); + /** @set_timeslice: Set timeslice for engine */ + int (*set_timeslice)(struct xe_engine *e, u32 timeslice_us); + /** @set_preempt_timeout: Set preemption timeout for engine */ + int (*set_preempt_timeout)(struct xe_engine *e, u32 preempt_timeout_us); + /** @set_job_timeout: Set job timeout for engine */ + int (*set_job_timeout)(struct xe_engine *e, u32 job_timeout_ms); + /** + * @suspend: Suspend engine from executing, allowed to be called + * multiple times in a row before resume with the caveat that + * suspend_wait returns before calling suspend again. + */ + int (*suspend)(struct xe_engine *e); + /** + * @suspend_wait: Wait for an engine to suspend executing, should be + * call after suspend. + */ + void (*suspend_wait)(struct xe_engine *e); + /** + * @resume: Resume engine execution, engine must be in a suspended + * state and dma fence returned from most recent suspend call must be + * signalled when this function is called. + */ + void (*resume)(struct xe_engine *e); +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c new file mode 100644 index 000000000000..00f298acc436 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -0,0 +1,390 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_engine.h" +#include "xe_exec.h" +#include "xe_macros.h" +#include "xe_sched_job.h" +#include "xe_sync.h" +#include "xe_vm.h" + +/** + * DOC: Execbuf (User GPU command submission) + * + * Execs have historically been rather complicated in DRM drivers (at least in + * the i915) because a few things: + * + * - Passing in a list BO which are read / written to creating implicit syncs + * - Binding at exec time + * - Flow controlling the ring at exec time + * + * In XE we avoid all of this complication by not allowing a BO list to be + * passed into an exec, using the dma-buf implicit sync uAPI, have binds as + * seperate operations, and using the DRM scheduler to flow control the ring. + * Let's deep dive on each of these. + * + * We can get away from a BO list by forcing the user to use in / out fences on + * every exec rather than the kernel tracking dependencies of BO (e.g. if the + * user knows an exec writes to a BO and reads from the BO in the next exec, it + * is the user's responsibility to pass in / out fence between the two execs). + * + * Implicit dependencies for external BOs are handled by using the dma-buf + * implicit dependency uAPI (TODO: add link). To make this works each exec must + * install the job's fence into the DMA_RESV_USAGE_WRITE slot of every external + * BO mapped in the VM. + * + * We do not allow a user to trigger a bind at exec time rather we have a VM + * bind IOCTL which uses the same in / out fence interface as exec. In that + * sense, a VM bind is basically the same operation as an exec from the user + * perspective. e.g. If an exec depends on a VM bind use the in / out fence + * interface (struct drm_xe_sync) to synchronize like syncing between two + * dependent execs. + * + * Although a user cannot trigger a bind, we still have to rebind userptrs in + * the VM that have been invalidated since the last exec, likewise we also have + * to rebind BOs that have been evicted by the kernel. We schedule these rebinds + * behind any pending kernel operations on any external BOs in VM or any BOs + * private to the VM. This is accomplished by the rebinds waiting on BOs + * DMA_RESV_USAGE_KERNEL slot (kernel ops) and kernel ops waiting on all BOs + * slots (inflight execs are in the DMA_RESV_USAGE_BOOKING for private BOs and + * in DMA_RESV_USAGE_WRITE for external BOs). + * + * Rebinds / dma-resv usage applies to non-compute mode VMs only as for compute + * mode VMs we use preempt fences and a rebind worker (TODO: add link). + * + * There is no need to flow control the ring in the exec as we write the ring at + * submission time and set the DRM scheduler max job limit SIZE_OF_RING / + * MAX_JOB_SIZE. The DRM scheduler will then hold all jobs until space in the + * ring is available. + * + * All of this results in a rather simple exec implementation. + * + * Flow + * ~~~~ + * + * .. code-block:: + * + * Parse input arguments + * Wait for any async VM bind passed as in-fences to start + * <----------------------------------------------------------------------| + * Lock global VM lock in read mode | + * Pin userptrs (also finds userptr invalidated since last exec) | + * Lock exec (VM dma-resv lock, external BOs dma-resv locks) | + * Validate BOs that have been evicted | + * Create job | + * Rebind invalidated userptrs + evicted BOs (non-compute-mode) | + * Add rebind fence dependency to job | + * Add job VM dma-resv bookkeeping slot (non-compute mode) | + * Add job to external BOs dma-resv write slots (non-compute mode) | + * Check if any userptrs invalidated since pin ------ Drop locks ---------| + * Install in / out fences for job + * Submit job + * Unlock all + */ + +static int xe_exec_begin(struct xe_engine *e, struct ww_acquire_ctx *ww, + struct ttm_validate_buffer tv_onstack[], + struct ttm_validate_buffer **tv, + struct list_head *objs) +{ + struct xe_vm *vm = e->vm; + struct xe_vma *vma; + LIST_HEAD(dups); + int err; + + *tv = NULL; + if (xe_vm_no_dma_fences(e->vm)) + return 0; + + err = xe_vm_lock_dma_resv(vm, ww, tv_onstack, tv, objs, true, 1); + if (err) + return err; + + /* + * Validate BOs that have been evicted (i.e. make sure the + * BOs have valid placements possibly moving an evicted BO back + * to a location where the GPU can access it). + */ + list_for_each_entry(vma, &vm->rebind_list, rebind_link) { + if (xe_vma_is_userptr(vma)) + continue; + + err = xe_bo_validate(vma->bo, vm, false); + if (err) { + xe_vm_unlock_dma_resv(vm, tv_onstack, *tv, ww, objs); + *tv = NULL; + return err; + } + } + + return 0; +} + +static void xe_exec_end(struct xe_engine *e, + struct ttm_validate_buffer *tv_onstack, + struct ttm_validate_buffer *tv, + struct ww_acquire_ctx *ww, + struct list_head *objs) +{ + if (!xe_vm_no_dma_fences(e->vm)) + xe_vm_unlock_dma_resv(e->vm, tv_onstack, tv, ww, objs); +} + +int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_exec *args = data; + struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs); + u64 __user *addresses_user = u64_to_user_ptr(args->address); + struct xe_engine *engine; + struct xe_sync_entry *syncs = NULL; + u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; + struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV]; + struct ttm_validate_buffer *tv = NULL; + u32 i, num_syncs = 0; + struct xe_sched_job *job; + struct dma_fence *rebind_fence; + struct xe_vm *vm; + struct ww_acquire_ctx ww; + struct list_head objs; + bool write_locked; + int err = 0; + + if (XE_IOCTL_ERR(xe, args->extensions)) + return -EINVAL; + + engine = xe_engine_lookup(xef, args->engine_id); + if (XE_IOCTL_ERR(xe, !engine)) + return -ENOENT; + + if (XE_IOCTL_ERR(xe, engine->flags & ENGINE_FLAG_VM)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, engine->width != args->num_batch_buffer)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, engine->flags & ENGINE_FLAG_BANNED)) { + err = -ECANCELED; + goto err_engine; + } + + if (args->num_syncs) { + syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); + if (!syncs) { + err = -ENOMEM; + goto err_engine; + } + } + + vm = engine->vm; + + for (i = 0; i < args->num_syncs; i++) { + err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs++], + &syncs_user[i], true, + xe_vm_no_dma_fences(vm)); + if (err) + goto err_syncs; + } + + if (xe_engine_is_parallel(engine)) { + err = __copy_from_user(addresses, addresses_user, sizeof(u64) * + engine->width); + if (err) { + err = -EFAULT; + goto err_syncs; + } + } + + /* + * We can't install a job into the VM dma-resv shared slot before an + * async VM bind passed in as a fence without the risk of deadlocking as + * the bind can trigger an eviction which in turn depends on anything in + * the VM dma-resv shared slots. Not an ideal solution, but we wait for + * all dependent async VM binds to start (install correct fences into + * dma-resv slots) before moving forward. + */ + if (!xe_vm_no_dma_fences(vm) && + vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS) { + for (i = 0; i < args->num_syncs; i++) { + struct dma_fence *fence = syncs[i].fence; + if (fence) { + err = xe_vm_async_fence_wait_start(fence); + if (err) + goto err_syncs; + } + } + } + +retry: + if (!xe_vm_no_dma_fences(vm) && xe_vm_userptr_check_repin(vm)) { + err = down_write_killable(&vm->lock); + write_locked = true; + } else { + /* We don't allow execs while the VM is in error state */ + err = down_read_interruptible(&vm->lock); + write_locked = false; + } + if (err) + goto err_syncs; + + /* We don't allow execs while the VM is in error state */ + if (vm->async_ops.error) { + err = vm->async_ops.error; + goto err_unlock_list; + } + + /* + * Extreme corner where we exit a VM error state with a munmap style VM + * unbind inflight which requires a rebind. In this case the rebind + * needs to install some fences into the dma-resv slots. The worker to + * do this queued, let that worker make progress by dropping vm->lock, + * flushing the worker and retrying the exec. + */ + if (vm->async_ops.munmap_rebind_inflight) { + if (write_locked) + up_write(&vm->lock); + else + up_read(&vm->lock); + flush_work(&vm->async_ops.work); + goto retry; + } + + if (write_locked) { + err = xe_vm_userptr_pin(vm); + downgrade_write(&vm->lock); + write_locked = false; + if (err) + goto err_unlock_list; + } + + err = xe_exec_begin(engine, &ww, tv_onstack, &tv, &objs); + if (err) + goto err_unlock_list; + + if (xe_vm_is_closed(engine->vm)) { + drm_warn(&xe->drm, "Trying to schedule after vm is closed\n"); + err = -EIO; + goto err_engine_end; + } + + job = xe_sched_job_create(engine, xe_engine_is_parallel(engine) ? + addresses : &args->address); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err_engine_end; + } + + /* + * Rebind any invalidated userptr or evicted BOs in the VM, non-compute + * VM mode only. + */ + rebind_fence = xe_vm_rebind(vm, false); + if (IS_ERR(rebind_fence)) { + err = PTR_ERR(rebind_fence); + goto err_put_job; + } + + /* + * We store the rebind_fence in the VM so subsequent execs don't get + * scheduled before the rebinds of userptrs / evicted BOs is complete. + */ + if (rebind_fence) { + dma_fence_put(vm->rebind_fence); + vm->rebind_fence = rebind_fence; + } + if (vm->rebind_fence) { + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + &vm->rebind_fence->flags)) { + dma_fence_put(vm->rebind_fence); + vm->rebind_fence = NULL; + } else { + dma_fence_get(vm->rebind_fence); + err = drm_sched_job_add_dependency(&job->drm, + vm->rebind_fence); + if (err) + goto err_put_job; + } + } + + /* Wait behind munmap style rebinds */ + if (!xe_vm_no_dma_fences(vm)) { + err = drm_sched_job_add_resv_dependencies(&job->drm, + &vm->resv, + DMA_RESV_USAGE_KERNEL); + if (err) + goto err_put_job; + } + + for (i = 0; i < num_syncs && !err; i++) + err = xe_sync_entry_add_deps(&syncs[i], job); + if (err) + goto err_put_job; + + if (!xe_vm_no_dma_fences(vm)) { + err = down_read_interruptible(&vm->userptr.notifier_lock); + if (err) + goto err_put_job; + + err = __xe_vm_userptr_needs_repin(vm); + if (err) + goto err_repin; + } + + /* + * Point of no return, if we error after this point just set an error on + * the job and let the DRM scheduler / backend clean up the job. + */ + xe_sched_job_arm(job); + if (!xe_vm_no_dma_fences(vm)) { + /* Block userptr invalidations / BO eviction */ + dma_resv_add_fence(&vm->resv, + &job->drm.s_fence->finished, + DMA_RESV_USAGE_BOOKKEEP); + + /* + * Make implicit sync work across drivers, assuming all external + * BOs are written as we don't pass in a read / write list. + */ + xe_vm_fence_all_extobjs(vm, &job->drm.s_fence->finished, + DMA_RESV_USAGE_WRITE); + } + + for (i = 0; i < num_syncs; i++) + xe_sync_entry_signal(&syncs[i], job, + &job->drm.s_fence->finished); + + xe_sched_job_push(job); + +err_repin: + if (!xe_vm_no_dma_fences(vm)) + up_read(&vm->userptr.notifier_lock); +err_put_job: + if (err) + xe_sched_job_put(job); +err_engine_end: + xe_exec_end(engine, tv_onstack, tv, &ww, &objs); +err_unlock_list: + if (write_locked) + up_write(&vm->lock); + else + up_read(&vm->lock); + if (err == -EAGAIN) + goto retry; +err_syncs: + for (i = 0; i < num_syncs; i++) + xe_sync_entry_cleanup(&syncs[i]); + kfree(syncs); +err_engine: + xe_engine_put(engine); + + return err; +} diff --git a/drivers/gpu/drm/xe/xe_exec.h b/drivers/gpu/drm/xe/xe_exec.h new file mode 100644 index 000000000000..e4932494cea3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_exec.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_EXEC_H_ +#define _XE_EXEC_H_ + +struct drm_device; +struct drm_file; + +int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file); + +#endif diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c new file mode 100644 index 000000000000..47587571123a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -0,0 +1,489 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include + +#include "xe_execlist.h" + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_engine.h" +#include "xe_hw_fence.h" +#include "xe_gt.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_mmio.h" +#include "xe_mocs.h" +#include "xe_ring_ops_types.h" +#include "xe_sched_job.h" + +#include "i915_reg.h" +#include "gt/intel_gpu_commands.h" +#include "gt/intel_gt_regs.h" +#include "gt/intel_lrc_reg.h" +#include "gt/intel_engine_regs.h" + +#define XE_EXECLIST_HANG_LIMIT 1 + +#define GEN11_SW_CTX_ID_SHIFT 37 +#define GEN11_SW_CTX_ID_WIDTH 11 +#define XEHP_SW_CTX_ID_SHIFT 39 +#define XEHP_SW_CTX_ID_WIDTH 16 + +#define GEN11_SW_CTX_ID \ + GENMASK_ULL(GEN11_SW_CTX_ID_WIDTH + GEN11_SW_CTX_ID_SHIFT - 1, \ + GEN11_SW_CTX_ID_SHIFT) + +#define XEHP_SW_CTX_ID \ + GENMASK_ULL(XEHP_SW_CTX_ID_WIDTH + XEHP_SW_CTX_ID_SHIFT - 1, \ + XEHP_SW_CTX_ID_SHIFT) + + +static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, + u32 ctx_id) +{ + struct xe_gt *gt = hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + u64 lrc_desc; + + printk(KERN_INFO "__start_lrc(%s, 0x%p, %u)\n", hwe->name, lrc, ctx_id); + + lrc_desc = xe_lrc_descriptor(lrc); + + if (GRAPHICS_VERx100(xe) >= 1250) { + XE_BUG_ON(!FIELD_FIT(XEHP_SW_CTX_ID, ctx_id)); + lrc_desc |= FIELD_PREP(XEHP_SW_CTX_ID, ctx_id); + } else { + XE_BUG_ON(!FIELD_FIT(GEN11_SW_CTX_ID, ctx_id)); + lrc_desc |= FIELD_PREP(GEN11_SW_CTX_ID, ctx_id); + } + + if (hwe->class == XE_ENGINE_CLASS_COMPUTE) + xe_mmio_write32(hwe->gt, GEN12_RCU_MODE.reg, + _MASKED_BIT_ENABLE(GEN12_RCU_MODE_CCS_ENABLE)); + + xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); + lrc->ring.old_tail = lrc->ring.tail; + + /* + * Make sure the context image is complete before we submit it to HW. + * + * Ostensibly, writes (including the WCB) should be flushed prior to + * an uncached write such as our mmio register access, the empirical + * evidence (esp. on Braswell) suggests that the WC write into memory + * may not be visible to the HW prior to the completion of the UC + * register write and that we may begin execution from the context + * before its image is complete leading to invalid PD chasing. + */ + wmb(); + + xe_mmio_write32(gt, RING_HWS_PGA(hwe->mmio_base).reg, + xe_bo_ggtt_addr(hwe->hwsp)); + xe_mmio_read32(gt, RING_HWS_PGA(hwe->mmio_base).reg); + xe_mmio_write32(gt, RING_MODE_GEN7(hwe->mmio_base).reg, + _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE)); + + xe_mmio_write32(gt, RING_EXECLIST_SQ_CONTENTS(hwe->mmio_base).reg + 0, + lower_32_bits(lrc_desc)); + xe_mmio_write32(gt, RING_EXECLIST_SQ_CONTENTS(hwe->mmio_base).reg + 4, + upper_32_bits(lrc_desc)); + xe_mmio_write32(gt, RING_EXECLIST_CONTROL(hwe->mmio_base).reg, + EL_CTRL_LOAD); +} + +static void __xe_execlist_port_start(struct xe_execlist_port *port, + struct xe_execlist_engine *exl) +{ + struct xe_device *xe = gt_to_xe(port->hwe->gt); + int max_ctx = FIELD_MAX(GEN11_SW_CTX_ID); + + if (GRAPHICS_VERx100(xe) >= 1250) + max_ctx = FIELD_MAX(XEHP_SW_CTX_ID); + + xe_execlist_port_assert_held(port); + + if (port->running_exl != exl || !exl->has_run) { + port->last_ctx_id++; + + /* 0 is reserved for the kernel context */ + if (port->last_ctx_id > max_ctx) + port->last_ctx_id = 1; + } + + __start_lrc(port->hwe, exl->engine->lrc, port->last_ctx_id); + port->running_exl = exl; + exl->has_run = true; +} + +static void __xe_execlist_port_idle(struct xe_execlist_port *port) +{ + u32 noop[2] = { MI_NOOP, MI_NOOP }; + + xe_execlist_port_assert_held(port); + + if (!port->running_exl) + return; + + printk(KERN_INFO "__xe_execlist_port_idle(%d:%d)\n", port->hwe->class, + port->hwe->instance); + + xe_lrc_write_ring(&port->hwe->kernel_lrc, noop, sizeof(noop)); + __start_lrc(port->hwe, &port->hwe->kernel_lrc, 0); + port->running_exl = NULL; +} + +static bool xe_execlist_is_idle(struct xe_execlist_engine *exl) +{ + struct xe_lrc *lrc = exl->engine->lrc; + + return lrc->ring.tail == lrc->ring.old_tail; +} + +static void __xe_execlist_port_start_next_active(struct xe_execlist_port *port) +{ + struct xe_execlist_engine *exl = NULL; + int i; + + xe_execlist_port_assert_held(port); + + for (i = ARRAY_SIZE(port->active) - 1; i >= 0; i--) { + while (!list_empty(&port->active[i])) { + exl = list_first_entry(&port->active[i], + struct xe_execlist_engine, + active_link); + list_del(&exl->active_link); + + if (xe_execlist_is_idle(exl)) { + exl->active_priority = XE_ENGINE_PRIORITY_UNSET; + continue; + } + + list_add_tail(&exl->active_link, &port->active[i]); + __xe_execlist_port_start(port, exl); + return; + } + } + + __xe_execlist_port_idle(port); +} + +static u64 read_execlist_status(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + u32 hi, lo; + + lo = xe_mmio_read32(gt, RING_EXECLIST_STATUS_LO(hwe->mmio_base).reg); + hi = xe_mmio_read32(gt, RING_EXECLIST_STATUS_HI(hwe->mmio_base).reg); + + printk(KERN_INFO "EXECLIST_STATUS %d:%d = 0x%08x %08x\n", hwe->class, + hwe->instance, hi, lo); + + return lo | (u64)hi << 32; +} + +static void xe_execlist_port_irq_handler_locked(struct xe_execlist_port *port) +{ + u64 status; + + xe_execlist_port_assert_held(port); + + status = read_execlist_status(port->hwe); + if (status & BIT(7)) + return; + + __xe_execlist_port_start_next_active(port); +} + +static void xe_execlist_port_irq_handler(struct xe_hw_engine *hwe, + u16 intr_vec) +{ + struct xe_execlist_port *port = hwe->exl_port; + + spin_lock(&port->lock); + xe_execlist_port_irq_handler_locked(port); + spin_unlock(&port->lock); +} + +static void xe_execlist_port_wake_locked(struct xe_execlist_port *port, + enum xe_engine_priority priority) +{ + xe_execlist_port_assert_held(port); + + if (port->running_exl && port->running_exl->active_priority >= priority) + return; + + __xe_execlist_port_start_next_active(port); +} + +static void xe_execlist_make_active(struct xe_execlist_engine *exl) +{ + struct xe_execlist_port *port = exl->port; + enum xe_engine_priority priority = exl->active_priority; + + XE_BUG_ON(priority == XE_ENGINE_PRIORITY_UNSET); + XE_BUG_ON(priority < 0); + XE_BUG_ON(priority >= ARRAY_SIZE(exl->port->active)); + + spin_lock_irq(&port->lock); + + if (exl->active_priority != priority && + exl->active_priority != XE_ENGINE_PRIORITY_UNSET) { + /* Priority changed, move it to the right list */ + list_del(&exl->active_link); + exl->active_priority = XE_ENGINE_PRIORITY_UNSET; + } + + if (exl->active_priority == XE_ENGINE_PRIORITY_UNSET) { + exl->active_priority = priority; + list_add_tail(&exl->active_link, &port->active[priority]); + } + + xe_execlist_port_wake_locked(exl->port, priority); + + spin_unlock_irq(&port->lock); +} + +static void xe_execlist_port_irq_fail_timer(struct timer_list *timer) +{ + struct xe_execlist_port *port = + container_of(timer, struct xe_execlist_port, irq_fail); + + spin_lock_irq(&port->lock); + xe_execlist_port_irq_handler_locked(port); + spin_unlock_irq(&port->lock); + + port->irq_fail.expires = jiffies + msecs_to_jiffies(1000); + add_timer(&port->irq_fail); +} + +struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe, + struct xe_hw_engine *hwe) +{ + struct drm_device *drm = &xe->drm; + struct xe_execlist_port *port; + int i; + + port = drmm_kzalloc(drm, sizeof(*port), GFP_KERNEL); + if (!port) + return ERR_PTR(-ENOMEM); + + port->hwe = hwe; + + spin_lock_init(&port->lock); + for (i = 0; i < ARRAY_SIZE(port->active); i++) + INIT_LIST_HEAD(&port->active[i]); + + port->last_ctx_id = 1; + port->running_exl = NULL; + + hwe->irq_handler = xe_execlist_port_irq_handler; + + /* TODO: Fix the interrupt code so it doesn't race like mad */ + timer_setup(&port->irq_fail, xe_execlist_port_irq_fail_timer, 0); + port->irq_fail.expires = jiffies + msecs_to_jiffies(1000); + add_timer(&port->irq_fail); + + return port; +} + +void xe_execlist_port_destroy(struct xe_execlist_port *port) +{ + del_timer(&port->irq_fail); + + /* Prevent an interrupt while we're destroying */ + spin_lock_irq(>_to_xe(port->hwe->gt)->irq.lock); + port->hwe->irq_handler = NULL; + spin_unlock_irq(>_to_xe(port->hwe->gt)->irq.lock); +} + +static struct dma_fence * +execlist_run_job(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + struct xe_engine *e = job->engine; + struct xe_execlist_engine *exl = job->engine->execlist; + + e->ring_ops->emit_job(job); + xe_execlist_make_active(exl); + + return dma_fence_get(job->fence); +} + +static void execlist_job_free(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + + xe_sched_job_put(job); +} + +static const struct drm_sched_backend_ops drm_sched_ops = { + .run_job = execlist_run_job, + .free_job = execlist_job_free, +}; + +static int execlist_engine_init(struct xe_engine *e) +{ + struct drm_gpu_scheduler *sched; + struct xe_execlist_engine *exl; + int err; + + XE_BUG_ON(xe_device_guc_submission_enabled(gt_to_xe(e->gt))); + + exl = kzalloc(sizeof(*exl), GFP_KERNEL); + if (!exl) + return -ENOMEM; + + exl->engine = e; + + err = drm_sched_init(&exl->sched, &drm_sched_ops, NULL, 1, + e->lrc[0].ring.size / MAX_JOB_SIZE_BYTES, + XE_SCHED_HANG_LIMIT, XE_SCHED_JOB_TIMEOUT, + NULL, NULL, e->hwe->name, + gt_to_xe(e->gt)->drm.dev); + if (err) + goto err_free; + + sched = &exl->sched; + err = drm_sched_entity_init(&exl->entity, 0, &sched, 1, NULL); + if (err) + goto err_sched; + + exl->port = e->hwe->exl_port; + exl->has_run = false; + exl->active_priority = XE_ENGINE_PRIORITY_UNSET; + e->execlist = exl; + e->entity = &exl->entity; + + switch (e->class) { + case XE_ENGINE_CLASS_RENDER: + sprintf(e->name, "rcs%d", ffs(e->logical_mask) - 1); + break; + case XE_ENGINE_CLASS_VIDEO_DECODE: + sprintf(e->name, "vcs%d", ffs(e->logical_mask) - 1); + break; + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + sprintf(e->name, "vecs%d", ffs(e->logical_mask) - 1); + break; + case XE_ENGINE_CLASS_COPY: + sprintf(e->name, "bcs%d", ffs(e->logical_mask) - 1); + break; + case XE_ENGINE_CLASS_COMPUTE: + sprintf(e->name, "ccs%d", ffs(e->logical_mask) - 1); + break; + default: + XE_WARN_ON(e->class); + } + + return 0; + +err_sched: + drm_sched_fini(&exl->sched); +err_free: + kfree(exl); + return err; +} + +static void execlist_engine_fini_async(struct work_struct *w) +{ + struct xe_execlist_engine *ee = + container_of(w, struct xe_execlist_engine, fini_async); + struct xe_engine *e = ee->engine; + struct xe_execlist_engine *exl = e->execlist; + unsigned long flags; + + XE_BUG_ON(xe_device_guc_submission_enabled(gt_to_xe(e->gt))); + + spin_lock_irqsave(&exl->port->lock, flags); + if (WARN_ON(exl->active_priority != XE_ENGINE_PRIORITY_UNSET)) + list_del(&exl->active_link); + spin_unlock_irqrestore(&exl->port->lock, flags); + + if (e->flags & ENGINE_FLAG_PERSISTENT) + xe_device_remove_persitent_engines(gt_to_xe(e->gt), e); + drm_sched_entity_fini(&exl->entity); + drm_sched_fini(&exl->sched); + kfree(exl); + + xe_engine_fini(e); +} + +static void execlist_engine_kill(struct xe_engine *e) +{ + /* NIY */ +} + +static void execlist_engine_fini(struct xe_engine *e) +{ + INIT_WORK(&e->execlist->fini_async, execlist_engine_fini_async); + queue_work(system_unbound_wq, &e->execlist->fini_async); +} + +static int execlist_engine_set_priority(struct xe_engine *e, + enum xe_engine_priority priority) +{ + /* NIY */ + return 0; +} + +static int execlist_engine_set_timeslice(struct xe_engine *e, u32 timeslice_us) +{ + /* NIY */ + return 0; +} + +static int execlist_engine_set_preempt_timeout(struct xe_engine *e, + u32 preempt_timeout_us) +{ + /* NIY */ + return 0; +} + +static int execlist_engine_set_job_timeout(struct xe_engine *e, + u32 job_timeout_ms) +{ + /* NIY */ + return 0; +} + +static int execlist_engine_suspend(struct xe_engine *e) +{ + /* NIY */ + return 0; +} + +static void execlist_engine_suspend_wait(struct xe_engine *e) + +{ + /* NIY */ +} + +static void execlist_engine_resume(struct xe_engine *e) +{ + xe_mocs_init_engine(e); +} + +static const struct xe_engine_ops execlist_engine_ops = { + .init = execlist_engine_init, + .kill = execlist_engine_kill, + .fini = execlist_engine_fini, + .set_priority = execlist_engine_set_priority, + .set_timeslice = execlist_engine_set_timeslice, + .set_preempt_timeout = execlist_engine_set_preempt_timeout, + .set_job_timeout = execlist_engine_set_job_timeout, + .suspend = execlist_engine_suspend, + .suspend_wait = execlist_engine_suspend_wait, + .resume = execlist_engine_resume, +}; + +int xe_execlist_init(struct xe_gt *gt) +{ + /* GuC submission enabled, nothing to do */ + if (xe_device_guc_submission_enabled(gt_to_xe(gt))) + return 0; + + gt->engine_ops = &execlist_engine_ops; + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_execlist.h b/drivers/gpu/drm/xe/xe_execlist.h new file mode 100644 index 000000000000..6a0442a6eff6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_execlist.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_EXECLIST_H_ +#define _XE_EXECLIST_H_ + +#include "xe_execlist_types.h" + +struct xe_device; +struct xe_gt; + +#define xe_execlist_port_assert_held(port) lockdep_assert_held(&(port)->lock); + +int xe_execlist_init(struct xe_gt *gt); +struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe, + struct xe_hw_engine *hwe); +void xe_execlist_port_destroy(struct xe_execlist_port *port); + +#endif diff --git a/drivers/gpu/drm/xe/xe_execlist_types.h b/drivers/gpu/drm/xe/xe_execlist_types.h new file mode 100644 index 000000000000..9b1239b47292 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_execlist_types.h @@ -0,0 +1,49 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_EXECLIST_TYPES_H_ +#define _XE_EXECLIST_TYPES_H_ + +#include +#include +#include + +#include "xe_engine_types.h" + +struct xe_hw_engine; +struct xe_execlist_engine; + +struct xe_execlist_port { + struct xe_hw_engine *hwe; + + spinlock_t lock; + + struct list_head active[XE_ENGINE_PRIORITY_COUNT]; + + u32 last_ctx_id; + + struct xe_execlist_engine *running_exl; + + struct timer_list irq_fail; +}; + +struct xe_execlist_engine { + struct xe_engine *engine; + + struct drm_gpu_scheduler sched; + + struct drm_sched_entity entity; + + struct xe_execlist_port *port; + + bool has_run; + + struct work_struct fini_async; + + enum xe_engine_priority active_priority; + struct list_head active_link; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_force_wake.c b/drivers/gpu/drm/xe/xe_force_wake.c new file mode 100644 index 000000000000..0320ce7ba3d1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_force_wake.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_mmio.h" +#include "gt/intel_gt_regs.h" + +#define XE_FORCE_WAKE_ACK_TIMEOUT_MS 50 + +static struct xe_gt * +fw_to_gt(struct xe_force_wake *fw) +{ + return fw->gt; +} + +static struct xe_device * +fw_to_xe(struct xe_force_wake *fw) +{ + return gt_to_xe(fw_to_gt(fw)); +} + +static void domain_init(struct xe_force_wake_domain *domain, + enum xe_force_wake_domain_id id, + u32 reg, u32 ack, u32 val, u32 mask) +{ + domain->id = id; + domain->reg_ctl = reg; + domain->reg_ack = ack; + domain->val = val; + domain->mask = mask; +} + +#define FORCEWAKE_ACK_GT_MTL _MMIO(0xdfc) + +void xe_force_wake_init_gt(struct xe_gt *gt, struct xe_force_wake *fw) +{ + struct xe_device *xe = gt_to_xe(gt); + + fw->gt = gt; + mutex_init(&fw->lock); + + /* Assuming gen11+ so assert this assumption is correct */ + XE_BUG_ON(GRAPHICS_VER(gt_to_xe(gt)) < 11); + + if (xe->info.platform == XE_METEORLAKE) { + domain_init(&fw->domains[XE_FW_DOMAIN_ID_GT], + XE_FW_DOMAIN_ID_GT, + FORCEWAKE_GT_GEN9.reg, + FORCEWAKE_ACK_GT_MTL.reg, + BIT(0), BIT(16)); + } else { + domain_init(&fw->domains[XE_FW_DOMAIN_ID_GT], + XE_FW_DOMAIN_ID_GT, + FORCEWAKE_GT_GEN9.reg, + FORCEWAKE_ACK_GT_GEN9.reg, + BIT(0), BIT(16)); + } +} + +void xe_force_wake_init_engines(struct xe_gt *gt, struct xe_force_wake *fw) +{ + int i, j; + + /* Assuming gen11+ so assert this assumption is correct */ + XE_BUG_ON(GRAPHICS_VER(gt_to_xe(gt)) < 11); + + if (!xe_gt_is_media_type(gt)) + domain_init(&fw->domains[XE_FW_DOMAIN_ID_RENDER], + XE_FW_DOMAIN_ID_RENDER, + FORCEWAKE_RENDER_GEN9.reg, + FORCEWAKE_ACK_RENDER_GEN9.reg, + BIT(0), BIT(16)); + + for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + domain_init(&fw->domains[XE_FW_DOMAIN_ID_MEDIA_VDBOX0 + j], + XE_FW_DOMAIN_ID_MEDIA_VDBOX0 + j, + FORCEWAKE_MEDIA_VDBOX_GEN11(j).reg, + FORCEWAKE_ACK_MEDIA_VDBOX_GEN11(j).reg, + BIT(0), BIT(16)); + } + + for (i = XE_HW_ENGINE_VECS0, j =0; i <= XE_HW_ENGINE_VECS3; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + domain_init(&fw->domains[XE_FW_DOMAIN_ID_MEDIA_VEBOX0 + j], + XE_FW_DOMAIN_ID_MEDIA_VEBOX0 + j, + FORCEWAKE_MEDIA_VEBOX_GEN11(j).reg, + FORCEWAKE_ACK_MEDIA_VEBOX_GEN11(j).reg, + BIT(0), BIT(16)); + } +} + +void xe_force_wake_prune(struct xe_gt *gt, struct xe_force_wake *fw) +{ + int i, j; + + /* Call after fuses have been read, prune domains that are fused off */ + + for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) + if (!(gt->info.engine_mask & BIT(i))) + fw->domains[XE_FW_DOMAIN_ID_MEDIA_VDBOX0 + j].reg_ctl = 0; + + for (i = XE_HW_ENGINE_VECS0, j =0; i <= XE_HW_ENGINE_VECS3; ++i, ++j) + if (!(gt->info.engine_mask & BIT(i))) + fw->domains[XE_FW_DOMAIN_ID_MEDIA_VEBOX0 + j].reg_ctl = 0; +} + +static void domain_wake(struct xe_gt *gt, struct xe_force_wake_domain *domain) +{ + xe_mmio_write32(gt, domain->reg_ctl, domain->mask | domain->val); +} + +static int domain_wake_wait(struct xe_gt *gt, + struct xe_force_wake_domain *domain) +{ + return xe_mmio_wait32(gt, domain->reg_ack, domain->val, domain->val, + XE_FORCE_WAKE_ACK_TIMEOUT_MS); +} + +static void domain_sleep(struct xe_gt *gt, struct xe_force_wake_domain *domain) +{ + xe_mmio_write32(gt, domain->reg_ctl, domain->mask); +} + +static int domain_sleep_wait(struct xe_gt *gt, + struct xe_force_wake_domain *domain) +{ + return xe_mmio_wait32(gt, domain->reg_ack, 0, domain->val, + XE_FORCE_WAKE_ACK_TIMEOUT_MS); +} + +#define for_each_fw_domain_masked(domain__, mask__, fw__, tmp__) \ + for (tmp__ = (mask__); tmp__ ;) \ + for_each_if((domain__ = ((fw__)->domains + \ + __mask_next_bit(tmp__))) && \ + domain__->reg_ctl) + +int xe_force_wake_get(struct xe_force_wake *fw, + enum xe_force_wake_domains domains) +{ + struct xe_device *xe = fw_to_xe(fw); + struct xe_gt *gt = fw_to_gt(fw); + struct xe_force_wake_domain *domain; + enum xe_force_wake_domains tmp, woken = 0; + int ret, ret2 = 0; + + mutex_lock(&fw->lock); + for_each_fw_domain_masked(domain, domains, fw, tmp) { + if (!domain->ref++) { + woken |= BIT(domain->id); + domain_wake(gt, domain); + } + } + for_each_fw_domain_masked(domain, woken, fw, tmp) { + ret = domain_wake_wait(gt, domain); + ret2 |= ret; + if (ret) + drm_notice(&xe->drm, "Force wake domain (%d) failed to ack wake, ret=%d\n", + domain->id, ret); + } + fw->awake_domains |= woken; + mutex_unlock(&fw->lock); + + return ret2; +} + +int xe_force_wake_put(struct xe_force_wake *fw, + enum xe_force_wake_domains domains) +{ + struct xe_device *xe = fw_to_xe(fw); + struct xe_gt *gt = fw_to_gt(fw); + struct xe_force_wake_domain *domain; + enum xe_force_wake_domains tmp, sleep = 0; + int ret, ret2 = 0; + + mutex_lock(&fw->lock); + for_each_fw_domain_masked(domain, domains, fw, tmp) { + if (!--domain->ref) { + sleep |= BIT(domain->id); + domain_sleep(gt, domain); + } + } + for_each_fw_domain_masked(domain, sleep, fw, tmp) { + ret = domain_sleep_wait(gt, domain); + ret2 |= ret; + if (ret) + drm_notice(&xe->drm, "Force wake domain (%d) failed to ack sleep, ret=%d\n", + domain->id, ret); + } + fw->awake_domains &= ~sleep; + mutex_unlock(&fw->lock); + + return ret2; +} diff --git a/drivers/gpu/drm/xe/xe_force_wake.h b/drivers/gpu/drm/xe/xe_force_wake.h new file mode 100644 index 000000000000..5adb8daa3b71 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_force_wake.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_FORCE_WAKE_H_ +#define _XE_FORCE_WAKE_H_ + +#include "xe_force_wake_types.h" +#include "xe_macros.h" + +struct xe_gt; + +void xe_force_wake_init_gt(struct xe_gt *gt, + struct xe_force_wake *fw); +void xe_force_wake_init_engines(struct xe_gt *gt, + struct xe_force_wake *fw); +void xe_force_wake_prune(struct xe_gt *gt, + struct xe_force_wake *fw); +int xe_force_wake_get(struct xe_force_wake *fw, + enum xe_force_wake_domains domains); +int xe_force_wake_put(struct xe_force_wake *fw, + enum xe_force_wake_domains domains); + +static inline int +xe_force_wake_ref(struct xe_force_wake *fw, + enum xe_force_wake_domains domain) +{ + XE_BUG_ON(!domain); + return fw->domains[ffs(domain) - 1].ref; +} + +static inline void +xe_force_wake_assert_held(struct xe_force_wake *fw, + enum xe_force_wake_domains domain) +{ + XE_BUG_ON(!(fw->awake_domains & domain)); +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_force_wake_types.h b/drivers/gpu/drm/xe/xe_force_wake_types.h new file mode 100644 index 000000000000..208dd629d7b1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_force_wake_types.h @@ -0,0 +1,84 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_FORCE_WAKE_TYPES_H_ +#define _XE_FORCE_WAKE_TYPES_H_ + +#include +#include + +enum xe_force_wake_domain_id { + XE_FW_DOMAIN_ID_GT = 0, + XE_FW_DOMAIN_ID_RENDER, + XE_FW_DOMAIN_ID_MEDIA, + XE_FW_DOMAIN_ID_MEDIA_VDBOX0, + XE_FW_DOMAIN_ID_MEDIA_VDBOX1, + XE_FW_DOMAIN_ID_MEDIA_VDBOX2, + XE_FW_DOMAIN_ID_MEDIA_VDBOX3, + XE_FW_DOMAIN_ID_MEDIA_VDBOX4, + XE_FW_DOMAIN_ID_MEDIA_VDBOX5, + XE_FW_DOMAIN_ID_MEDIA_VDBOX6, + XE_FW_DOMAIN_ID_MEDIA_VDBOX7, + XE_FW_DOMAIN_ID_MEDIA_VEBOX0, + XE_FW_DOMAIN_ID_MEDIA_VEBOX1, + XE_FW_DOMAIN_ID_MEDIA_VEBOX2, + XE_FW_DOMAIN_ID_MEDIA_VEBOX3, + XE_FW_DOMAIN_ID_GSC, + XE_FW_DOMAIN_ID_COUNT +}; + +enum xe_force_wake_domains { + XE_FW_GT = BIT(XE_FW_DOMAIN_ID_GT), + XE_FW_RENDER = BIT(XE_FW_DOMAIN_ID_RENDER), + XE_FW_MEDIA = BIT(XE_FW_DOMAIN_ID_MEDIA), + XE_FW_MEDIA_VDBOX0 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX0), + XE_FW_MEDIA_VDBOX1 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX1), + XE_FW_MEDIA_VDBOX2 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX2), + XE_FW_MEDIA_VDBOX3 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX3), + XE_FW_MEDIA_VDBOX4 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX4), + XE_FW_MEDIA_VDBOX5 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX5), + XE_FW_MEDIA_VDBOX6 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX6), + XE_FW_MEDIA_VDBOX7 = BIT(XE_FW_DOMAIN_ID_MEDIA_VDBOX7), + XE_FW_MEDIA_VEBOX0 = BIT(XE_FW_DOMAIN_ID_MEDIA_VEBOX0), + XE_FW_MEDIA_VEBOX1 = BIT(XE_FW_DOMAIN_ID_MEDIA_VEBOX1), + XE_FW_MEDIA_VEBOX2 = BIT(XE_FW_DOMAIN_ID_MEDIA_VEBOX2), + XE_FW_MEDIA_VEBOX3 = BIT(XE_FW_DOMAIN_ID_MEDIA_VEBOX3), + XE_FW_GSC = BIT(XE_FW_DOMAIN_ID_GSC), + XE_FORCEWAKE_ALL = BIT(XE_FW_DOMAIN_ID_COUNT) - 1 +}; + +/** + * struct xe_force_wake_domain - XE force wake domains + */ +struct xe_force_wake_domain { + /** @id: domain force wake id */ + enum xe_force_wake_domain_id id; + /** @reg_ctl: domain wake control register address */ + u32 reg_ctl; + /** @reg_ack: domain ack register address */ + u32 reg_ack; + /** @val: domain wake write value */ + u32 val; + /** @mask: domain mask */ + u32 mask; + /** @ref: domain reference */ + u32 ref; +}; + +/** + * struct xe_force_wake - XE force wake + */ +struct xe_force_wake { + /** @gt: back pointers to GT */ + struct xe_gt *gt; + /** @lock: protects everything force wake struct */ + struct mutex lock; + /** @awake_domains: mask of all domains awake */ + enum xe_force_wake_domains awake_domains; + /** @domains: force wake domains */ + struct xe_force_wake_domain domains[XE_FW_DOMAIN_ID_COUNT]; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c new file mode 100644 index 000000000000..eab74a509f68 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_ggtt.h" + +#include +#include + +#include + +#include "xe_device.h" +#include "xe_bo.h" +#include "xe_gt.h" +#include "xe_mmio.h" +#include "xe_wopcm.h" + +#include "i915_reg.h" +#include "gt/intel_gt_regs.h" + +/* FIXME: Common file, preferably auto-gen */ +#define MTL_GGTT_PTE_PAT0 BIT(52) +#define MTL_GGTT_PTE_PAT1 BIT(53) + +u64 xe_ggtt_pte_encode(struct xe_bo *bo, u64 bo_offset) +{ + struct xe_device *xe = xe_bo_device(bo); + u64 pte; + bool is_lmem; + + pte = xe_bo_addr(bo, bo_offset, GEN8_PAGE_SIZE, &is_lmem); + pte |= GEN8_PAGE_PRESENT; + + if (is_lmem) + pte |= GEN12_GGTT_PTE_LM; + + /* FIXME: vfunc + pass in caching rules */ + if (xe->info.platform == XE_METEORLAKE) { + pte |= MTL_GGTT_PTE_PAT0; + pte |= MTL_GGTT_PTE_PAT1; + } + + return pte; +} + +static unsigned int probe_gsm_size(struct pci_dev *pdev) +{ + u16 gmch_ctl, ggms; + + pci_read_config_word(pdev, SNB_GMCH_CTRL, &gmch_ctl); + ggms = (gmch_ctl >> BDW_GMCH_GGMS_SHIFT) & BDW_GMCH_GGMS_MASK; + return ggms ? SZ_1M << ggms : 0; +} + +void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte) +{ + XE_BUG_ON(addr & GEN8_PTE_MASK); + XE_BUG_ON(addr >= ggtt->size); + + writeq(pte, &ggtt->gsm[addr >> GEN8_PTE_SHIFT]); +} + +static void xe_ggtt_clear(struct xe_ggtt *ggtt, u64 start, u64 size) +{ + u64 end = start + size - 1; + u64 scratch_pte; + + XE_BUG_ON(start >= end); + + if (ggtt->scratch) + scratch_pte = xe_ggtt_pte_encode(ggtt->scratch, 0); + else + scratch_pte = 0; + + while (start < end) { + xe_ggtt_set_pte(ggtt, start, scratch_pte); + start += GEN8_PAGE_SIZE; + } +} + +static void ggtt_fini_noalloc(struct drm_device *drm, void *arg) +{ + struct xe_ggtt *ggtt = arg; + + mutex_destroy(&ggtt->lock); + drm_mm_takedown(&ggtt->mm); + + xe_bo_unpin_map_no_vm(ggtt->scratch); +} + +int xe_ggtt_init_noalloc(struct xe_gt *gt, struct xe_ggtt *ggtt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + unsigned int gsm_size; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + + ggtt->gt = gt; + + gsm_size = probe_gsm_size(pdev); + if (gsm_size == 0) { + drm_err(&xe->drm, "Hardware reported no preallocated GSM\n"); + return -ENOMEM; + } + + ggtt->gsm = gt->mmio.regs + SZ_8M; + ggtt->size = (gsm_size / 8) * (u64)GEN8_PAGE_SIZE; + + /* + * 8B per entry, each points to a 4KB page. + * + * The GuC owns the WOPCM space, thus we can't allocate GGTT address in + * this area. Even though we likely configure the WOPCM to less than the + * maximum value, to simplify the driver load (no need to fetch HuC + + * GuC firmwares and determine there sizes before initializing the GGTT) + * just start the GGTT allocation above the max WOPCM size. This might + * waste space in the GGTT (WOPCM is 2MB on modern platforms) but we can + * live with this. + * + * Another benifit of this is the GuC bootrom can't access anything + * below the WOPCM max size so anything the bootom needs to access (e.g. + * a RSA key) needs to be placed in the GGTT above the WOPCM max size. + * Starting the GGTT allocations above the WOPCM max give us the correct + * placement for free. + */ + drm_mm_init(&ggtt->mm, xe_wopcm_size(xe), + ggtt->size - xe_wopcm_size(xe)); + mutex_init(&ggtt->lock); + + return drmm_add_action_or_reset(&xe->drm, ggtt_fini_noalloc, ggtt); +} + +static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt) +{ + struct drm_mm_node *hole; + u64 start, end; + + /* Display may have allocated inside ggtt, so be careful with clearing here */ + mutex_lock(&ggtt->lock); + drm_mm_for_each_hole(hole, &ggtt->mm, start, end) + xe_ggtt_clear(ggtt, start, end - start); + + xe_ggtt_invalidate(ggtt->gt); + mutex_unlock(&ggtt->lock); +} + +int xe_ggtt_init(struct xe_gt *gt, struct xe_ggtt *ggtt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + ggtt->scratch = xe_bo_create_locked(xe, gt, NULL, GEN8_PAGE_SIZE, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_PINNED_BIT); + if (IS_ERR(ggtt->scratch)) { + err = PTR_ERR(ggtt->scratch); + goto err; + } + + err = xe_bo_pin(ggtt->scratch); + xe_bo_unlock_no_vm(ggtt->scratch); + if (err) { + xe_bo_put(ggtt->scratch); + goto err; + } + + xe_ggtt_initial_clear(ggtt); + return 0; +err: + ggtt->scratch = NULL; + return err; +} + +#define GEN12_GUC_TLB_INV_CR _MMIO(0xcee8) +#define GEN12_GUC_TLB_INV_CR_INVALIDATE (1 << 0) +#define PVC_GUC_TLB_INV_DESC0 _MMIO(0xcf7c) +#define PVC_GUC_TLB_INV_DESC0_VALID (1 << 0) +#define PVC_GUC_TLB_INV_DESC1 _MMIO(0xcf80) +#define PVC_GUC_TLB_INV_DESC1_INVALIDATE (1 << 6) + +void xe_ggtt_invalidate(struct xe_gt *gt) +{ + /* TODO: vfunc for GuC vs. non-GuC */ + + /* TODO: i915 makes comments about this being uncached and + * therefore flushing WC buffers. Is that really true here? + */ + xe_mmio_write32(gt, GFX_FLSH_CNTL_GEN6.reg, GFX_FLSH_CNTL_EN); + if (xe_device_guc_submission_enabled(gt_to_xe(gt))) { + struct xe_device *xe = gt_to_xe(gt); + + /* TODO: also use vfunc here */ + if (xe->info.platform == XE_PVC) { + xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC1.reg, + PVC_GUC_TLB_INV_DESC1_INVALIDATE); + xe_mmio_write32(gt, PVC_GUC_TLB_INV_DESC0.reg, + PVC_GUC_TLB_INV_DESC0_VALID); + } else + xe_mmio_write32(gt, GEN12_GUC_TLB_INV_CR.reg, + GEN12_GUC_TLB_INV_CR_INVALIDATE); + } +} + +void xe_ggtt_printk(struct xe_ggtt *ggtt, const char *prefix) +{ + u64 addr, scratch_pte; + + scratch_pte = xe_ggtt_pte_encode(ggtt->scratch, 0); + + printk("%sGlobal GTT:", prefix); + for (addr = 0; addr < ggtt->size; addr += GEN8_PAGE_SIZE) { + unsigned int i = addr / GEN8_PAGE_SIZE; + + XE_BUG_ON(addr > U32_MAX); + if (ggtt->gsm[i] == scratch_pte) + continue; + + printk("%s ggtt[0x%08x] = 0x%016llx", + prefix, (u32)addr, ggtt->gsm[i]); + } +} + +int xe_ggtt_insert_special_node_locked(struct xe_ggtt *ggtt, struct drm_mm_node *node, + u32 size, u32 align, u32 mm_flags) +{ + return drm_mm_insert_node_generic(&ggtt->mm, node, size, align, 0, + mm_flags); +} + +int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node, + u32 size, u32 align) +{ + int ret; + + mutex_lock(&ggtt->lock); + ret = xe_ggtt_insert_special_node_locked(ggtt, node, size, + align, DRM_MM_INSERT_HIGH); + mutex_unlock(&ggtt->lock); + + return ret; +} + +void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) +{ + u64 start = bo->ggtt_node.start; + u64 offset, pte; + + for (offset = 0; offset < bo->size; offset += GEN8_PAGE_SIZE) { + pte = xe_ggtt_pte_encode(bo, offset); + xe_ggtt_set_pte(ggtt, start + offset, pte); + } + + xe_ggtt_invalidate(ggtt->gt); +} + +int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) +{ + int err; + + if (XE_WARN_ON(bo->ggtt_node.size)) { + /* Someone's already inserted this BO in the GGTT */ + XE_BUG_ON(bo->ggtt_node.size != bo->size); + return 0; + } + + err = xe_bo_validate(bo, NULL, false); + if (err) + return err; + + mutex_lock(&ggtt->lock); + err = drm_mm_insert_node(&ggtt->mm, &bo->ggtt_node, bo->size); + if (!err) + xe_ggtt_map_bo(ggtt, bo); + mutex_unlock(&ggtt->lock); + + return 0; +} + +void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node) +{ + mutex_lock(&ggtt->lock); + + xe_ggtt_clear(ggtt, node->start, node->size); + drm_mm_remove_node(node); + node->size = 0; + + xe_ggtt_invalidate(ggtt->gt); + + mutex_unlock(&ggtt->lock); +} + +void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo) +{ + if (XE_WARN_ON(!bo->ggtt_node.size)) + return; + + /* This BO is not currently in the GGTT */ + XE_BUG_ON(bo->ggtt_node.size != bo->size); + + xe_ggtt_remove_node(ggtt, &bo->ggtt_node); +} diff --git a/drivers/gpu/drm/xe/xe_ggtt.h b/drivers/gpu/drm/xe/xe_ggtt.h new file mode 100644 index 000000000000..289c6852ad1a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ggtt.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_GGTT_H_ +#define _XE_GGTT_H_ + +#include "xe_ggtt_types.h" + +u64 xe_ggtt_pte_encode(struct xe_bo *bo, u64 bo_offset); +void xe_ggtt_set_pte(struct xe_ggtt *ggtt, u64 addr, u64 pte); +void xe_ggtt_invalidate(struct xe_gt *gt); +int xe_ggtt_init_noalloc(struct xe_gt *gt, struct xe_ggtt *ggtt); +int xe_ggtt_init(struct xe_gt *gt, struct xe_ggtt *ggtt); +void xe_ggtt_printk(struct xe_ggtt *ggtt, const char *prefix); + +int xe_ggtt_insert_special_node(struct xe_ggtt *ggtt, struct drm_mm_node *node, + u32 size, u32 align); +int xe_ggtt_insert_special_node_locked(struct xe_ggtt *ggtt, + struct drm_mm_node *node, + u32 size, u32 align, u32 mm_flags); +void xe_ggtt_remove_node(struct xe_ggtt *ggtt, struct drm_mm_node *node); +void xe_ggtt_map_bo(struct xe_ggtt *ggtt, struct xe_bo *bo); +int xe_ggtt_insert_bo(struct xe_ggtt *ggtt, struct xe_bo *bo); +void xe_ggtt_remove_bo(struct xe_ggtt *ggtt, struct xe_bo *bo); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ggtt_types.h b/drivers/gpu/drm/xe/xe_ggtt_types.h new file mode 100644 index 000000000000..e04193001763 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ggtt_types.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GGTT_TYPES_H_ +#define _XE_GGTT_TYPES_H_ + +#include + +struct xe_bo; +struct xe_gt; + +struct xe_ggtt { + struct xe_gt *gt; + + u64 size; + + struct xe_bo *scratch; + + struct mutex lock; + + u64 __iomem *gsm; + + struct drm_mm mm; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.c b/drivers/gpu/drm/xe/xe_gpu_scheduler.c new file mode 100644 index 000000000000..e4ad1d6ce1d5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.c @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_gpu_scheduler.h" + +static void xe_sched_process_msg_queue(struct xe_gpu_scheduler *sched) +{ + if (!READ_ONCE(sched->base.pause_submit)) + queue_work(sched->base.submit_wq, &sched->work_process_msg); +} + +static void xe_sched_process_msg_queue_if_ready(struct xe_gpu_scheduler *sched) +{ + struct xe_sched_msg *msg; + + spin_lock(&sched->base.job_list_lock); + msg = list_first_entry_or_null(&sched->msgs, struct xe_sched_msg, link); + if (msg) + xe_sched_process_msg_queue(sched); + spin_unlock(&sched->base.job_list_lock); +} + +static struct xe_sched_msg * +xe_sched_get_msg(struct xe_gpu_scheduler *sched) +{ + struct xe_sched_msg *msg; + + spin_lock(&sched->base.job_list_lock); + msg = list_first_entry_or_null(&sched->msgs, + struct xe_sched_msg, link); + if (msg) + list_del(&msg->link); + spin_unlock(&sched->base.job_list_lock); + + return msg; +} + +static void xe_sched_process_msg_work(struct work_struct *w) +{ + struct xe_gpu_scheduler *sched = + container_of(w, struct xe_gpu_scheduler, work_process_msg); + struct xe_sched_msg *msg; + + if (READ_ONCE(sched->base.pause_submit)) + return; + + msg = xe_sched_get_msg(sched); + if (msg) { + sched->ops->process_msg(msg); + + xe_sched_process_msg_queue_if_ready(sched); + } +} + +int xe_sched_init(struct xe_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, + const struct xe_sched_backend_ops *xe_ops, + struct workqueue_struct *submit_wq, + uint32_t hw_submission, unsigned hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, + struct device *dev) +{ + sched->ops = xe_ops; + INIT_LIST_HEAD(&sched->msgs); + INIT_WORK(&sched->work_process_msg, xe_sched_process_msg_work); + + return drm_sched_init(&sched->base, ops, submit_wq, 1, hw_submission, + hang_limit, timeout, timeout_wq, score, name, + dev); +} + +void xe_sched_fini(struct xe_gpu_scheduler *sched) +{ + xe_sched_submission_stop(sched); + drm_sched_fini(&sched->base); +} + +void xe_sched_submission_start(struct xe_gpu_scheduler *sched) +{ + drm_sched_wqueue_start(&sched->base); + queue_work(sched->base.submit_wq, &sched->work_process_msg); +} + +void xe_sched_submission_stop(struct xe_gpu_scheduler *sched) +{ + drm_sched_wqueue_stop(&sched->base); + cancel_work_sync(&sched->work_process_msg); +} + +void xe_sched_add_msg(struct xe_gpu_scheduler *sched, + struct xe_sched_msg *msg) +{ + spin_lock(&sched->base.job_list_lock); + list_add_tail(&msg->link, &sched->msgs); + spin_unlock(&sched->base.job_list_lock); + + xe_sched_process_msg_queue(sched); +} diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.h b/drivers/gpu/drm/xe/xe_gpu_scheduler.h new file mode 100644 index 000000000000..10c6bb9c9386 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GPU_SCHEDULER_H_ +#define _XE_GPU_SCHEDULER_H_ + +#include "xe_gpu_scheduler_types.h" +#include "xe_sched_job_types.h" + +int xe_sched_init(struct xe_gpu_scheduler *sched, + const struct drm_sched_backend_ops *ops, + const struct xe_sched_backend_ops *xe_ops, + struct workqueue_struct *submit_wq, + uint32_t hw_submission, unsigned hang_limit, + long timeout, struct workqueue_struct *timeout_wq, + atomic_t *score, const char *name, + struct device *dev); +void xe_sched_fini(struct xe_gpu_scheduler *sched); + +void xe_sched_submission_start(struct xe_gpu_scheduler *sched); +void xe_sched_submission_stop(struct xe_gpu_scheduler *sched); + +void xe_sched_add_msg(struct xe_gpu_scheduler *sched, + struct xe_sched_msg *msg); + +static inline void xe_sched_stop(struct xe_gpu_scheduler *sched) +{ + drm_sched_stop(&sched->base, NULL); +} + +static inline void xe_sched_tdr_queue_imm(struct xe_gpu_scheduler *sched) +{ + drm_sched_tdr_queue_imm(&sched->base); +} + +static inline void xe_sched_resubmit_jobs(struct xe_gpu_scheduler *sched) +{ + drm_sched_resubmit_jobs(&sched->base); +} + +static inline bool +xe_sched_invalidate_job(struct xe_sched_job *job, int threshold) +{ + return drm_sched_invalidate_job(&job->drm, threshold); +} + +static inline void xe_sched_add_pending_job(struct xe_gpu_scheduler *sched, + struct xe_sched_job *job) +{ + list_add(&job->drm.list, &sched->base.pending_list); +} + +static inline +struct xe_sched_job *xe_sched_first_pending_job(struct xe_gpu_scheduler *sched) +{ + return list_first_entry_or_null(&sched->base.pending_list, + struct xe_sched_job, drm.list); +} + +static inline int +xe_sched_entity_init(struct xe_sched_entity *entity, + struct xe_gpu_scheduler *sched) +{ + return drm_sched_entity_init(entity, 0, + (struct drm_gpu_scheduler **)&sched, + 1, NULL); +} + +#define xe_sched_entity_fini drm_sched_entity_fini + +#endif diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler_types.h b/drivers/gpu/drm/xe/xe_gpu_scheduler_types.h new file mode 100644 index 000000000000..6731b13da8bb --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler_types.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_GPU_SCHEDULER_TYPES_H_ +#define _XE_GPU_SCHEDULER_TYPES_H_ + +#include + +/** + * struct xe_sched_msg - an in-band (relative to GPU scheduler run queue) + * message + * + * Generic enough for backend defined messages, backend can expand if needed. + */ +struct xe_sched_msg { + /** @link: list link into the gpu scheduler list of messages */ + struct list_head link; + /** + * @private_data: opaque pointer to message private data (backend defined) + */ + void *private_data; + /** @opcode: opcode of message (backend defined) */ + unsigned int opcode; +}; + +/** + * struct xe_sched_backend_ops - Define the backend operations called by the + * scheduler + */ +struct xe_sched_backend_ops { + /** + * @process_msg: Process a message. Allowed to block, it is this + * function's responsibility to free message if dynamically allocated. + */ + void (*process_msg)(struct xe_sched_msg *msg); +}; + +/** + * struct xe_gpu_scheduler - Xe GPU scheduler + */ +struct xe_gpu_scheduler { + /** @base: DRM GPU scheduler */ + struct drm_gpu_scheduler base; + /** @ops: Xe scheduler ops */ + const struct xe_sched_backend_ops *ops; + /** @msgs: list of messages to be processed in @work_process_msg */ + struct list_head msgs; + /** @work_process_msg: processes messages */ + struct work_struct work_process_msg; +}; + +#define xe_sched_entity drm_sched_entity +#define xe_sched_policy drm_sched_policy + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c new file mode 100644 index 000000000000..5f8fa9d98d5a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -0,0 +1,830 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include + +#include "xe_bb.h" +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_engine.h" +#include "xe_execlist.h" +#include "xe_force_wake.h" +#include "xe_ggtt.h" +#include "xe_gt.h" +#include "xe_gt_clock.h" +#include "xe_gt_mcr.h" +#include "xe_gt_pagefault.h" +#include "xe_gt_sysfs.h" +#include "xe_gt_topology.h" +#include "xe_hw_fence.h" +#include "xe_irq.h" +#include "xe_lrc.h" +#include "xe_map.h" +#include "xe_migrate.h" +#include "xe_mmio.h" +#include "xe_mocs.h" +#include "xe_reg_sr.h" +#include "xe_ring_ops.h" +#include "xe_sa.h" +#include "xe_sched_job.h" +#include "xe_ttm_gtt_mgr.h" +#include "xe_ttm_vram_mgr.h" +#include "xe_tuning.h" +#include "xe_uc.h" +#include "xe_vm.h" +#include "xe_wa.h" +#include "xe_wopcm.h" + +#include "gt/intel_gt_regs.h" + +struct xe_gt *xe_find_full_gt(struct xe_gt *gt) +{ + struct xe_gt *search; + u8 id; + + XE_BUG_ON(!xe_gt_is_media_type(gt)); + + for_each_gt(search, gt_to_xe(gt), id) { + if (search->info.vram_id == gt->info.vram_id) + return search; + } + + XE_BUG_ON("NOT POSSIBLE"); + return NULL; +} + +int xe_gt_alloc(struct xe_device *xe, struct xe_gt *gt) +{ + struct drm_device *drm = &xe->drm; + + XE_BUG_ON(gt->info.type == XE_GT_TYPE_UNINITIALIZED); + + if (!xe_gt_is_media_type(gt)) { + gt->mem.ggtt = drmm_kzalloc(drm, sizeof(*gt->mem.ggtt), + GFP_KERNEL); + if (!gt->mem.ggtt) + return -ENOMEM; + + gt->mem.vram_mgr = drmm_kzalloc(drm, sizeof(*gt->mem.vram_mgr), + GFP_KERNEL); + if (!gt->mem.vram_mgr) + return -ENOMEM; + + gt->mem.gtt_mgr = drmm_kzalloc(drm, sizeof(*gt->mem.gtt_mgr), + GFP_KERNEL); + if (!gt->mem.gtt_mgr) + return -ENOMEM; + } else { + struct xe_gt *full_gt = xe_find_full_gt(gt); + + gt->mem.ggtt = full_gt->mem.ggtt; + gt->mem.vram_mgr = full_gt->mem.vram_mgr; + gt->mem.gtt_mgr = full_gt->mem.gtt_mgr; + } + + gt->ordered_wq = alloc_ordered_workqueue("gt-ordered-wq", 0); + + return 0; +} + +/* FIXME: These should be in a common file */ +#define CHV_PPAT_SNOOP REG_BIT(6) +#define GEN8_PPAT_AGE(x) ((x)<<4) +#define GEN8_PPAT_LLCeLLC (3<<2) +#define GEN8_PPAT_LLCELLC (2<<2) +#define GEN8_PPAT_LLC (1<<2) +#define GEN8_PPAT_WB (3<<0) +#define GEN8_PPAT_WT (2<<0) +#define GEN8_PPAT_WC (1<<0) +#define GEN8_PPAT_UC (0<<0) +#define GEN8_PPAT_ELLC_OVERRIDE (0<<2) +#define GEN8_PPAT(i, x) ((u64)(x) << ((i) * 8)) +#define GEN12_PPAT_CLOS(x) ((x)<<2) + +static void tgl_setup_private_ppat(struct xe_gt *gt) +{ + /* TGL doesn't support LLC or AGE settings */ + xe_mmio_write32(gt, GEN12_PAT_INDEX(0).reg, GEN8_PPAT_WB); + xe_mmio_write32(gt, GEN12_PAT_INDEX(1).reg, GEN8_PPAT_WC); + xe_mmio_write32(gt, GEN12_PAT_INDEX(2).reg, GEN8_PPAT_WT); + xe_mmio_write32(gt, GEN12_PAT_INDEX(3).reg, GEN8_PPAT_UC); + xe_mmio_write32(gt, GEN12_PAT_INDEX(4).reg, GEN8_PPAT_WB); + xe_mmio_write32(gt, GEN12_PAT_INDEX(5).reg, GEN8_PPAT_WB); + xe_mmio_write32(gt, GEN12_PAT_INDEX(6).reg, GEN8_PPAT_WB); + xe_mmio_write32(gt, GEN12_PAT_INDEX(7).reg, GEN8_PPAT_WB); +} + +static void pvc_setup_private_ppat(struct xe_gt *gt) +{ + xe_mmio_write32(gt, GEN12_PAT_INDEX(0).reg, GEN8_PPAT_UC); + xe_mmio_write32(gt, GEN12_PAT_INDEX(1).reg, GEN8_PPAT_WC); + xe_mmio_write32(gt, GEN12_PAT_INDEX(2).reg, GEN8_PPAT_WT); + xe_mmio_write32(gt, GEN12_PAT_INDEX(3).reg, GEN8_PPAT_WB); + xe_mmio_write32(gt, GEN12_PAT_INDEX(4).reg, + GEN12_PPAT_CLOS(1) | GEN8_PPAT_WT); + xe_mmio_write32(gt, GEN12_PAT_INDEX(5).reg, + GEN12_PPAT_CLOS(1) | GEN8_PPAT_WB); + xe_mmio_write32(gt, GEN12_PAT_INDEX(6).reg, + GEN12_PPAT_CLOS(2) | GEN8_PPAT_WT); + xe_mmio_write32(gt, GEN12_PAT_INDEX(7).reg, + GEN12_PPAT_CLOS(2) | GEN8_PPAT_WB); +} + +#define MTL_PPAT_L4_CACHE_POLICY_MASK REG_GENMASK(3, 2) +#define MTL_PAT_INDEX_COH_MODE_MASK REG_GENMASK(1, 0) +#define MTL_PPAT_3_UC REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 3) +#define MTL_PPAT_1_WT REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 1) +#define MTL_PPAT_0_WB REG_FIELD_PREP(MTL_PPAT_L4_CACHE_POLICY_MASK, 0) +#define MTL_3_COH_2W REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 3) +#define MTL_2_COH_1W REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 2) +#define MTL_0_COH_NON REG_FIELD_PREP(MTL_PAT_INDEX_COH_MODE_MASK, 0) + +static void mtl_setup_private_ppat(struct xe_gt *gt) +{ + xe_mmio_write32(gt, GEN12_PAT_INDEX(0).reg, MTL_PPAT_0_WB); + xe_mmio_write32(gt, GEN12_PAT_INDEX(1).reg, + MTL_PPAT_1_WT | MTL_2_COH_1W); + xe_mmio_write32(gt, GEN12_PAT_INDEX(2).reg, + MTL_PPAT_3_UC | MTL_2_COH_1W); + xe_mmio_write32(gt, GEN12_PAT_INDEX(3).reg, + MTL_PPAT_0_WB | MTL_2_COH_1W); + xe_mmio_write32(gt, GEN12_PAT_INDEX(4).reg, + MTL_PPAT_0_WB | MTL_3_COH_2W); +} + +static void setup_private_ppat(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (xe->info.platform == XE_METEORLAKE) + mtl_setup_private_ppat(gt); + else if (xe->info.platform == XE_PVC) + pvc_setup_private_ppat(gt); + else + tgl_setup_private_ppat(gt); +} + +static int gt_ttm_mgr_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + struct sysinfo si; + u64 gtt_size; + + si_meminfo(&si); + gtt_size = (u64)si.totalram * si.mem_unit * 3/4; + + if (gt->mem.vram.size) { + err = xe_ttm_vram_mgr_init(gt, gt->mem.vram_mgr); + if (err) + return err; + gtt_size = min(max((XE_DEFAULT_GTT_SIZE_MB << 20), + gt->mem.vram.size), + gtt_size); + xe->info.mem_region_mask |= BIT(gt->info.vram_id) << 1; + } + + err = xe_ttm_gtt_mgr_init(gt, gt->mem.gtt_mgr, gtt_size); + if (err) + return err; + + return 0; +} + +static void gt_fini(struct drm_device *drm, void *arg) +{ + struct xe_gt *gt = arg; + int i; + + destroy_workqueue(gt->ordered_wq); + + for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) + xe_hw_fence_irq_finish(>->fence_irq[i]); +} + +static void gt_reset_worker(struct work_struct *w); + +int emit_nop_job(struct xe_gt *gt, struct xe_engine *e) +{ + struct xe_sched_job *job; + struct xe_bb *bb; + struct dma_fence *fence; + u64 batch_ofs; + long timeout; + + bb = xe_bb_new(gt, 4, false); + if (IS_ERR(bb)) + return PTR_ERR(bb); + + batch_ofs = xe_bo_ggtt_addr(gt->kernel_bb_pool.bo); + job = xe_bb_create_wa_job(e, bb, batch_ofs); + if (IS_ERR(job)) { + xe_bb_free(bb, NULL); + return PTR_ERR(bb); + } + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + timeout = dma_fence_wait_timeout(fence, false, HZ); + dma_fence_put(fence); + xe_bb_free(bb, NULL); + if (timeout < 0) + return timeout; + else if (!timeout) + return -ETIME; + + return 0; +} + +int emit_wa_job(struct xe_gt *gt, struct xe_engine *e) +{ + struct xe_reg_sr *sr = &e->hwe->reg_lrc; + struct xe_reg_sr_entry *entry; + unsigned long reg; + struct xe_sched_job *job; + struct xe_bb *bb; + struct dma_fence *fence; + u64 batch_ofs; + long timeout; + int count = 0; + + bb = xe_bb_new(gt, SZ_4K, false); /* Just pick a large BB size */ + if (IS_ERR(bb)) + return PTR_ERR(bb); + + xa_for_each(&sr->xa, reg, entry) + ++count; + + if (count) { + bb->cs[bb->len++] = MI_LOAD_REGISTER_IMM(count); + xa_for_each(&sr->xa, reg, entry) { + bb->cs[bb->len++] = reg; + bb->cs[bb->len++] = entry->set_bits; + } + } + bb->cs[bb->len++] = MI_NOOP; + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + + batch_ofs = xe_bo_ggtt_addr(gt->kernel_bb_pool.bo); + job = xe_bb_create_wa_job(e, bb, batch_ofs); + if (IS_ERR(job)) { + xe_bb_free(bb, NULL); + return PTR_ERR(bb); + } + + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + timeout = dma_fence_wait_timeout(fence, false, HZ); + dma_fence_put(fence); + xe_bb_free(bb, NULL); + if (timeout < 0) + return timeout; + else if (!timeout) + return -ETIME; + + return 0; +} + +int xe_gt_record_default_lrcs(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + int err = 0; + + for_each_hw_engine(hwe, gt, id) { + struct xe_engine *e, *nop_e; + struct xe_vm *vm; + void *default_lrc; + + if (gt->default_lrc[hwe->class]) + continue; + + xe_reg_sr_init(&hwe->reg_lrc, "LRC", xe); + xe_wa_process_lrc(hwe); + + default_lrc = drmm_kzalloc(&xe->drm, + xe_lrc_size(xe, hwe->class), + GFP_KERNEL); + if (!default_lrc) + return -ENOMEM; + + vm = xe_migrate_get_vm(gt->migrate); + e = xe_engine_create(xe, vm, BIT(hwe->logical_instance), 1, + hwe, ENGINE_FLAG_WA); + if (IS_ERR(e)) { + err = PTR_ERR(e); + goto put_vm; + } + + /* Prime golden LRC with known good state */ + err = emit_wa_job(gt, e); + if (err) + goto put_engine; + + nop_e = xe_engine_create(xe, vm, BIT(hwe->logical_instance), + 1, hwe, ENGINE_FLAG_WA); + if (IS_ERR(nop_e)) { + err = PTR_ERR(nop_e); + goto put_engine; + } + + /* Switch to different LRC */ + err = emit_nop_job(gt, nop_e); + if (err) + goto put_nop_e; + + /* Reload golden LRC to record the effect of any indirect W/A */ + err = emit_nop_job(gt, e); + if (err) + goto put_nop_e; + + xe_map_memcpy_from(xe, default_lrc, + &e->lrc[0].bo->vmap, + xe_lrc_pphwsp_offset(&e->lrc[0]), + xe_lrc_size(xe, hwe->class)); + + gt->default_lrc[hwe->class] = default_lrc; +put_nop_e: + xe_engine_put(nop_e); +put_engine: + xe_engine_put(e); +put_vm: + xe_vm_put(vm); + if (err) + break; + } + + return err; +} + +int xe_gt_init_early(struct xe_gt *gt) +{ + int err; + + xe_force_wake_init_gt(gt, gt_to_fw(gt)); + + err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (err) + return err; + + xe_gt_topology_init(gt); + xe_gt_mcr_init(gt); + + err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); + if (err) + return err; + + xe_reg_sr_init(>->reg_sr, "GT", gt_to_xe(gt)); + xe_wa_process_gt(gt); + xe_tuning_process_gt(gt); + + return 0; +} + +/** + * xe_gt_init_noalloc - Init GT up to the point where allocations can happen. + * @gt: The GT to initialize. + * + * This function prepares the GT to allow memory allocations to VRAM, but is not + * allowed to allocate memory itself. This state is useful for display readout, + * because the inherited display framebuffer will otherwise be overwritten as it + * is usually put at the start of VRAM. + * + * Returns: 0 on success, negative error code on error. + */ +int xe_gt_init_noalloc(struct xe_gt *gt) +{ + int err, err2; + + if (xe_gt_is_media_type(gt)) + return 0; + + xe_device_mem_access_get(gt_to_xe(gt)); + err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (err) + goto err; + + err = gt_ttm_mgr_init(gt); + if (err) + goto err_force_wake; + + err = xe_ggtt_init_noalloc(gt, gt->mem.ggtt); + +err_force_wake: + err2 = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); + XE_WARN_ON(err2); + xe_device_mem_access_put(gt_to_xe(gt)); +err: + return err; +} + +static int gt_fw_domain_init(struct xe_gt *gt) +{ + int err, i; + + xe_device_mem_access_get(gt_to_xe(gt)); + err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (err) + goto err_hw_fence_irq; + + if (!xe_gt_is_media_type(gt)) { + err = xe_ggtt_init(gt, gt->mem.ggtt); + if (err) + goto err_force_wake; + } + + /* Allow driver to load if uC init fails (likely missing firmware) */ + err = xe_uc_init(>->uc); + XE_WARN_ON(err); + + err = xe_uc_init_hwconfig(>->uc); + if (err) + goto err_force_wake; + + /* Enables per hw engine IRQs */ + xe_gt_irq_postinstall(gt); + + /* Rerun MCR init as we now have hw engine list */ + xe_gt_mcr_init(gt); + + err = xe_hw_engines_init_early(gt); + if (err) + goto err_force_wake; + + err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); + XE_WARN_ON(err); + xe_device_mem_access_put(gt_to_xe(gt)); + + return 0; + +err_force_wake: + xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); +err_hw_fence_irq: + for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) + xe_hw_fence_irq_finish(>->fence_irq[i]); + xe_device_mem_access_put(gt_to_xe(gt)); + + return err; +} + +static int all_fw_domain_init(struct xe_gt *gt) +{ + int err, i; + + xe_device_mem_access_get(gt_to_xe(gt)); + err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (err) + goto err_hw_fence_irq; + + setup_private_ppat(gt); + + xe_reg_sr_apply_mmio(>->reg_sr, gt); + + err = xe_gt_clock_init(gt); + if (err) + goto err_force_wake; + + xe_mocs_init(gt); + err = xe_execlist_init(gt); + if (err) + goto err_force_wake; + + err = xe_hw_engines_init(gt); + if (err) + goto err_force_wake; + + err = xe_uc_init_post_hwconfig(>->uc); + if (err) + goto err_force_wake; + + /* + * FIXME: This should be ok as SA should only be used by gt->migrate and + * vm->gt->migrate and both should be pointing to a non-media GT. But to + * realy safe, convert gt->kernel_bb_pool to a pointer and point a media + * GT to the kernel_bb_pool on a real tile. + */ + if (!xe_gt_is_media_type(gt)) { + err = xe_sa_bo_manager_init(gt, >->kernel_bb_pool, SZ_1M, 16); + if (err) + goto err_force_wake; + + /* + * USM has its only SA pool to non-block behind user operations + */ + if (gt_to_xe(gt)->info.supports_usm) { + err = xe_sa_bo_manager_init(gt, >->usm.bb_pool, + SZ_1M, 16); + if (err) + goto err_force_wake; + } + } + + if (!xe_gt_is_media_type(gt)) { + gt->migrate = xe_migrate_init(gt); + if (IS_ERR(gt->migrate)) + goto err_force_wake; + } else { + gt->migrate = xe_find_full_gt(gt)->migrate; + } + + err = xe_uc_init_hw(>->uc); + if (err) + goto err_force_wake; + + err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); + XE_WARN_ON(err); + xe_device_mem_access_put(gt_to_xe(gt)); + + return 0; + +err_force_wake: + xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); +err_hw_fence_irq: + for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) + xe_hw_fence_irq_finish(>->fence_irq[i]); + xe_device_mem_access_put(gt_to_xe(gt)); + + return err; +} + +int xe_gt_init(struct xe_gt *gt) +{ + int err; + int i; + + INIT_WORK(>->reset.worker, gt_reset_worker); + + for (i = 0; i < XE_ENGINE_CLASS_MAX; ++i) { + gt->ring_ops[i] = xe_ring_ops_get(gt, i); + xe_hw_fence_irq_init(>->fence_irq[i]); + } + + err = xe_gt_pagefault_init(gt); + if (err) + return err; + + xe_gt_sysfs_init(gt); + + err = gt_fw_domain_init(gt); + if (err) + return err; + + xe_force_wake_init_engines(gt, gt_to_fw(gt)); + + err = all_fw_domain_init(gt); + if (err) + return err; + + xe_force_wake_prune(gt, gt_to_fw(gt)); + + err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_fini, gt); + if (err) + return err; + + return 0; +} + +int do_gt_reset(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + xe_mmio_write32(gt, GEN6_GDRST.reg, GEN11_GRDOM_FULL); + err = xe_mmio_wait32(gt, GEN6_GDRST.reg, 0, GEN11_GRDOM_FULL, 5); + if (err) + drm_err(&xe->drm, + "GT reset failed to clear GEN11_GRDOM_FULL\n"); + + return err; +} + +static int do_gt_restart(struct xe_gt *gt) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + int err; + + setup_private_ppat(gt); + + xe_reg_sr_apply_mmio(>->reg_sr, gt); + + err = xe_wopcm_init(>->uc.wopcm); + if (err) + return err; + + for_each_hw_engine(hwe, gt, id) + xe_hw_engine_enable_ring(hwe); + + err = xe_uc_init_hw(>->uc); + if (err) + return err; + + xe_mocs_init(gt); + err = xe_uc_start(>->uc); + if (err) + return err; + + for_each_hw_engine(hwe, gt, id) { + xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); + xe_reg_sr_apply_whitelist(&hwe->reg_whitelist, + hwe->mmio_base, gt); + } + + return 0; +} + +static int gt_reset(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + /* We only support GT resets with GuC submission */ + if (!xe_device_guc_submission_enabled(gt_to_xe(gt))) + return -ENODEV; + + drm_info(&xe->drm, "GT reset started\n"); + + xe_device_mem_access_get(gt_to_xe(gt)); + err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (err) + goto err_msg; + + xe_uc_stop_prepare(>->uc); + xe_gt_pagefault_reset(gt); + + err = xe_uc_stop(>->uc); + if (err) + goto err_out; + + err = do_gt_reset(gt); + if (err) + goto err_out; + + err = do_gt_restart(gt); + if (err) + goto err_out; + + xe_device_mem_access_put(gt_to_xe(gt)); + err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); + XE_WARN_ON(err); + + drm_info(&xe->drm, "GT reset done\n"); + + return 0; + +err_out: + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); +err_msg: + XE_WARN_ON(xe_uc_start(>->uc)); + xe_device_mem_access_put(gt_to_xe(gt)); + drm_err(&xe->drm, "GT reset failed, err=%d\n", err); + + return err; +} + +static void gt_reset_worker(struct work_struct *w) +{ + struct xe_gt *gt = container_of(w, typeof(*gt), reset.worker); + + gt_reset(gt); +} + +void xe_gt_reset_async(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + drm_info(&xe->drm, "Try GT reset\n"); + + /* Don't do a reset while one is already in flight */ + if (xe_uc_reset_prepare(>->uc)) + return; + + drm_info(&xe->drm, "Doing GT reset\n"); + queue_work(gt->ordered_wq, >->reset.worker); +} + +void xe_gt_suspend_prepare(struct xe_gt *gt) +{ + xe_device_mem_access_get(gt_to_xe(gt)); + XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + + xe_uc_stop_prepare(>->uc); + + xe_device_mem_access_put(gt_to_xe(gt)); + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); +} + +int xe_gt_suspend(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + /* For now suspend/resume is only allowed with GuC */ + if (!xe_device_guc_submission_enabled(gt_to_xe(gt))) + return -ENODEV; + + xe_device_mem_access_get(gt_to_xe(gt)); + err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (err) + goto err_msg; + + err = xe_uc_suspend(>->uc); + if (err) + goto err_force_wake; + + xe_device_mem_access_put(gt_to_xe(gt)); + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + drm_info(&xe->drm, "GT suspended\n"); + + return 0; + +err_force_wake: + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); +err_msg: + xe_device_mem_access_put(gt_to_xe(gt)); + drm_err(&xe->drm, "GT suspend failed: %d\n", err); + + return err; +} + +int xe_gt_resume(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + xe_device_mem_access_get(gt_to_xe(gt)); + err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (err) + goto err_msg; + + err = do_gt_restart(gt); + if (err) + goto err_force_wake; + + xe_device_mem_access_put(gt_to_xe(gt)); + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + drm_info(&xe->drm, "GT resumed\n"); + + return 0; + +err_force_wake: + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); +err_msg: + xe_device_mem_access_put(gt_to_xe(gt)); + drm_err(&xe->drm, "GT resume failed: %d\n", err); + + return err; +} + +void xe_gt_migrate_wait(struct xe_gt *gt) +{ + xe_migrate_wait(gt->migrate); +} + +struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, + enum xe_engine_class class, + u16 instance, bool logical) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) + if (hwe->class == class && + ((!logical && hwe->instance == instance) || + (logical && hwe->logical_instance == instance))) + return hwe; + + return NULL; +} + +struct xe_hw_engine *xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt, + enum xe_engine_class class) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) { + switch (class) { + case XE_ENGINE_CLASS_RENDER: + case XE_ENGINE_CLASS_COMPUTE: + if (hwe->class == XE_ENGINE_CLASS_RENDER || + hwe->class == XE_ENGINE_CLASS_COMPUTE) + return hwe; + break; + default: + if (hwe->class == class) + return hwe; + } + } + + return NULL; +} diff --git a/drivers/gpu/drm/xe/xe_gt.h b/drivers/gpu/drm/xe/xe_gt.h new file mode 100644 index 000000000000..5dc08a993cfe --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_H_ +#define _XE_GT_H_ + +#include + +#include "xe_device_types.h" +#include "xe_hw_engine.h" + +#define for_each_hw_engine(hwe__, gt__, id__) \ + for ((id__) = 0; (id__) < ARRAY_SIZE((gt__)->hw_engines); (id__)++) \ + for_each_if (((hwe__) = (gt__)->hw_engines + (id__)) && \ + xe_hw_engine_is_valid((hwe__))) + +int xe_gt_alloc(struct xe_device *xe, struct xe_gt *gt); +int xe_gt_init_early(struct xe_gt *gt); +int xe_gt_init_noalloc(struct xe_gt *gt); +int xe_gt_init(struct xe_gt *gt); +int xe_gt_record_default_lrcs(struct xe_gt *gt); +void xe_gt_suspend_prepare(struct xe_gt *gt); +int xe_gt_suspend(struct xe_gt *gt); +int xe_gt_resume(struct xe_gt *gt); +void xe_gt_reset_async(struct xe_gt *gt); +void xe_gt_migrate_wait(struct xe_gt *gt); + +struct xe_gt *xe_find_full_gt(struct xe_gt *gt); + +/** + * xe_gt_any_hw_engine_by_reset_domain - scan the list of engines and return the + * first that matches the same reset domain as @class + * @gt: GT structure + * @class: hw engine class to lookup + */ +struct xe_hw_engine * +xe_gt_any_hw_engine_by_reset_domain(struct xe_gt *gt, enum xe_engine_class class); + +struct xe_hw_engine *xe_gt_hw_engine(struct xe_gt *gt, + enum xe_engine_class class, + u16 instance, + bool logical); + +static inline bool xe_gt_is_media_type(struct xe_gt *gt) +{ + return gt->info.type == XE_GT_TYPE_MEDIA; +} + +static inline struct xe_device * gt_to_xe(struct xe_gt *gt) +{ + return gt->xe; +} + +static inline bool xe_gt_is_usm_hwe(struct xe_gt *gt, struct xe_hw_engine *hwe) +{ + struct xe_device *xe = gt_to_xe(gt); + + return xe->info.supports_usm && hwe->class == XE_ENGINE_CLASS_COPY && + hwe->instance == gt->usm.reserved_bcs_instance; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_clock.c b/drivers/gpu/drm/xe/xe_gt_clock.c new file mode 100644 index 000000000000..575433e9718a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_clock.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "i915_reg.h" +#include "gt/intel_gt_regs.h" + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_clock.h" +#include "xe_macros.h" +#include "xe_mmio.h" + +static u32 read_reference_ts_freq(struct xe_gt *gt) +{ + u32 ts_override = xe_mmio_read32(gt, GEN9_TIMESTAMP_OVERRIDE.reg); + u32 base_freq, frac_freq; + + base_freq = ((ts_override & GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_MASK) >> + GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_SHIFT) + 1; + base_freq *= 1000000; + + frac_freq = ((ts_override & + GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK) >> + GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_SHIFT); + frac_freq = 1000000 / (frac_freq + 1); + + return base_freq + frac_freq; +} + +static u32 get_crystal_clock_freq(u32 rpm_config_reg) +{ + const u32 f19_2_mhz = 19200000; + const u32 f24_mhz = 24000000; + const u32 f25_mhz = 25000000; + const u32 f38_4_mhz = 38400000; + u32 crystal_clock = + (rpm_config_reg & GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >> + GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT; + + switch (crystal_clock) { + case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ: + return f24_mhz; + case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ: + return f19_2_mhz; + case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_38_4_MHZ: + return f38_4_mhz; + case GEN11_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_25_MHZ: + return f25_mhz; + default: + XE_BUG_ON("NOT_POSSIBLE"); + return 0; + } +} + +int xe_gt_clock_init(struct xe_gt *gt) +{ + u32 ctc_reg = xe_mmio_read32(gt, CTC_MODE.reg); + u32 freq = 0; + + /* Assuming gen11+ so assert this assumption is correct */ + XE_BUG_ON(GRAPHICS_VER(gt_to_xe(gt)) < 11); + + if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) { + freq = read_reference_ts_freq(gt); + } else { + u32 c0 = xe_mmio_read32(gt, RPM_CONFIG0.reg); + + freq = get_crystal_clock_freq(c0); + + /* + * Now figure out how the command stream's timestamp + * register increments from this frequency (it might + * increment only every few clock cycle). + */ + freq >>= 3 - ((c0 & GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >> + GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT); + } + + gt->info.clock_freq = freq; + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_clock.h b/drivers/gpu/drm/xe/xe_gt_clock.h new file mode 100644 index 000000000000..511923afd224 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_clock.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_CLOCK_H_ +#define _XE_GT_CLOCK_H_ + +struct xe_gt; + +int xe_gt_clock_init(struct xe_gt *gt); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c new file mode 100644 index 000000000000..cd1888784141 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include + +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_debugfs.h" +#include "xe_gt_mcr.h" +#include "xe_gt_pagefault.h" +#include "xe_gt_topology.h" +#include "xe_hw_engine.h" +#include "xe_macros.h" +#include "xe_uc_debugfs.h" + +static struct xe_gt *node_to_gt(struct drm_info_node *node) +{ + return node->info_ent->data; +} + +static int hw_engines(struct seq_file *m, void *data) +{ + struct xe_gt *gt = node_to_gt(m->private); + struct xe_device *xe = gt_to_xe(gt); + struct drm_printer p = drm_seq_file_printer(m); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + int err; + + xe_device_mem_access_get(xe); + err = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (err) { + xe_device_mem_access_put(xe); + return err; + } + + for_each_hw_engine(hwe, gt, id) + xe_hw_engine_print_state(hwe, &p); + + xe_device_mem_access_put(xe); + err = xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (err) + return err; + + return 0; +} + +static int force_reset(struct seq_file *m, void *data) +{ + struct xe_gt *gt = node_to_gt(m->private); + + xe_gt_reset_async(gt); + + return 0; +} + +static int sa_info(struct seq_file *m, void *data) +{ + struct xe_gt *gt = node_to_gt(m->private); + struct drm_printer p = drm_seq_file_printer(m); + + drm_suballoc_dump_debug_info(>->kernel_bb_pool.base, &p, + gt->kernel_bb_pool.gpu_addr); + + return 0; +} + +static int topology(struct seq_file *m, void *data) +{ + struct xe_gt *gt = node_to_gt(m->private); + struct drm_printer p = drm_seq_file_printer(m); + + xe_gt_topology_dump(gt, &p); + + return 0; +} + +static int steering(struct seq_file *m, void *data) +{ + struct xe_gt *gt = node_to_gt(m->private); + struct drm_printer p = drm_seq_file_printer(m); + + xe_gt_mcr_steering_dump(gt, &p); + + return 0; +} + +#ifdef CONFIG_DRM_XE_DEBUG +static int invalidate_tlb(struct seq_file *m, void *data) +{ + struct xe_gt *gt = node_to_gt(m->private); + int seqno; + int ret = 0; + + seqno = xe_gt_tlb_invalidation(gt); + XE_WARN_ON(seqno < 0); + if (seqno > 0) + ret = xe_gt_tlb_invalidation_wait(gt, seqno); + XE_WARN_ON(ret < 0); + + return 0; +} +#endif + +static const struct drm_info_list debugfs_list[] = { + {"hw_engines", hw_engines, 0}, + {"force_reset", force_reset, 0}, + {"sa_info", sa_info, 0}, + {"topology", topology, 0}, + {"steering", steering, 0}, +#ifdef CONFIG_DRM_XE_DEBUG + {"invalidate_tlb", invalidate_tlb, 0}, +#endif +}; + +void xe_gt_debugfs_register(struct xe_gt *gt) +{ + struct drm_minor *minor = gt_to_xe(gt)->drm.primary; + struct dentry *root; + struct drm_info_list *local; + char name[8]; + int i; + + XE_BUG_ON(!minor->debugfs_root); + + sprintf(name, "gt%d", gt->info.id); + root = debugfs_create_dir(name, minor->debugfs_root); + if (IS_ERR(root)) { + XE_WARN_ON("Create GT directory failed"); + return; + } + + /* + * Allocate local copy as we need to pass in the GT to the debugfs + * entry and drm_debugfs_create_files just references the drm_info_list + * passed in (e.g. can't define this on the stack). + */ +#define DEBUGFS_SIZE ARRAY_SIZE(debugfs_list) * sizeof(struct drm_info_list) + local = drmm_kmalloc(>_to_xe(gt)->drm, DEBUGFS_SIZE, GFP_KERNEL); + if (!local) { + XE_WARN_ON("Couldn't allocate memory"); + return; + } + + memcpy(local, debugfs_list, DEBUGFS_SIZE); +#undef DEBUGFS_SIZE + + for (i = 0; i < ARRAY_SIZE(debugfs_list); ++i) + local[i].data = gt; + + drm_debugfs_create_files(local, + ARRAY_SIZE(debugfs_list), + root, minor); + + xe_uc_debugfs_register(>->uc, root); +} diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.h b/drivers/gpu/drm/xe/xe_gt_debugfs.h new file mode 100644 index 000000000000..5a329f118a57 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_DEBUGFS_H_ +#define _XE_GT_DEBUGFS_H_ + +struct xe_gt; + +void xe_gt_debugfs_register(struct xe_gt *gt); + +#endif diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.c b/drivers/gpu/drm/xe/xe_gt_mcr.c new file mode 100644 index 000000000000..b69c0d6c6b2f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_mcr.c @@ -0,0 +1,552 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_gt_topology.h" +#include "xe_gt_types.h" +#include "xe_mmio.h" + +#include "gt/intel_gt_regs.h" + +/** + * DOC: GT Multicast/Replicated (MCR) Register Support + * + * Some GT registers are designed as "multicast" or "replicated" registers: + * multiple instances of the same register share a single MMIO offset. MCR + * registers are generally used when the hardware needs to potentially track + * independent values of a register per hardware unit (e.g., per-subslice, + * per-L3bank, etc.). The specific types of replication that exist vary + * per-platform. + * + * MMIO accesses to MCR registers are controlled according to the settings + * programmed in the platform's MCR_SELECTOR register(s). MMIO writes to MCR + * registers can be done in either a (i.e., a single write updates all + * instances of the register to the same value) or unicast (a write updates only + * one specific instance). Reads of MCR registers always operate in a unicast + * manner regardless of how the multicast/unicast bit is set in MCR_SELECTOR. + * Selection of a specific MCR instance for unicast operations is referred to + * as "steering." + * + * If MCR register operations are steered toward a hardware unit that is + * fused off or currently powered down due to power gating, the MMIO operation + * is "terminated" by the hardware. Terminated read operations will return a + * value of zero and terminated unicast write operations will be silently + * ignored. + */ + +enum { + MCR_OP_READ, + MCR_OP_WRITE +}; + +static const struct xe_mmio_range xelp_l3bank_steering_table[] = { + { 0x00B100, 0x00B3FF }, + {}, +}; + +/* + * Although the bspec lists more "MSLICE" ranges than shown here, some of those + * are of a "GAM" subclass that has special rules and doesn't need to be + * included here. + */ +static const struct xe_mmio_range xehp_mslice_steering_table[] = { + { 0x00DD00, 0x00DDFF }, + { 0x00E900, 0x00FFFF }, /* 0xEA00 - OxEFFF is unused */ + {}, +}; + +static const struct xe_mmio_range xehp_lncf_steering_table[] = { + { 0x00B000, 0x00B0FF }, + { 0x00D880, 0x00D8FF }, + {}, +}; + +/* + * We have several types of MCR registers where steering to (0,0) will always + * provide us with a non-terminated value. We'll stick them all in the same + * table for simplicity. + */ +static const struct xe_mmio_range xehpc_instance0_steering_table[] = { + { 0x004000, 0x004AFF }, /* HALF-BSLICE */ + { 0x008800, 0x00887F }, /* CC */ + { 0x008A80, 0x008AFF }, /* TILEPSMI */ + { 0x00B000, 0x00B0FF }, /* HALF-BSLICE */ + { 0x00B100, 0x00B3FF }, /* L3BANK */ + { 0x00C800, 0x00CFFF }, /* HALF-BSLICE */ + { 0x00D800, 0x00D8FF }, /* HALF-BSLICE */ + { 0x00DD00, 0x00DDFF }, /* BSLICE */ + { 0x00E900, 0x00E9FF }, /* HALF-BSLICE */ + { 0x00EC00, 0x00EEFF }, /* HALF-BSLICE */ + { 0x00F000, 0x00FFFF }, /* HALF-BSLICE */ + { 0x024180, 0x0241FF }, /* HALF-BSLICE */ + {}, +}; + +static const struct xe_mmio_range xelpg_instance0_steering_table[] = { + { 0x000B00, 0x000BFF }, /* SQIDI */ + { 0x001000, 0x001FFF }, /* SQIDI */ + { 0x004000, 0x0048FF }, /* GAM */ + { 0x008700, 0x0087FF }, /* SQIDI */ + { 0x00B000, 0x00B0FF }, /* NODE */ + { 0x00C800, 0x00CFFF }, /* GAM */ + { 0x00D880, 0x00D8FF }, /* NODE */ + { 0x00DD00, 0x00DDFF }, /* OAAL2 */ + {}, +}; + +static const struct xe_mmio_range xelpg_l3bank_steering_table[] = { + { 0x00B100, 0x00B3FF }, + {}, +}; + +static const struct xe_mmio_range xelp_dss_steering_table[] = { + { 0x008150, 0x00815F }, + { 0x009520, 0x00955F }, + { 0x00DE80, 0x00E8FF }, + { 0x024A00, 0x024A7F }, + {}, +}; + +/* DSS steering is used for GSLICE ranges as well */ +static const struct xe_mmio_range xehp_dss_steering_table[] = { + { 0x005200, 0x0052FF }, /* GSLICE */ + { 0x005400, 0x007FFF }, /* GSLICE */ + { 0x008140, 0x00815F }, /* GSLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ + { 0x008D00, 0x008DFF }, /* DSS */ + { 0x0094D0, 0x00955F }, /* GSLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ + { 0x009680, 0x0096FF }, /* DSS */ + { 0x00D800, 0x00D87F }, /* GSLICE */ + { 0x00DC00, 0x00DCFF }, /* GSLICE */ + { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved ) */ + { 0x017000, 0x017FFF }, /* GSLICE */ + { 0x024A00, 0x024A7F }, /* DSS */ + {}, +}; + +/* DSS steering is used for COMPUTE ranges as well */ +static const struct xe_mmio_range xehpc_dss_steering_table[] = { + { 0x008140, 0x00817F }, /* COMPUTE (0x8140-0x814F & 0x8160-0x817F), DSS (0x8150-0x815F) */ + { 0x0094D0, 0x00955F }, /* COMPUTE (0x94D0-0x951F), DSS (0x9520-0x955F) */ + { 0x009680, 0x0096FF }, /* DSS */ + { 0x00DC00, 0x00DCFF }, /* COMPUTE */ + { 0x00DE80, 0x00E7FF }, /* DSS (0xDF00-0xE1FF reserved ) */ + {}, +}; + +/* DSS steering is used for SLICE ranges as well */ +static const struct xe_mmio_range xelpg_dss_steering_table[] = { + { 0x005200, 0x0052FF }, /* SLICE */ + { 0x005500, 0x007FFF }, /* SLICE */ + { 0x008140, 0x00815F }, /* SLICE (0x8140-0x814F), DSS (0x8150-0x815F) */ + { 0x0094D0, 0x00955F }, /* SLICE (0x94D0-0x951F), DSS (0x9520-0x955F) */ + { 0x009680, 0x0096FF }, /* DSS */ + { 0x00D800, 0x00D87F }, /* SLICE */ + { 0x00DC00, 0x00DCFF }, /* SLICE */ + { 0x00DE80, 0x00E8FF }, /* DSS (0xE000-0xE0FF reserved) */ + {}, +}; + +static const struct xe_mmio_range xelpmp_oaddrm_steering_table[] = { + { 0x393200, 0x39323F }, + { 0x393400, 0x3934FF }, + {}, +}; + +/* + * DG2 GAM registers are a special case; this table is checked directly in + * xe_gt_mcr_get_nonterminated_steering and is not hooked up via + * gt->steering[]. + */ +static const struct xe_mmio_range dg2_gam_ranges[] = { + { 0x004000, 0x004AFF }, + { 0x00C800, 0x00CFFF }, + { 0x00F000, 0x00FFFF }, + {}, +}; + +static void init_steering_l3bank(struct xe_gt *gt) +{ + if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { + u32 mslice_mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK, + xe_mmio_read32(gt, GEN10_MIRROR_FUSE3.reg)); + u32 bank_mask = REG_FIELD_GET(GT_L3_EXC_MASK, + xe_mmio_read32(gt, XEHP_FUSE4.reg)); + + /* + * Group selects mslice, instance selects bank within mslice. + * Bank 0 is always valid _except_ when the bank mask is 010b. + */ + gt->steering[L3BANK].group_target = __ffs(mslice_mask); + gt->steering[L3BANK].instance_target = + bank_mask & BIT(0) ? 0 : 2; + } else { + u32 fuse = REG_FIELD_GET(GEN10_L3BANK_MASK, + ~xe_mmio_read32(gt, GEN10_MIRROR_FUSE3.reg)); + + gt->steering[L3BANK].group_target = 0; /* unused */ + gt->steering[L3BANK].instance_target = __ffs(fuse); + } +} + +static void init_steering_mslice(struct xe_gt *gt) +{ + u32 mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK, + xe_mmio_read32(gt, GEN10_MIRROR_FUSE3.reg)); + + /* + * mslice registers are valid (not terminated) if either the meml3 + * associated with the mslice is present, or at least one DSS associated + * with the mslice is present. There will always be at least one meml3 + * so we can just use that to find a non-terminated mslice and ignore + * the DSS fusing. + */ + gt->steering[MSLICE].group_target = __ffs(mask); + gt->steering[MSLICE].instance_target = 0; /* unused */ + + /* + * LNCF termination is also based on mslice presence, so we'll set + * it up here. Either LNCF within a non-terminated mslice will work, + * so we just always pick LNCF 0 here. + */ + gt->steering[LNCF].group_target = __ffs(mask) << 1; + gt->steering[LNCF].instance_target = 0; /* unused */ +} + +static void init_steering_dss(struct xe_gt *gt) +{ + unsigned int dss = min(xe_dss_mask_group_ffs(gt->fuse_topo.g_dss_mask, 0, 0), + xe_dss_mask_group_ffs(gt->fuse_topo.c_dss_mask, 0, 0)); + unsigned int dss_per_grp = gt_to_xe(gt)->info.platform == XE_PVC ? 8 : 4; + + gt->steering[DSS].group_target = dss / dss_per_grp; + gt->steering[DSS].instance_target = dss % dss_per_grp; +} + +static void init_steering_oaddrm(struct xe_gt *gt) +{ + /* + * First instance is only terminated if the entire first media slice + * is absent (i.e., no VCS0 or VECS0). + */ + if (gt->info.engine_mask & (XE_HW_ENGINE_VCS0 | XE_HW_ENGINE_VECS0)) + gt->steering[OADDRM].group_target = 0; + else + gt->steering[OADDRM].group_target = 1; + + gt->steering[DSS].instance_target = 0; /* unused */ +} + +static void init_steering_inst0(struct xe_gt *gt) +{ + gt->steering[DSS].group_target = 0; /* unused */ + gt->steering[DSS].instance_target = 0; /* unused */ +} + +static const struct { + const char *name; + void (*init)(struct xe_gt *); +} xe_steering_types[] = { + { "L3BANK", init_steering_l3bank }, + { "MSLICE", init_steering_mslice }, + { "LNCF", NULL }, /* initialized by mslice init */ + { "DSS", init_steering_dss }, + { "OADDRM", init_steering_oaddrm }, + { "INSTANCE 0", init_steering_inst0 }, +}; + +void xe_gt_mcr_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + BUILD_BUG_ON(ARRAY_SIZE(xe_steering_types) != NUM_STEERING_TYPES); + + spin_lock_init(>->mcr_lock); + + if (gt->info.type == XE_GT_TYPE_MEDIA) { + drm_WARN_ON(&xe->drm, MEDIA_VER(xe) < 13); + + gt->steering[OADDRM].ranges = xelpmp_oaddrm_steering_table; + } else if (GRAPHICS_VERx100(xe) >= 1270) { + gt->steering[INSTANCE0].ranges = xelpg_instance0_steering_table; + gt->steering[L3BANK].ranges = xelpg_l3bank_steering_table; + gt->steering[DSS].ranges = xelpg_dss_steering_table; + } else if (xe->info.platform == XE_PVC) { + gt->steering[INSTANCE0].ranges = xehpc_instance0_steering_table; + gt->steering[DSS].ranges = xehpc_dss_steering_table; + } else if (xe->info.platform == XE_DG2) { + gt->steering[MSLICE].ranges = xehp_mslice_steering_table; + gt->steering[LNCF].ranges = xehp_lncf_steering_table; + gt->steering[DSS].ranges = xehp_dss_steering_table; + } else { + gt->steering[L3BANK].ranges = xelp_l3bank_steering_table; + gt->steering[DSS].ranges = xelp_dss_steering_table; + } + + /* Select non-terminated steering target for each type */ + for (int i = 0; i < NUM_STEERING_TYPES; i++) + if (gt->steering[i].ranges && xe_steering_types[i].init) + xe_steering_types[i].init(gt); +} + +/* + * xe_gt_mcr_get_nonterminated_steering - find group/instance values that + * will steer a register to a non-terminated instance + * @gt: GT structure + * @reg: register for which the steering is required + * @group: return variable for group steering + * @instance: return variable for instance steering + * + * This function returns a group/instance pair that is guaranteed to work for + * read steering of the given register. Note that a value will be returned even + * if the register is not replicated and therefore does not actually require + * steering. + * + * Returns true if the caller should steer to the @group/@instance values + * returned. Returns false if the caller need not perform any steering (i.e., + * the DG2 GAM range special case). + */ +static bool xe_gt_mcr_get_nonterminated_steering(struct xe_gt *gt, + i915_mcr_reg_t reg, + u8 *group, u8 *instance) +{ + for (int type = 0; type < NUM_STEERING_TYPES; type++) { + if (!gt->steering[type].ranges) + continue; + + for (int i = 0; gt->steering[type].ranges[i].end > 0; i++) { + if (xe_mmio_in_range(>->steering[type].ranges[i], reg.reg)) { + *group = gt->steering[type].group_target; + *instance = gt->steering[type].instance_target; + return true; + } + } + } + + /* + * All MCR registers should usually be part of one of the steering + * ranges we're tracking. However there's one special case: DG2 + * GAM registers are technically multicast registers, but are special + * in a number of ways: + * - they have their own dedicated steering control register (they + * don't share 0xFDC with other MCR classes) + * - all reads should be directed to instance 1 (unicast reads against + * other instances are not allowed), and instance 1 is already the + * the hardware's default steering target, which we never change + * + * Ultimately this means that we can just treat them as if they were + * unicast registers and all operations will work properly. + */ + for (int i = 0; dg2_gam_ranges[i].end > 0; i++) + if (xe_mmio_in_range(&dg2_gam_ranges[i], reg.reg)) + return false; + + /* + * Not found in a steering table and not a DG2 GAM register? We'll + * just steer to 0/0 as a guess and raise a warning. + */ + drm_WARN(>_to_xe(gt)->drm, true, + "Did not find MCR register %#x in any MCR steering table\n", + reg.reg); + *group = 0; + *instance = 0; + + return true; +} + +#define STEER_SEMAPHORE 0xFD0 + +/* + * Obtain exclusive access to MCR steering. On MTL and beyond we also need + * to synchronize with external clients (e.g., firmware), so a semaphore + * register will also need to be taken. + */ +static void mcr_lock(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int ret; + + spin_lock(>->mcr_lock); + + /* + * Starting with MTL we also need to grab a semaphore register + * to synchronize with external agents (e.g., firmware) that now + * shares the same steering control register. + */ + if (GRAPHICS_VERx100(xe) >= 1270) + ret = wait_for_us(xe_mmio_read32(gt, STEER_SEMAPHORE) == 0x1, 10); + + drm_WARN_ON_ONCE(&xe->drm, ret == -ETIMEDOUT); +} + +static void mcr_unlock(struct xe_gt *gt) { + /* Release hardware semaphore */ + if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) + xe_mmio_write32(gt, STEER_SEMAPHORE, 0x1); + + spin_unlock(>->mcr_lock); +} + +/* + * Access a register with specific MCR steering + * + * Caller needs to make sure the relevant forcewake wells are up. + */ +static u32 rw_with_mcr_steering(struct xe_gt *gt, i915_mcr_reg_t reg, u8 rw_flag, + int group, int instance, u32 value) +{ + u32 steer_reg, steer_val, val = 0; + + lockdep_assert_held(>->mcr_lock); + + if (GRAPHICS_VERx100(gt_to_xe(gt)) >= 1270) { + steer_reg = MTL_MCR_SELECTOR.reg; + steer_val = REG_FIELD_PREP(MTL_MCR_GROUPID, group) | + REG_FIELD_PREP(MTL_MCR_INSTANCEID, instance); + } else { + steer_reg = GEN8_MCR_SELECTOR.reg; + steer_val = REG_FIELD_PREP(GEN11_MCR_SLICE_MASK, group) | + REG_FIELD_PREP(GEN11_MCR_SUBSLICE_MASK, instance); + } + + /* + * Always leave the hardware in multicast mode when doing reads + * (see comment about Wa_22013088509 below) and only change it + * to unicast mode when doing writes of a specific instance. + * + * No need to save old steering reg value. + */ + if (rw_flag == MCR_OP_READ) + steer_val |= GEN11_MCR_MULTICAST; + + xe_mmio_write32(gt, steer_reg, steer_val); + + if (rw_flag == MCR_OP_READ) + val = xe_mmio_read32(gt, reg.reg); + else + xe_mmio_write32(gt, reg.reg, value); + + /* + * If we turned off the multicast bit (during a write) we're required + * to turn it back on before finishing. The group and instance values + * don't matter since they'll be re-programmed on the next MCR + * operation. + */ + if (rw_flag == MCR_OP_WRITE) + xe_mmio_write32(gt, steer_reg, GEN11_MCR_MULTICAST); + + return val; +} + +/** + * xe_gt_mcr_unicast_read_any - reads a non-terminated instance of an MCR register + * @gt: GT structure + * @reg: register to read + * + * Reads a GT MCR register. The read will be steered to a non-terminated + * instance (i.e., one that isn't fused off or powered down by power gating). + * This function assumes the caller is already holding any necessary forcewake + * domains. + * + * Returns the value from a non-terminated instance of @reg. + */ +u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, i915_mcr_reg_t reg) +{ + u8 group, instance; + u32 val; + bool steer; + + steer = xe_gt_mcr_get_nonterminated_steering(gt, reg, &group, &instance); + + if (steer) { + mcr_lock(gt); + val = rw_with_mcr_steering(gt, reg, MCR_OP_READ, + group, instance, 0); + mcr_unlock(gt); + } else { + /* DG2 GAM special case rules; treat as if unicast */ + val = xe_mmio_read32(gt, reg.reg); + } + + return val; +} + +/** + * xe_gt_mcr_unicast_read - read a specific instance of an MCR register + * @gt: GT structure + * @reg: the MCR register to read + * @group: the MCR group + * @instance: the MCR instance + * + * Returns the value read from an MCR register after steering toward a specific + * group/instance. + */ +u32 xe_gt_mcr_unicast_read(struct xe_gt *gt, + i915_mcr_reg_t reg, + int group, int instance) +{ + u32 val; + + mcr_lock(gt); + val = rw_with_mcr_steering(gt, reg, MCR_OP_READ, group, instance, 0); + mcr_unlock(gt); + + return val; +} + +/** + * xe_gt_mcr_unicast_write - write a specific instance of an MCR register + * @gt: GT structure + * @reg: the MCR register to write + * @value: value to write + * @group: the MCR group + * @instance: the MCR instance + * + * Write an MCR register in unicast mode after steering toward a specific + * group/instance. + */ +void xe_gt_mcr_unicast_write(struct xe_gt *gt, i915_mcr_reg_t reg, u32 value, + int group, int instance) +{ + mcr_lock(gt); + rw_with_mcr_steering(gt, reg, MCR_OP_WRITE, group, instance, value); + mcr_unlock(gt); +} + +/** + * xe_gt_mcr_multicast_write - write a value to all instances of an MCR register + * @gt: GT structure + * @reg: the MCR register to write + * @value: value to write + * + * Write an MCR register in multicast mode to update all instances. + */ +void xe_gt_mcr_multicast_write(struct xe_gt *gt, i915_mcr_reg_t reg, u32 value) +{ + /* + * Synchronize with any unicast operations. Once we have exclusive + * access, the MULTICAST bit should already be set, so there's no need + * to touch the steering register. + */ + mcr_lock(gt); + xe_mmio_write32(gt, reg.reg, value); + mcr_unlock(gt); +} + +void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p) +{ + for (int i = 0; i < NUM_STEERING_TYPES; i++) { + if (gt->steering[i].ranges) { + drm_printf(p, "%s steering: group=%#x, instance=%#x\n", + xe_steering_types[i].name, + gt->steering[i].group_target, + gt->steering[i].instance_target); + for (int j = 0; gt->steering[i].ranges[j].end; j++) + drm_printf(p, "\t0x%06x - 0x%06x\n", + gt->steering[i].ranges[j].start, + gt->steering[i].ranges[j].end); + } + } +} diff --git a/drivers/gpu/drm/xe/xe_gt_mcr.h b/drivers/gpu/drm/xe/xe_gt_mcr.h new file mode 100644 index 000000000000..62ec6eb654a0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_mcr.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_MCR_H_ +#define _XE_GT_MCR_H_ + +#include "i915_reg_defs.h" + +struct drm_printer; +struct xe_gt; + +void xe_gt_mcr_init(struct xe_gt *gt); + +u32 xe_gt_mcr_unicast_read(struct xe_gt *gt, i915_mcr_reg_t reg, + int group, int instance); +u32 xe_gt_mcr_unicast_read_any(struct xe_gt *gt, i915_mcr_reg_t reg); + +void xe_gt_mcr_unicast_write(struct xe_gt *gt, i915_mcr_reg_t reg, u32 value, + int group, int instance); +void xe_gt_mcr_multicast_write(struct xe_gt *gt, i915_mcr_reg_t reg, u32 value); + +void xe_gt_mcr_steering_dump(struct xe_gt *gt, struct drm_printer *p); + +#endif /* _XE_GT_MCR_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c new file mode 100644 index 000000000000..7125113b7390 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c @@ -0,0 +1,750 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include +#include + +#include "xe_bo.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_gt_pagefault.h" +#include "xe_migrate.h" +#include "xe_pt.h" +#include "xe_trace.h" +#include "xe_vm.h" + +struct pagefault { + u64 page_addr; + u32 asid; + u16 pdata; + u8 vfid; + u8 access_type; + u8 fault_type; + u8 fault_level; + u8 engine_class; + u8 engine_instance; + u8 fault_unsuccessful; +}; + +enum access_type { + ACCESS_TYPE_READ = 0, + ACCESS_TYPE_WRITE = 1, + ACCESS_TYPE_ATOMIC = 2, + ACCESS_TYPE_RESERVED = 3, +}; + +enum fault_type { + NOT_PRESENT = 0, + WRITE_ACCESS_VIOLATION = 1, + ATOMIC_ACCESS_VIOLATION = 2, +}; + +struct acc { + u64 va_range_base; + u32 asid; + u32 sub_granularity; + u8 granularity; + u8 vfid; + u8 access_type; + u8 engine_class; + u8 engine_instance; +}; + +static struct xe_gt * +guc_to_gt(struct xe_guc *guc) +{ + return container_of(guc, struct xe_gt, uc.guc); +} + +static int send_tlb_invalidation(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 action[] = { + XE_GUC_ACTION_TLB_INVALIDATION, + 0, + XE_GUC_TLB_INVAL_FULL << XE_GUC_TLB_INVAL_TYPE_SHIFT | + XE_GUC_TLB_INVAL_MODE_HEAVY << XE_GUC_TLB_INVAL_MODE_SHIFT | + XE_GUC_TLB_INVAL_FLUSH_CACHE, + }; + int seqno; + int ret; + + /* + * XXX: The seqno algorithm relies on TLB invalidation being processed + * in order which they currently are, if that changes the algorithm will + * need to be updated. + */ + mutex_lock(&guc->ct.lock); + seqno = gt->usm.tlb_invalidation_seqno; + action[1] = seqno; + gt->usm.tlb_invalidation_seqno = (gt->usm.tlb_invalidation_seqno + 1) % + TLB_INVALIDATION_SEQNO_MAX; + if (!gt->usm.tlb_invalidation_seqno) + gt->usm.tlb_invalidation_seqno = 1; + ret = xe_guc_ct_send_locked(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_TLB_INVALIDATE, 1); + if (!ret) + ret = seqno; + mutex_unlock(&guc->ct.lock); + + return ret; +} + +static bool access_is_atomic(enum access_type access_type) +{ + return access_type == ACCESS_TYPE_ATOMIC; +} + +static bool vma_is_valid(struct xe_gt *gt, struct xe_vma *vma) +{ + return BIT(gt->info.id) & vma->gt_present && + !(BIT(gt->info.id) & vma->usm.gt_invalidated); +} + +static bool vma_matches(struct xe_vma *vma, struct xe_vma *lookup) +{ + if (lookup->start > vma->end || lookup->end < vma->start) + return false; + + return true; +} + +static bool only_needs_bo_lock(struct xe_bo *bo) +{ + return bo && bo->vm; +} + +static struct xe_vma *lookup_vma(struct xe_vm *vm, u64 page_addr) +{ + struct xe_vma *vma = NULL, lookup; + + lookup.start = page_addr; + lookup.end = lookup.start + SZ_4K - 1; + if (vm->usm.last_fault_vma) { /* Fast lookup */ + if (vma_matches(vm->usm.last_fault_vma, &lookup)) + vma = vm->usm.last_fault_vma; + } + if (!vma) + vma = xe_vm_find_overlapping_vma(vm, &lookup); + + return vma; +} + +static int handle_pagefault(struct xe_gt *gt, struct pagefault *pf) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_vm *vm; + struct xe_vma *vma = NULL; + struct xe_bo *bo; + LIST_HEAD(objs); + LIST_HEAD(dups); + struct ttm_validate_buffer tv_bo, tv_vm; + struct ww_acquire_ctx ww; + struct dma_fence *fence; + bool write_locked; + int ret = 0; + bool atomic; + + /* ASID to VM */ + mutex_lock(&xe->usm.lock); + vm = xa_load(&xe->usm.asid_to_vm, pf->asid); + if (vm) + xe_vm_get(vm); + mutex_unlock(&xe->usm.lock); + if (!vm || !xe_vm_in_fault_mode(vm)) + return -EINVAL; + +retry_userptr: + /* + * TODO: Avoid exclusive lock if VM doesn't have userptrs, or + * start out read-locked? + */ + down_write(&vm->lock); + write_locked = true; + vma = lookup_vma(vm, pf->page_addr); + if (!vma) { + ret = -EINVAL; + goto unlock_vm; + } + + if (!xe_vma_is_userptr(vma) || !xe_vma_userptr_check_repin(vma)) { + downgrade_write(&vm->lock); + write_locked = false; + } + + trace_xe_vma_pagefault(vma); + + atomic = access_is_atomic(pf->access_type); + + /* Check if VMA is valid */ + if (vma_is_valid(gt, vma) && !atomic) + goto unlock_vm; + + /* TODO: Validate fault */ + + if (xe_vma_is_userptr(vma) && write_locked) { + spin_lock(&vm->userptr.invalidated_lock); + list_del_init(&vma->userptr.invalidate_link); + spin_unlock(&vm->userptr.invalidated_lock); + + ret = xe_vma_userptr_pin_pages(vma); + if (ret) + goto unlock_vm; + + downgrade_write(&vm->lock); + write_locked = false; + } + + /* Lock VM and BOs dma-resv */ + bo = vma->bo; + if (only_needs_bo_lock(bo)) { + /* This path ensures the BO's LRU is updated */ + ret = xe_bo_lock(bo, &ww, xe->info.tile_count, false); + } else { + tv_vm.num_shared = xe->info.tile_count; + tv_vm.bo = xe_vm_ttm_bo(vm); + list_add(&tv_vm.head, &objs); + if (bo) { + tv_bo.bo = &bo->ttm; + tv_bo.num_shared = xe->info.tile_count; + list_add(&tv_bo.head, &objs); + } + ret = ttm_eu_reserve_buffers(&ww, &objs, false, &dups); + } + if (ret) + goto unlock_vm; + + if (atomic) { + if (xe_vma_is_userptr(vma)) { + ret = -EACCES; + goto unlock_dma_resv; + } + + /* Migrate to VRAM, move should invalidate the VMA first */ + ret = xe_bo_migrate(bo, XE_PL_VRAM0 + gt->info.vram_id); + if (ret) + goto unlock_dma_resv; + } else if (bo) { + /* Create backing store if needed */ + ret = xe_bo_validate(bo, vm, true); + if (ret) + goto unlock_dma_resv; + } + + /* Bind VMA only to the GT that has faulted */ + trace_xe_vma_pf_bind(vma); + fence = __xe_pt_bind_vma(gt, vma, xe_gt_migrate_engine(gt), NULL, 0, + vma->gt_present & BIT(gt->info.id)); + if (IS_ERR(fence)) { + ret = PTR_ERR(fence); + goto unlock_dma_resv; + } + + /* + * XXX: Should we drop the lock before waiting? This only helps if doing + * GPU binds which is currently only done if we have to wait for more + * than 10ms on a move. + */ + dma_fence_wait(fence, false); + dma_fence_put(fence); + + if (xe_vma_is_userptr(vma)) + ret = xe_vma_userptr_check_repin(vma); + vma->usm.gt_invalidated &= ~BIT(gt->info.id); + +unlock_dma_resv: + if (only_needs_bo_lock(bo)) + xe_bo_unlock(bo, &ww); + else + ttm_eu_backoff_reservation(&ww, &objs); +unlock_vm: + if (!ret) + vm->usm.last_fault_vma = vma; + if (write_locked) + up_write(&vm->lock); + else + up_read(&vm->lock); + if (ret == -EAGAIN) + goto retry_userptr; + + if (!ret) { + /* + * FIXME: Doing a full TLB invalidation for now, likely could + * defer TLB invalidate + fault response to a callback of fence + * too + */ + ret = send_tlb_invalidation(>->uc.guc); + if (ret >= 0) + ret = 0; + } + xe_vm_put(vm); + + return ret; +} + +static int send_pagefault_reply(struct xe_guc *guc, + struct xe_guc_pagefault_reply *reply) +{ + u32 action[] = { + XE_GUC_ACTION_PAGE_FAULT_RES_DESC, + reply->dw0, + reply->dw1, + }; + + return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); +} + +static void print_pagefault(struct xe_device *xe, struct pagefault *pf) +{ + drm_warn(&xe->drm, "\n\tASID: %d\n" + "\tVFID: %d\n" + "\tPDATA: 0x%04x\n" + "\tFaulted Address: 0x%08x%08x\n" + "\tFaultType: %d\n" + "\tAccessType: %d\n" + "\tFaultLevel: %d\n" + "\tEngineClass: %d\n" + "\tEngineInstance: %d\n", + pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr), + lower_32_bits(pf->page_addr), + pf->fault_type, pf->access_type, pf->fault_level, + pf->engine_class, pf->engine_instance); +} + +#define PF_MSG_LEN_DW 4 + +static int get_pagefault(struct pf_queue *pf_queue, struct pagefault *pf) +{ + const struct xe_guc_pagefault_desc *desc; + int ret = 0; + + spin_lock_irq(&pf_queue->lock); + if (pf_queue->head != pf_queue->tail) { + desc = (const struct xe_guc_pagefault_desc *) + (pf_queue->data + pf_queue->head); + + pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0); + pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0); + pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0); + pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) << + PFD_PDATA_HI_SHIFT; + pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0); + pf->asid = FIELD_GET(PFD_ASID, desc->dw1); + pf->vfid = FIELD_GET(PFD_VFID, desc->dw2); + pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2); + pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2); + pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) << + PFD_VIRTUAL_ADDR_HI_SHIFT; + pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) << + PFD_VIRTUAL_ADDR_LO_SHIFT; + + pf_queue->head = (pf_queue->head + PF_MSG_LEN_DW) % + PF_QUEUE_NUM_DW; + } else { + ret = -1; + } + spin_unlock_irq(&pf_queue->lock); + + return ret; +} + +static bool pf_queue_full(struct pf_queue *pf_queue) +{ + lockdep_assert_held(&pf_queue->lock); + + return CIRC_SPACE(pf_queue->tail, pf_queue->head, PF_QUEUE_NUM_DW) <= + PF_MSG_LEN_DW; +} + +int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct pf_queue *pf_queue; + unsigned long flags; + u32 asid; + bool full; + + if (unlikely(len != PF_MSG_LEN_DW)) + return -EPROTO; + + asid = FIELD_GET(PFD_ASID, msg[1]); + pf_queue = >->usm.pf_queue[asid % NUM_PF_QUEUE]; + + spin_lock_irqsave(&pf_queue->lock, flags); + full = pf_queue_full(pf_queue); + if (!full) { + memcpy(pf_queue->data + pf_queue->tail, msg, len * sizeof(u32)); + pf_queue->tail = (pf_queue->tail + len) % PF_QUEUE_NUM_DW; + queue_work(gt->usm.pf_wq, &pf_queue->worker); + } else { + XE_WARN_ON("PF Queue full, shouldn't be possible"); + } + spin_unlock_irqrestore(&pf_queue->lock, flags); + + return full ? -ENOSPC : 0; +} + +static void pf_queue_work_func(struct work_struct *w) +{ + struct pf_queue *pf_queue = container_of(w, struct pf_queue, worker); + struct xe_gt *gt = pf_queue->gt; + struct xe_device *xe = gt_to_xe(gt); + struct xe_guc_pagefault_reply reply = {}; + struct pagefault pf = {}; + int ret; + + ret = get_pagefault(pf_queue, &pf); + if (ret) + return; + + ret = handle_pagefault(gt, &pf); + if (unlikely(ret)) { + print_pagefault(xe, &pf); + pf.fault_unsuccessful = 1; + drm_warn(&xe->drm, "Fault response: Unsuccessful %d\n", ret); + } + + reply.dw0 = FIELD_PREP(PFR_VALID, 1) | + FIELD_PREP(PFR_SUCCESS, pf.fault_unsuccessful) | + FIELD_PREP(PFR_REPLY, PFR_ACCESS) | + FIELD_PREP(PFR_DESC_TYPE, FAULT_RESPONSE_DESC) | + FIELD_PREP(PFR_ASID, pf.asid); + + reply.dw1 = FIELD_PREP(PFR_VFID, pf.vfid) | + FIELD_PREP(PFR_ENG_INSTANCE, pf.engine_instance) | + FIELD_PREP(PFR_ENG_CLASS, pf.engine_class) | + FIELD_PREP(PFR_PDATA, pf.pdata); + + send_pagefault_reply(>->uc.guc, &reply); +} + +static void acc_queue_work_func(struct work_struct *w); + +int xe_gt_pagefault_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int i; + + if (!xe->info.supports_usm) + return 0; + + gt->usm.tlb_invalidation_seqno = 1; + for (i = 0; i < NUM_PF_QUEUE; ++i) { + gt->usm.pf_queue[i].gt = gt; + spin_lock_init(>->usm.pf_queue[i].lock); + INIT_WORK(>->usm.pf_queue[i].worker, pf_queue_work_func); + } + for (i = 0; i < NUM_ACC_QUEUE; ++i) { + gt->usm.acc_queue[i].gt = gt; + spin_lock_init(>->usm.acc_queue[i].lock); + INIT_WORK(>->usm.acc_queue[i].worker, acc_queue_work_func); + } + + gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue", + WQ_UNBOUND | WQ_HIGHPRI, NUM_PF_QUEUE); + if (!gt->usm.pf_wq) + return -ENOMEM; + + gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue", + WQ_UNBOUND | WQ_HIGHPRI, + NUM_ACC_QUEUE); + if (!gt->usm.acc_wq) + return -ENOMEM; + + return 0; +} + +void xe_gt_pagefault_reset(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + int i; + + if (!xe->info.supports_usm) + return; + + for (i = 0; i < NUM_PF_QUEUE; ++i) { + spin_lock_irq(>->usm.pf_queue[i].lock); + gt->usm.pf_queue[i].head = 0; + gt->usm.pf_queue[i].tail = 0; + spin_unlock_irq(>->usm.pf_queue[i].lock); + } + + for (i = 0; i < NUM_ACC_QUEUE; ++i) { + spin_lock(>->usm.acc_queue[i].lock); + gt->usm.acc_queue[i].head = 0; + gt->usm.acc_queue[i].tail = 0; + spin_unlock(>->usm.acc_queue[i].lock); + } +} + +int xe_gt_tlb_invalidation(struct xe_gt *gt) +{ + return send_tlb_invalidation(>->uc.guc); +} + +static bool tlb_invalidation_seqno_past(struct xe_gt *gt, int seqno) +{ + if (gt->usm.tlb_invalidation_seqno_recv >= seqno) + return true; + + if (seqno - gt->usm.tlb_invalidation_seqno_recv > + (TLB_INVALIDATION_SEQNO_MAX / 2)) + return true; + + return false; +} + +int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_guc *guc = >->uc.guc; + int ret; + + /* + * XXX: See above, this algorithm only works if seqno are always in + * order + */ + ret = wait_event_timeout(guc->ct.wq, + tlb_invalidation_seqno_past(gt, seqno), + HZ / 5); + if (!ret) { + drm_err(&xe->drm, "TLB invalidation time'd out, seqno=%d, recv=%d\n", + seqno, gt->usm.tlb_invalidation_seqno_recv); + return -ETIME; + } + + return 0; +} + +int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + int expected_seqno; + + if (unlikely(len != 1)) + return -EPROTO; + + /* Sanity check on seqno */ + expected_seqno = (gt->usm.tlb_invalidation_seqno_recv + 1) % + TLB_INVALIDATION_SEQNO_MAX; + XE_WARN_ON(expected_seqno != msg[0]); + + gt->usm.tlb_invalidation_seqno_recv = msg[0]; + smp_wmb(); + wake_up_all(&guc->ct.wq); + + return 0; +} + +static int granularity_in_byte(int val) +{ + switch (val) { + case 0: + return SZ_128K; + case 1: + return SZ_2M; + case 2: + return SZ_16M; + case 3: + return SZ_64M; + default: + return 0; + } +} + +static int sub_granularity_in_byte(int val) +{ + return (granularity_in_byte(val) / 32); +} + +static void print_acc(struct xe_device *xe, struct acc *acc) +{ + drm_warn(&xe->drm, "Access counter request:\n" + "\tType: %s\n" + "\tASID: %d\n" + "\tVFID: %d\n" + "\tEngine: %d:%d\n" + "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n" + "\tSub_Granularity Vector: 0x%08x\n" + "\tVA Range base: 0x%016llx\n", + acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL", + acc->asid, acc->vfid, acc->engine_class, acc->engine_instance, + granularity_in_byte(acc->granularity) / SZ_1K, + sub_granularity_in_byte(acc->granularity) / SZ_1K, + acc->sub_granularity, acc->va_range_base); +} + +static struct xe_vma *get_acc_vma(struct xe_vm *vm, struct acc *acc) +{ + u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) * + sub_granularity_in_byte(acc->granularity); + struct xe_vma lookup; + + lookup.start = page_va; + lookup.end = lookup.start + SZ_4K - 1; + + return xe_vm_find_overlapping_vma(vm, &lookup); +} + +static int handle_acc(struct xe_gt *gt, struct acc *acc) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_vm *vm; + struct xe_vma *vma; + struct xe_bo *bo; + LIST_HEAD(objs); + LIST_HEAD(dups); + struct ttm_validate_buffer tv_bo, tv_vm; + struct ww_acquire_ctx ww; + int ret = 0; + + /* We only support ACC_TRIGGER at the moment */ + if (acc->access_type != ACC_TRIGGER) + return -EINVAL; + + /* ASID to VM */ + mutex_lock(&xe->usm.lock); + vm = xa_load(&xe->usm.asid_to_vm, acc->asid); + if (vm) + xe_vm_get(vm); + mutex_unlock(&xe->usm.lock); + if (!vm || !xe_vm_in_fault_mode(vm)) + return -EINVAL; + + down_read(&vm->lock); + + /* Lookup VMA */ + vma = get_acc_vma(vm, acc); + if (!vma) { + ret = -EINVAL; + goto unlock_vm; + } + + trace_xe_vma_acc(vma); + + /* Userptr can't be migrated, nothing to do */ + if (xe_vma_is_userptr(vma)) + goto unlock_vm; + + /* Lock VM and BOs dma-resv */ + bo = vma->bo; + if (only_needs_bo_lock(bo)) { + /* This path ensures the BO's LRU is updated */ + ret = xe_bo_lock(bo, &ww, xe->info.tile_count, false); + } else { + tv_vm.num_shared = xe->info.tile_count; + tv_vm.bo = xe_vm_ttm_bo(vm); + list_add(&tv_vm.head, &objs); + tv_bo.bo = &bo->ttm; + tv_bo.num_shared = xe->info.tile_count; + list_add(&tv_bo.head, &objs); + ret = ttm_eu_reserve_buffers(&ww, &objs, false, &dups); + } + if (ret) + goto unlock_vm; + + /* Migrate to VRAM, move should invalidate the VMA first */ + ret = xe_bo_migrate(bo, XE_PL_VRAM0 + gt->info.vram_id); + + if (only_needs_bo_lock(bo)) + xe_bo_unlock(bo, &ww); + else + ttm_eu_backoff_reservation(&ww, &objs); +unlock_vm: + up_read(&vm->lock); + xe_vm_put(vm); + + return ret; +} + +#define make_u64(hi__, low__) ((u64)(hi__) << 32 | (u64)(low__)) + +static int get_acc(struct acc_queue *acc_queue, struct acc *acc) +{ + const struct xe_guc_acc_desc *desc; + int ret = 0; + + spin_lock(&acc_queue->lock); + if (acc_queue->head != acc_queue->tail) { + desc = (const struct xe_guc_acc_desc *) + (acc_queue->data + acc_queue->head); + + acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2); + acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 | + FIELD_GET(ACC_SUBG_LO, desc->dw0); + acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1); + acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1); + acc->asid = FIELD_GET(ACC_ASID, desc->dw1); + acc->vfid = FIELD_GET(ACC_VFID, desc->dw2); + acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0); + acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI, + desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO); + } else { + ret = -1; + } + spin_unlock(&acc_queue->lock); + + return ret; +} + +static void acc_queue_work_func(struct work_struct *w) +{ + struct acc_queue *acc_queue = container_of(w, struct acc_queue, worker); + struct xe_gt *gt = acc_queue->gt; + struct xe_device *xe = gt_to_xe(gt); + struct acc acc = {}; + int ret; + + ret = get_acc(acc_queue, &acc); + if (ret) + return; + + ret = handle_acc(gt, &acc); + if (unlikely(ret)) { + print_acc(xe, &acc); + drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret); + } +} + +#define ACC_MSG_LEN_DW 4 + +static bool acc_queue_full(struct acc_queue *acc_queue) +{ + lockdep_assert_held(&acc_queue->lock); + + return CIRC_SPACE(acc_queue->tail, acc_queue->head, ACC_QUEUE_NUM_DW) <= + ACC_MSG_LEN_DW; +} + +int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct acc_queue *acc_queue; + u32 asid; + bool full; + + if (unlikely(len != ACC_MSG_LEN_DW)) + return -EPROTO; + + asid = FIELD_GET(ACC_ASID, msg[1]); + acc_queue = >->usm.acc_queue[asid % NUM_ACC_QUEUE]; + + spin_lock(&acc_queue->lock); + full = acc_queue_full(acc_queue); + if (!full) { + memcpy(acc_queue->data + acc_queue->tail, msg, + len * sizeof(u32)); + acc_queue->tail = (acc_queue->tail + len) % ACC_QUEUE_NUM_DW; + queue_work(gt->usm.acc_wq, &acc_queue->worker); + } else { + drm_warn(>_to_xe(gt)->drm, "ACC Queue full, dropping ACC"); + } + spin_unlock(&acc_queue->lock); + + return full ? -ENOSPC : 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.h b/drivers/gpu/drm/xe/xe_gt_pagefault.h new file mode 100644 index 000000000000..35f68027cc9c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_PAGEFAULT_H_ +#define _XE_GT_PAGEFAULT_H_ + +#include + +struct xe_gt; +struct xe_guc; + +int xe_gt_pagefault_init(struct xe_gt *gt); +void xe_gt_pagefault_reset(struct xe_gt *gt); +int xe_gt_tlb_invalidation(struct xe_gt *gt); +int xe_gt_tlb_invalidation_wait(struct xe_gt *gt, int seqno); +int xe_guc_pagefault_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_tlb_invalidation_done_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_access_counter_notify_handler(struct xe_guc *guc, u32 *msg, u32 len); + +#endif /* _XE_GT_PAGEFAULT_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_sysfs.c b/drivers/gpu/drm/xe/xe_gt_sysfs.c new file mode 100644 index 000000000000..2d966d935b8e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sysfs.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include +#include +#include "xe_gt.h" +#include "xe_gt_sysfs.h" + +static void xe_gt_sysfs_kobj_release(struct kobject *kobj) +{ + kfree(kobj); +} + +static struct kobj_type xe_gt_sysfs_kobj_type = { + .release = xe_gt_sysfs_kobj_release, + .sysfs_ops = &kobj_sysfs_ops, +}; + +static void gt_sysfs_fini(struct drm_device *drm, void *arg) +{ + struct xe_gt *gt = arg; + + kobject_put(gt->sysfs); +} + +int xe_gt_sysfs_init(struct xe_gt *gt) +{ + struct device *dev = gt_to_xe(gt)->drm.dev; + struct kobj_gt *kg; + int err; + + kg = kzalloc(sizeof(*kg), GFP_KERNEL); + if (!kg) + return -ENOMEM; + + kobject_init(&kg->base, &xe_gt_sysfs_kobj_type); + kg->gt = gt; + + err = kobject_add(&kg->base, &dev->kobj, "gt%d", gt->info.id); + if (err) { + kobject_put(&kg->base); + return err; + } + + gt->sysfs = &kg->base; + + err = drmm_add_action_or_reset(>_to_xe(gt)->drm, gt_sysfs_fini, gt); + if (err) + return err; + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_gt_sysfs.h b/drivers/gpu/drm/xe/xe_gt_sysfs.h new file mode 100644 index 000000000000..ecbfcc5c7d42 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sysfs.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_SYSFS_H_ +#define _XE_GT_SYSFS_H_ + +#include "xe_gt_sysfs_types.h" + +int xe_gt_sysfs_init(struct xe_gt *gt); + +static inline struct xe_gt * +kobj_to_gt(struct kobject *kobj) +{ + return container_of(kobj, struct kobj_gt, base)->gt; +} + +#endif /* _XE_GT_SYSFS_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_sysfs_types.h b/drivers/gpu/drm/xe/xe_gt_sysfs_types.h new file mode 100644 index 000000000000..d3bc6b83360f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_sysfs_types.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_SYSFS_TYPES_H_ +#define _XE_GT_SYSFS_TYPES_H_ + +#include + +struct xe_gt; + +/** + * struct kobj_gt - A GT's kobject struct that connects the kobject and the GT + * + * When dealing with multiple GTs, this struct helps to understand which GT + * needs to be addressed on a given sysfs call. + */ +struct kobj_gt { + /** @base: The actual kobject */ + struct kobject base; + /** @gt: A pointer to the GT itself */ + struct xe_gt *gt; +}; + +#endif /* _XE_GT_SYSFS_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_gt_topology.c b/drivers/gpu/drm/xe/xe_gt_topology.c new file mode 100644 index 000000000000..8e02e362ba27 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_topology.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_gt.h" +#include "xe_gt_topology.h" +#include "xe_mmio.h" + +#define XE_MAX_DSS_FUSE_BITS (32 * XE_MAX_DSS_FUSE_REGS) +#define XE_MAX_EU_FUSE_BITS (32 * XE_MAX_EU_FUSE_REGS) + +#define XELP_EU_ENABLE 0x9134 /* "_DISABLE" on Xe_LP */ +#define XELP_EU_MASK REG_GENMASK(7, 0) +#define XELP_GT_GEOMETRY_DSS_ENABLE 0x913c +#define XEHP_GT_COMPUTE_DSS_ENABLE 0x9144 +#define XEHPC_GT_COMPUTE_DSS_ENABLE_EXT 0x9148 + +static void +load_dss_mask(struct xe_gt *gt, xe_dss_mask_t mask, int numregs, ...) +{ + va_list argp; + u32 fuse_val[XE_MAX_DSS_FUSE_REGS] = {}; + int i; + + if (drm_WARN_ON(>_to_xe(gt)->drm, numregs > XE_MAX_DSS_FUSE_REGS)) + numregs = XE_MAX_DSS_FUSE_REGS; + + va_start(argp, numregs); + for (i = 0; i < numregs; i++) + fuse_val[i] = xe_mmio_read32(gt, va_arg(argp, u32)); + va_end(argp); + + bitmap_from_arr32(mask, fuse_val, numregs * 32); +} + +static void +load_eu_mask(struct xe_gt *gt, xe_eu_mask_t mask) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 reg = xe_mmio_read32(gt, XELP_EU_ENABLE); + u32 val = 0; + int i; + + BUILD_BUG_ON(XE_MAX_EU_FUSE_REGS > 1); + + /* + * Pre-Xe_HP platforms inverted the bit meaning (disable instead + * of enable). + */ + if (GRAPHICS_VERx100(xe) < 1250) + reg = ~reg & XELP_EU_MASK; + + /* On PVC, one bit = one EU */ + if (GRAPHICS_VERx100(xe) == 1260) { + val = reg; + } else { + /* All other platforms, one bit = 2 EU */ + for (i = 0; i < fls(reg); i++) + if (reg & BIT(i)) + val |= 0x3 << 2 * i; + } + + bitmap_from_arr32(mask, &val, XE_MAX_EU_FUSE_BITS); +} + +void +xe_gt_topology_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct drm_printer p = drm_debug_printer("GT topology"); + int num_geometry_regs, num_compute_regs; + + if (GRAPHICS_VERx100(xe) == 1260) { + num_geometry_regs = 0; + num_compute_regs = 2; + } else if (GRAPHICS_VERx100(xe) >= 1250) { + num_geometry_regs = 1; + num_compute_regs = 1; + } else { + num_geometry_regs = 1; + num_compute_regs = 0; + } + + load_dss_mask(gt, gt->fuse_topo.g_dss_mask, num_geometry_regs, + XELP_GT_GEOMETRY_DSS_ENABLE); + load_dss_mask(gt, gt->fuse_topo.c_dss_mask, num_compute_regs, + XEHP_GT_COMPUTE_DSS_ENABLE, + XEHPC_GT_COMPUTE_DSS_ENABLE_EXT); + load_eu_mask(gt, gt->fuse_topo.eu_mask_per_dss); + + xe_gt_topology_dump(gt, &p); +} + +unsigned int +xe_gt_topology_count_dss(xe_dss_mask_t mask) +{ + return bitmap_weight(mask, XE_MAX_DSS_FUSE_BITS); +} + +u64 +xe_gt_topology_dss_group_mask(xe_dss_mask_t mask, int grpsize) +{ + xe_dss_mask_t per_dss_mask = {}; + u64 grpmask = 0; + + WARN_ON(DIV_ROUND_UP(XE_MAX_DSS_FUSE_BITS, grpsize) > BITS_PER_TYPE(grpmask)); + + bitmap_fill(per_dss_mask, grpsize); + for (int i = 0; !bitmap_empty(mask, XE_MAX_DSS_FUSE_BITS); i++) { + if (bitmap_intersects(mask, per_dss_mask, grpsize)) + grpmask |= BIT(i); + + bitmap_shift_right(mask, mask, grpsize, XE_MAX_DSS_FUSE_BITS); + } + + return grpmask; +} + +void +xe_gt_topology_dump(struct xe_gt *gt, struct drm_printer *p) +{ + drm_printf(p, "dss mask (geometry): %*pb\n", XE_MAX_DSS_FUSE_BITS, + gt->fuse_topo.g_dss_mask); + drm_printf(p, "dss mask (compute): %*pb\n", XE_MAX_DSS_FUSE_BITS, + gt->fuse_topo.c_dss_mask); + + drm_printf(p, "EU mask per DSS: %*pb\n", XE_MAX_EU_FUSE_BITS, + gt->fuse_topo.eu_mask_per_dss); + +} + +/* + * Used to obtain the index of the first DSS. Can start searching from the + * beginning of a specific dss group (e.g., gslice, cslice, etc.) if + * groupsize and groupnum are non-zero. + */ +unsigned int +xe_dss_mask_group_ffs(xe_dss_mask_t mask, int groupsize, int groupnum) +{ + return find_next_bit(mask, XE_MAX_DSS_FUSE_BITS, groupnum * groupsize); +} diff --git a/drivers/gpu/drm/xe/xe_gt_topology.h b/drivers/gpu/drm/xe/xe_gt_topology.h new file mode 100644 index 000000000000..7a0abc64084f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_topology.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __XE_GT_TOPOLOGY_H__ +#define __XE_GT_TOPOLOGY_H__ + +#include "xe_gt_types.h" + +struct drm_printer; + +void xe_gt_topology_init(struct xe_gt *gt); + +void xe_gt_topology_dump(struct xe_gt *gt, struct drm_printer *p); + +unsigned int +xe_dss_mask_group_ffs(xe_dss_mask_t mask, int groupsize, int groupnum); + +#endif /* __XE_GT_TOPOLOGY_H__ */ diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h new file mode 100644 index 000000000000..c80a9215098d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -0,0 +1,320 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GT_TYPES_H_ +#define _XE_GT_TYPES_H_ + +#include "xe_force_wake_types.h" +#include "xe_hw_engine_types.h" +#include "xe_hw_fence_types.h" +#include "xe_reg_sr_types.h" +#include "xe_sa_types.h" +#include "xe_uc_types.h" + +struct xe_engine_ops; +struct xe_ggtt; +struct xe_migrate; +struct xe_ring_ops; +struct xe_ttm_gtt_mgr; +struct xe_ttm_vram_mgr; + +enum xe_gt_type { + XE_GT_TYPE_UNINITIALIZED, + XE_GT_TYPE_MAIN, + XE_GT_TYPE_REMOTE, + XE_GT_TYPE_MEDIA, +}; + +#define XE_MAX_DSS_FUSE_REGS 2 +#define XE_MAX_EU_FUSE_REGS 1 + +typedef unsigned long xe_dss_mask_t[BITS_TO_LONGS(32 * XE_MAX_DSS_FUSE_REGS)]; +typedef unsigned long xe_eu_mask_t[BITS_TO_LONGS(32 * XE_MAX_DSS_FUSE_REGS)]; + +struct xe_mmio_range { + u32 start; + u32 end; +}; + +/* + * The hardware has multiple kinds of multicast register ranges that need + * special register steering (and future platforms are expected to add + * additional types). + * + * During driver startup, we initialize the steering control register to + * direct reads to a slice/subslice that are valid for the 'subslice' class + * of multicast registers. If another type of steering does not have any + * overlap in valid steering targets with 'subslice' style registers, we will + * need to explicitly re-steer reads of registers of the other type. + * + * Only the replication types that may need additional non-default steering + * are listed here. + */ +enum xe_steering_type { + L3BANK, + MSLICE, + LNCF, + DSS, + OADDRM, + + /* + * On some platforms there are multiple types of MCR registers that + * will always return a non-terminated value at instance (0, 0). We'll + * lump those all into a single category to keep things simple. + */ + INSTANCE0, + + NUM_STEERING_TYPES +}; + +/** + * struct xe_gt - Top level struct of a graphics tile + * + * A graphics tile may be a physical split (duplicate pieces of silicon, + * different GGTT + VRAM) or a virtual split (shared GGTT + VRAM). Either way + * this structure encapsulates of everything a GT is (MMIO, VRAM, memory + * management, microcontrols, and a hardware set of engines). + */ +struct xe_gt { + /** @xe: backpointer to XE device */ + struct xe_device *xe; + + /** @info: GT info */ + struct { + /** @type: type of GT */ + enum xe_gt_type type; + /** @id: id of GT */ + u8 id; + /** @vram: id of the VRAM for this GT */ + u8 vram_id; + /** @clock_freq: clock frequency */ + u32 clock_freq; + /** @engine_mask: mask of engines present on GT */ + u64 engine_mask; + } info; + + /** + * @mmio: mmio info for GT, can be subset of the global device mmio + * space + */ + struct { + /** @size: size of MMIO space on GT */ + size_t size; + /** @regs: pointer to MMIO space on GT */ + void *regs; + /** @fw: force wake for GT */ + struct xe_force_wake fw; + /** + * @adj_limit: adjust MMIO address if address is below this + * value + */ + u32 adj_limit; + /** @adj_offset: offect to add to MMIO address when adjusting */ + u32 adj_offset; + } mmio; + + /** + * @reg_sr: table with registers to be restored on GT init/resume/reset + */ + struct xe_reg_sr reg_sr; + + /** + * @mem: memory management info for GT, multiple GTs can point to same + * objects (virtual split) + */ + struct { + /** + * @vram: VRAM info for GT, multiple GTs can point to same info + * (virtual split), can be subset of global device VRAM + */ + struct { + /** @io_start: start address of VRAM */ + resource_size_t io_start; + /** @size: size of VRAM */ + resource_size_t size; + /** @mapping: pointer to VRAM mappable space */ + void *__iomem mapping; + } vram; + /** @vram_mgr: VRAM TTM manager */ + struct xe_ttm_vram_mgr *vram_mgr; + /** @gtt_mr: GTT TTM manager */ + struct xe_ttm_gtt_mgr *gtt_mgr; + /** @ggtt: Global graphics translation table */ + struct xe_ggtt *ggtt; + } mem; + + /** @reset: state for GT resets */ + struct { + /** + * @worker: work so GT resets can done async allowing to reset + * code to safely flush all code paths + */ + struct work_struct worker; + } reset; + + /** @usm: unified shared memory state */ + struct { + /** + * @bb_pool: Pool from which batchbuffers, for USM operations + * (e.g. migrations, fixing page tables), are allocated. + * Dedicated pool needed so USM operations to not get blocked + * behind any user operations which may have resulted in a + * fault. + */ + struct xe_sa_manager bb_pool; + /** + * @reserved_bcs_instance: reserved BCS instance used for USM + * operations (e.g. mmigrations, fixing page tables) + */ + u16 reserved_bcs_instance; + /** + * @tlb_invalidation_seqno: TLB invalidation seqno, protected by + * CT lock + */ +#define TLB_INVALIDATION_SEQNO_MAX 0x100000 + int tlb_invalidation_seqno; + /** + * @tlb_invalidation_seqno_recv: last received TLB invalidation + * seqno, protected by CT lock + */ + int tlb_invalidation_seqno_recv; + /** @pf_wq: page fault work queue, unbound, high priority */ + struct workqueue_struct *pf_wq; + /** @acc_wq: access counter work queue, unbound, high priority */ + struct workqueue_struct *acc_wq; + /** + * @pf_queue: Page fault queue used to sync faults so faults can + * be processed not under the GuC CT lock. The queue is sized so + * it can sync all possible faults (1 per physical engine). + * Multiple queues exists for page faults from different VMs are + * be processed in parallel. + */ + struct pf_queue { + /** @gt: back pointer to GT */ + struct xe_gt *gt; +#define PF_QUEUE_NUM_DW 128 + /** @data: data in the page fault queue */ + u32 data[PF_QUEUE_NUM_DW]; + /** + * @head: head pointer in DWs for page fault queue, + * moved by worker which processes faults. + */ + u16 head; + /** + * @tail: tail pointer in DWs for page fault queue, + * moved by G2H handler. + */ + u16 tail; + /** @lock: protects page fault queue */ + spinlock_t lock; + /** @worker: to process page faults */ + struct work_struct worker; +#define NUM_PF_QUEUE 4 + } pf_queue[NUM_PF_QUEUE]; + /** + * @acc_queue: Same as page fault queue, cannot process access + * counters under CT lock. + */ + struct acc_queue { + /** @gt: back pointer to GT */ + struct xe_gt *gt; +#define ACC_QUEUE_NUM_DW 128 + /** @data: data in the page fault queue */ + u32 data[ACC_QUEUE_NUM_DW]; + /** + * @head: head pointer in DWs for page fault queue, + * moved by worker which processes faults. + */ + u16 head; + /** + * @tail: tail pointer in DWs for page fault queue, + * moved by G2H handler. + */ + u16 tail; + /** @lock: protects page fault queue */ + spinlock_t lock; + /** @worker: to process access counters */ + struct work_struct worker; +#define NUM_ACC_QUEUE 4 + } acc_queue[NUM_ACC_QUEUE]; + } usm; + + /** @ordered_wq: used to serialize GT resets and TDRs */ + struct workqueue_struct *ordered_wq; + + /** @uc: micro controllers on the GT */ + struct xe_uc uc; + + /** @engine_ops: submission backend engine operations */ + const struct xe_engine_ops *engine_ops; + + /** + * @ring_ops: ring operations for this hw engine (1 per engine class) + */ + const struct xe_ring_ops *ring_ops[XE_ENGINE_CLASS_MAX]; + + /** @fence_irq: fence IRQs (1 per engine class) */ + struct xe_hw_fence_irq fence_irq[XE_ENGINE_CLASS_MAX]; + + /** @default_lrc: default LRC state */ + void *default_lrc[XE_ENGINE_CLASS_MAX]; + + /** @hw_engines: hardware engines on the GT */ + struct xe_hw_engine hw_engines[XE_NUM_HW_ENGINES]; + + /** @kernel_bb_pool: Pool from which batchbuffers are allocated */ + struct xe_sa_manager kernel_bb_pool; + + /** @migrate: Migration helper for vram blits and clearing */ + struct xe_migrate *migrate; + + /** @pcode: GT's PCODE */ + struct { + /** @lock: protecting GT's PCODE mailbox data */ + struct mutex lock; + } pcode; + + /** @sysfs: sysfs' kobj used by xe_gt_sysfs */ + struct kobject *sysfs; + + /** @mocs: info */ + struct { + /** @uc_index: UC index */ + u8 uc_index; + /** @wb_index: WB index, only used on L3_CCS platforms */ + u8 wb_index; + } mocs; + + /** @fuse_topo: GT topology reported by fuse registers */ + struct { + /** @g_dss_mask: dual-subslices usable by geometry */ + xe_dss_mask_t g_dss_mask; + + /** @c_dss_mask: dual-subslices usable by compute */ + xe_dss_mask_t c_dss_mask; + + /** @eu_mask_per_dss: EU mask per DSS*/ + xe_eu_mask_t eu_mask_per_dss; + } fuse_topo; + + /** @steering: register steering for individual HW units */ + struct { + /* @ranges: register ranges used for this steering type */ + const struct xe_mmio_range *ranges; + + /** @group_target: target to steer accesses to */ + u16 group_target; + /** @instance_target: instance to steer accesses to */ + u16 instance_target; + } steering[NUM_STEERING_TYPES]; + + /** + * @mcr_lock: protects the MCR_SELECTOR register for the duration + * of a steered operation + */ + spinlock_t mcr_lock; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc.c b/drivers/gpu/drm/xe/xe_guc.c new file mode 100644 index 000000000000..3c285d849ef6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc.c @@ -0,0 +1,875 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_guc.h" +#include "xe_guc_ads.h" +#include "xe_guc_ct.h" +#include "xe_guc_hwconfig.h" +#include "xe_guc_log.h" +#include "xe_guc_reg.h" +#include "xe_guc_pc.h" +#include "xe_guc_submit.h" +#include "xe_gt.h" +#include "xe_platform_types.h" +#include "xe_uc_fw.h" +#include "xe_wopcm.h" +#include "xe_mmio.h" +#include "xe_force_wake.h" +#include "i915_reg_defs.h" +#include "gt/intel_gt_regs.h" + +/* TODO: move to common file */ +#define GUC_PVC_MOCS_INDEX_MASK REG_GENMASK(25, 24) +#define PVC_MOCS_UC_INDEX 1 +#define PVC_GUC_MOCS_INDEX(index) REG_FIELD_PREP(GUC_PVC_MOCS_INDEX_MASK,\ + index) + +static struct xe_gt * +guc_to_gt(struct xe_guc *guc) +{ + return container_of(guc, struct xe_gt, uc.guc); +} + +static struct xe_device * +guc_to_xe(struct xe_guc *guc) +{ + return gt_to_xe(guc_to_gt(guc)); +} + +/* GuC addresses above GUC_GGTT_TOP also don't map through the GTT */ +#define GUC_GGTT_TOP 0xFEE00000 +static u32 guc_bo_ggtt_addr(struct xe_guc *guc, + struct xe_bo *bo) +{ + u32 addr = xe_bo_ggtt_addr(bo); + + XE_BUG_ON(addr < xe_wopcm_size(guc_to_xe(guc))); + XE_BUG_ON(range_overflows_t(u32, addr, bo->size, GUC_GGTT_TOP)); + + return addr; +} + +static u32 guc_ctl_debug_flags(struct xe_guc *guc) +{ + u32 level = xe_guc_log_get_level(&guc->log); + u32 flags = 0; + + if (!GUC_LOG_LEVEL_IS_VERBOSE(level)) + flags |= GUC_LOG_DISABLED; + else + flags |= GUC_LOG_LEVEL_TO_VERBOSITY(level) << + GUC_LOG_VERBOSITY_SHIFT; + + return flags; +} + +static u32 guc_ctl_feature_flags(struct xe_guc *guc) +{ + return GUC_CTL_ENABLE_SLPC; +} + +static u32 guc_ctl_log_params_flags(struct xe_guc *guc) +{ + u32 offset = guc_bo_ggtt_addr(guc, guc->log.bo) >> PAGE_SHIFT; + u32 flags; + + #if (((CRASH_BUFFER_SIZE) % SZ_1M) == 0) + #define LOG_UNIT SZ_1M + #define LOG_FLAG GUC_LOG_LOG_ALLOC_UNITS + #else + #define LOG_UNIT SZ_4K + #define LOG_FLAG 0 + #endif + + #if (((CAPTURE_BUFFER_SIZE) % SZ_1M) == 0) + #define CAPTURE_UNIT SZ_1M + #define CAPTURE_FLAG GUC_LOG_CAPTURE_ALLOC_UNITS + #else + #define CAPTURE_UNIT SZ_4K + #define CAPTURE_FLAG 0 + #endif + + BUILD_BUG_ON(!CRASH_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(CRASH_BUFFER_SIZE, LOG_UNIT)); + BUILD_BUG_ON(!DEBUG_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(DEBUG_BUFFER_SIZE, LOG_UNIT)); + BUILD_BUG_ON(!CAPTURE_BUFFER_SIZE); + BUILD_BUG_ON(!IS_ALIGNED(CAPTURE_BUFFER_SIZE, CAPTURE_UNIT)); + + BUILD_BUG_ON((CRASH_BUFFER_SIZE / LOG_UNIT - 1) > + (GUC_LOG_CRASH_MASK >> GUC_LOG_CRASH_SHIFT)); + BUILD_BUG_ON((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) > + (GUC_LOG_DEBUG_MASK >> GUC_LOG_DEBUG_SHIFT)); + BUILD_BUG_ON((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) > + (GUC_LOG_CAPTURE_MASK >> GUC_LOG_CAPTURE_SHIFT)); + + flags = GUC_LOG_VALID | + GUC_LOG_NOTIFY_ON_HALF_FULL | + CAPTURE_FLAG | + LOG_FLAG | + ((CRASH_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_CRASH_SHIFT) | + ((DEBUG_BUFFER_SIZE / LOG_UNIT - 1) << GUC_LOG_DEBUG_SHIFT) | + ((CAPTURE_BUFFER_SIZE / CAPTURE_UNIT - 1) << + GUC_LOG_CAPTURE_SHIFT) | + (offset << GUC_LOG_BUF_ADDR_SHIFT); + + #undef LOG_UNIT + #undef LOG_FLAG + #undef CAPTURE_UNIT + #undef CAPTURE_FLAG + + return flags; +} + +static u32 guc_ctl_ads_flags(struct xe_guc *guc) +{ + u32 ads = guc_bo_ggtt_addr(guc, guc->ads.bo) >> PAGE_SHIFT; + u32 flags = ads << GUC_ADS_ADDR_SHIFT; + + return flags; +} + +static u32 guc_ctl_wa_flags(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + u32 flags = 0; + + /* Wa_22012773006:gen11,gen12 < XeHP */ + if (GRAPHICS_VER(xe) >= 11 && + GRAPHICS_VERx100(xe) < 1250) + flags |= GUC_WA_POLLCS; + + /* Wa_16011759253 */ + /* Wa_22011383443 */ + if (IS_SUBPLATFORM_STEP(xe, XE_DG2, XE_SUBPLATFORM_DG2_G10, STEP_A0, STEP_B0) || + IS_PLATFORM_STEP(xe, XE_PVC, STEP_A0, STEP_B0)) + flags |= GUC_WA_GAM_CREDITS; + + /* Wa_14014475959 */ + if (IS_PLATFORM_STEP(xe, XE_METEORLAKE, STEP_A0, STEP_B0) || + xe->info.platform == XE_DG2) + flags |= GUC_WA_HOLD_CCS_SWITCHOUT; + + /* + * Wa_14012197797 + * Wa_22011391025 + * + * The same WA bit is used for both and 22011391025 is applicable to + * all DG2. + */ + if (xe->info.platform == XE_DG2) + flags |= GUC_WA_DUAL_QUEUE; + + /* + * Wa_2201180203 + * GUC_WA_PRE_PARSER causes media workload hang for PVC A0 and PCIe + * errors. Disable this for PVC A0 steppings. + */ + if (GRAPHICS_VER(xe) <= 12 && + !IS_PLATFORM_STEP(xe, XE_PVC, STEP_A0, STEP_B0)) + flags |= GUC_WA_PRE_PARSER; + + /* Wa_16011777198 */ + if (IS_SUBPLATFORM_STEP(xe, XE_DG2, XE_SUBPLATFORM_DG2_G10, STEP_A0, STEP_C0) || + IS_SUBPLATFORM_STEP(xe, XE_DG2, XE_SUBPLATFORM_DG2_G11, STEP_A0, + STEP_B0)) + flags |= GUC_WA_RCS_RESET_BEFORE_RC6; + + /* + * Wa_22012727170 + * Wa_22012727685 + * + * This WA is applicable to PVC CT A0, but causes media regressions. + * Drop the WA for PVC. + */ + if (IS_SUBPLATFORM_STEP(xe, XE_DG2, XE_SUBPLATFORM_DG2_G10, STEP_A0, STEP_C0) || + IS_SUBPLATFORM_STEP(xe, XE_DG2, XE_SUBPLATFORM_DG2_G11, STEP_A0, + STEP_FOREVER)) + flags |= GUC_WA_CONTEXT_ISOLATION; + + /* Wa_16015675438, Wa_18020744125 */ + if (!xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_RENDER)) + flags |= GUC_WA_RCS_REGS_IN_CCS_REGS_LIST; + + /* Wa_1509372804 */ + if (IS_PLATFORM_STEP(xe, XE_PVC, STEP_A0, STEP_C0)) + flags |= GUC_WA_RENDER_RST_RC6_EXIT; + + + return flags; +} + +static u32 guc_ctl_devid(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + + return (((u32)xe->info.devid) << 16) | xe->info.revid; +} + +static void guc_init_params(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + u32 *params = guc->params; + int i; + + BUILD_BUG_ON(sizeof(guc->params) != GUC_CTL_MAX_DWORDS * sizeof(u32)); + BUILD_BUG_ON(SOFT_SCRATCH_COUNT != GUC_CTL_MAX_DWORDS + 2); + + params[GUC_CTL_LOG_PARAMS] = guc_ctl_log_params_flags(guc); + params[GUC_CTL_FEATURE] = guc_ctl_feature_flags(guc); + params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc); + params[GUC_CTL_ADS] = guc_ctl_ads_flags(guc); + params[GUC_CTL_WA] = guc_ctl_wa_flags(guc); + params[GUC_CTL_DEVID] = guc_ctl_devid(guc); + + for (i = 0; i < GUC_CTL_MAX_DWORDS; i++) + drm_dbg(&xe->drm, "GuC param[%2d] = 0x%08x\n", i, params[i]); +} + +/* + * Initialise the GuC parameter block before starting the firmware + * transfer. These parameters are read by the firmware on startup + * and cannot be changed thereafter. + */ +void guc_write_params(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + int i; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + xe_mmio_write32(gt, SOFT_SCRATCH(0).reg, 0); + + for (i = 0; i < GUC_CTL_MAX_DWORDS; i++) + xe_mmio_write32(gt, SOFT_SCRATCH(1 + i).reg, guc->params[i]); +} + +#define MEDIA_GUC_HOST_INTERRUPT _MMIO(0x190304) + +int xe_guc_init(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + int ret; + + guc->fw.type = XE_UC_FW_TYPE_GUC; + ret = xe_uc_fw_init(&guc->fw); + if (ret) + goto out; + + ret = xe_guc_log_init(&guc->log); + if (ret) + goto out; + + ret = xe_guc_ads_init(&guc->ads); + if (ret) + goto out; + + ret = xe_guc_ct_init(&guc->ct); + if (ret) + goto out; + + ret = xe_guc_pc_init(&guc->pc); + if (ret) + goto out; + + guc_init_params(guc); + + if (xe_gt_is_media_type(gt)) + guc->notify_reg = MEDIA_GUC_HOST_INTERRUPT.reg; + else + guc->notify_reg = GEN11_GUC_HOST_INTERRUPT.reg; + + xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOADABLE); + + return 0; + +out: + drm_err(&xe->drm, "GuC init failed with %d", ret); + return ret; +} + +/** + * xe_guc_init_post_hwconfig - initialize GuC post hwconfig load + * @guc: The GuC object + * + * Return: 0 on success, negative error code on error. + */ +int xe_guc_init_post_hwconfig(struct xe_guc *guc) +{ + return xe_guc_ads_init_post_hwconfig(&guc->ads); +} + +int xe_guc_post_load_init(struct xe_guc *guc) +{ + xe_guc_ads_populate_post_load(&guc->ads); + + return 0; +} + +int xe_guc_reset(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + u32 guc_status; + int ret; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + xe_mmio_write32(gt, GEN6_GDRST.reg, GEN11_GRDOM_GUC); + + ret = xe_mmio_wait32(gt, GEN6_GDRST.reg, 0, GEN11_GRDOM_GUC, 5); + if (ret) { + drm_err(&xe->drm, "GuC reset timed out, GEN6_GDRST=0x%8x\n", + xe_mmio_read32(gt, GEN6_GDRST.reg)); + goto err_out; + } + + guc_status = xe_mmio_read32(gt, GUC_STATUS.reg); + if (!(guc_status & GS_MIA_IN_RESET)) { + drm_err(&xe->drm, + "GuC status: 0x%x, MIA core expected to be in reset\n", + guc_status); + ret = -EIO; + goto err_out; + } + + return 0; + +err_out: + + return ret; +} + +static void guc_prepare_xfer(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + struct xe_device *xe = guc_to_xe(guc); + u32 shim_flags = GUC_ENABLE_READ_CACHE_LOGIC | + GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA | + GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA | + GUC_ENABLE_MIA_CLOCK_GATING; + + if (GRAPHICS_VERx100(xe) < 1250) + shim_flags |= GUC_DISABLE_SRAM_INIT_TO_ZEROES | + GUC_ENABLE_MIA_CACHING; + + if (xe->info.platform == XE_PVC) + shim_flags |= PVC_GUC_MOCS_INDEX(PVC_MOCS_UC_INDEX); + + /* Must program this register before loading the ucode with DMA */ + xe_mmio_write32(gt, GUC_SHIM_CONTROL.reg, shim_flags); + + xe_mmio_write32(gt, GEN9_GT_PM_CONFIG.reg, GT_DOORBELL_ENABLE); +} + +/* + * Supporting MMIO & in memory RSA + */ +static int guc_xfer_rsa(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 rsa[UOS_RSA_SCRATCH_COUNT]; + size_t copied; + int i; + + if (guc->fw.rsa_size > 256) { + u32 rsa_ggtt_addr = xe_bo_ggtt_addr(guc->fw.bo) + + xe_uc_fw_rsa_offset(&guc->fw); + xe_mmio_write32(gt, UOS_RSA_SCRATCH(0).reg, rsa_ggtt_addr); + return 0; + } + + copied = xe_uc_fw_copy_rsa(&guc->fw, rsa, sizeof(rsa)); + if (copied < sizeof(rsa)) + return -ENOMEM; + + for (i = 0; i < UOS_RSA_SCRATCH_COUNT; i++) + xe_mmio_write32(gt, UOS_RSA_SCRATCH(i).reg, rsa[i]); + + return 0; +} + +/* + * Read the GuC status register (GUC_STATUS) and store it in the + * specified location; then return a boolean indicating whether + * the value matches either of two values representing completion + * of the GuC boot process. + * + * This is used for polling the GuC status in a wait_for() + * loop below. + */ +static bool guc_ready(struct xe_guc *guc, u32 *status) +{ + u32 val = xe_mmio_read32(guc_to_gt(guc), GUC_STATUS.reg); + u32 uk_val = REG_FIELD_GET(GS_UKERNEL_MASK, val); + + *status = val; + return uk_val == XE_GUC_LOAD_STATUS_READY; +} + +static int guc_wait_ucode(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + u32 status; + int ret; + + /* + * Wait for the GuC to start up. + * NB: Docs recommend not using the interrupt for completion. + * Measurements indicate this should take no more than 20ms + * (assuming the GT clock is at maximum frequency). So, a + * timeout here indicates that the GuC has failed and is unusable. + * (Higher levels of the driver may decide to reset the GuC and + * attempt the ucode load again if this happens.) + * + * FIXME: There is a known (but exceedingly unlikely) race condition + * where the asynchronous frequency management code could reduce + * the GT clock while a GuC reload is in progress (during a full + * GT reset). A fix is in progress but there are complex locking + * issues to be resolved. In the meantime bump the timeout to + * 200ms. Even at slowest clock, this should be sufficient. And + * in the working case, a larger timeout makes no difference. + */ + ret = wait_for(guc_ready(guc, &status), 200); + if (ret) { + struct drm_device *drm = &xe->drm; + struct drm_printer p = drm_info_printer(drm->dev); + + drm_info(drm, "GuC load failed: status = 0x%08X\n", status); + drm_info(drm, "GuC load failed: status: Reset = %d, " + "BootROM = 0x%02X, UKernel = 0x%02X, " + "MIA = 0x%02X, Auth = 0x%02X\n", + REG_FIELD_GET(GS_MIA_IN_RESET, status), + REG_FIELD_GET(GS_BOOTROM_MASK, status), + REG_FIELD_GET(GS_UKERNEL_MASK, status), + REG_FIELD_GET(GS_MIA_MASK, status), + REG_FIELD_GET(GS_AUTH_STATUS_MASK, status)); + + if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) { + drm_info(drm, "GuC firmware signature verification failed\n"); + ret = -ENOEXEC; + } + + if (REG_FIELD_GET(GS_UKERNEL_MASK, status) == + XE_GUC_LOAD_STATUS_EXCEPTION) { + drm_info(drm, "GuC firmware exception. EIP: %#x\n", + xe_mmio_read32(guc_to_gt(guc), + SOFT_SCRATCH(13).reg)); + ret = -ENXIO; + } + + xe_guc_log_print(&guc->log, &p); + } else { + drm_dbg(&xe->drm, "GuC successfully loaded"); + } + + return ret; +} + +static int __xe_guc_upload(struct xe_guc *guc) +{ + int ret; + + guc_write_params(guc); + guc_prepare_xfer(guc); + + /* + * Note that GuC needs the CSS header plus uKernel code to be copied + * by the DMA engine in one operation, whereas the RSA signature is + * loaded separately, either by copying it to the UOS_RSA_SCRATCH + * register (if key size <= 256) or through a ggtt-pinned vma (if key + * size > 256). The RSA size and therefore the way we provide it to the + * HW is fixed for each platform and hard-coded in the bootrom. + */ + ret = guc_xfer_rsa(guc); + if (ret) + goto out; + /* + * Current uCode expects the code to be loaded at 8k; locations below + * this are used for the stack. + */ + ret = xe_uc_fw_upload(&guc->fw, 0x2000, UOS_MOVE); + if (ret) + goto out; + + /* Wait for authentication */ + ret = guc_wait_ucode(guc); + if (ret) + goto out; + + xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_RUNNING); + return 0; + +out: + xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOAD_FAIL); + return 0 /* FIXME: ret, don't want to stop load currently */; +} + +/** + * xe_guc_min_load_for_hwconfig - load minimal GuC and read hwconfig table + * @guc: The GuC object + * + * This function uploads a minimal GuC that does not support submissions but + * in a state where the hwconfig table can be read. Next, it reads and parses + * the hwconfig table so it can be used for subsequent steps in the driver load. + * Lastly, it enables CT communication (XXX: this is needed for PFs/VFs only). + * + * Return: 0 on success, negative error code on error. + */ +int xe_guc_min_load_for_hwconfig(struct xe_guc *guc) +{ + int ret; + + xe_guc_ads_populate_minimal(&guc->ads); + + ret = __xe_guc_upload(guc); + if (ret) + return ret; + + ret = xe_guc_hwconfig_init(guc); + if (ret) + return ret; + + ret = xe_guc_enable_communication(guc); + if (ret) + return ret; + + return 0; +} + +int xe_guc_upload(struct xe_guc *guc) +{ + xe_guc_ads_populate(&guc->ads); + + return __xe_guc_upload(guc); +} + +static void guc_handle_mmio_msg(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 msg; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + msg = xe_mmio_read32(gt, SOFT_SCRATCH(15).reg); + msg &= XE_GUC_RECV_MSG_EXCEPTION | + XE_GUC_RECV_MSG_CRASH_DUMP_POSTED; + xe_mmio_write32(gt, SOFT_SCRATCH(15).reg, 0); + + if (msg & XE_GUC_RECV_MSG_CRASH_DUMP_POSTED) + drm_err(&guc_to_xe(guc)->drm, + "Received early GuC crash dump notification!\n"); + + if (msg & XE_GUC_RECV_MSG_EXCEPTION) + drm_err(&guc_to_xe(guc)->drm, + "Received early GuC exception notification!\n"); +} + +void guc_enable_irq(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 events = xe_gt_is_media_type(gt) ? + REG_FIELD_PREP(ENGINE0_MASK, GUC_INTR_GUC2HOST) : + REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST); + + xe_mmio_write32(gt, GEN11_GUC_SG_INTR_ENABLE.reg, + REG_FIELD_PREP(ENGINE1_MASK, GUC_INTR_GUC2HOST)); + if (xe_gt_is_media_type(gt)) + xe_mmio_rmw32(gt, GEN11_GUC_SG_INTR_MASK.reg, events, 0); + else + xe_mmio_write32(gt, GEN11_GUC_SG_INTR_MASK.reg, ~events); +} + +int xe_guc_enable_communication(struct xe_guc *guc) +{ + int err; + + guc_enable_irq(guc); + + xe_mmio_rmw32(guc_to_gt(guc), GEN6_PMINTRMSK.reg, + ARAT_EXPIRED_INTRMSK, 0); + + err = xe_guc_ct_enable(&guc->ct); + if (err) + return err; + + guc_handle_mmio_msg(guc); + + return 0; +} + +int xe_guc_suspend(struct xe_guc *guc) +{ + int ret; + u32 action[] = { + XE_GUC_ACTION_CLIENT_SOFT_RESET, + }; + + ret = xe_guc_send_mmio(guc, action, ARRAY_SIZE(action)); + if (ret) { + drm_err(&guc_to_xe(guc)->drm, + "GuC suspend: CLIENT_SOFT_RESET fail: %d!\n", ret); + return ret; + } + + xe_guc_sanitize(guc); + return 0; +} + +void xe_guc_notify(struct xe_guc *guc) +{ + struct xe_gt *gt = guc_to_gt(guc); + + xe_mmio_write32(gt, guc->notify_reg, GUC_SEND_TRIGGER); +} + +int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr) +{ + u32 action[] = { + XE_GUC_ACTION_AUTHENTICATE_HUC, + rsa_addr + }; + + return xe_guc_ct_send_block(&guc->ct, action, ARRAY_SIZE(action)); +} + +#define MEDIA_SOFT_SCRATCH(n) _MMIO(0x190310 + (n) * 4) +#define MEDIA_SOFT_SCRATCH_COUNT 4 + +int xe_guc_send_mmio(struct xe_guc *guc, const u32 *request, u32 len) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + u32 header; + u32 reply_reg = xe_gt_is_media_type(gt) ? + MEDIA_SOFT_SCRATCH(0).reg : GEN11_SOFT_SCRATCH(0).reg; + int ret; + int i; + + XE_BUG_ON(guc->ct.enabled); + XE_BUG_ON(!len); + XE_BUG_ON(len > GEN11_SOFT_SCRATCH_COUNT); + XE_BUG_ON(len > MEDIA_SOFT_SCRATCH_COUNT); + XE_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, request[0]) != + GUC_HXG_ORIGIN_HOST); + XE_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, request[0]) != + GUC_HXG_TYPE_REQUEST); + +retry: + /* Not in critical data-path, just do if else for GT type */ + if (xe_gt_is_media_type(gt)) { + for (i = 0; i < len; ++i) + xe_mmio_write32(gt, MEDIA_SOFT_SCRATCH(i).reg, + request[i]); +#define LAST_INDEX MEDIA_SOFT_SCRATCH_COUNT - 1 + xe_mmio_read32(gt, MEDIA_SOFT_SCRATCH(LAST_INDEX).reg); + } else { + for (i = 0; i < len; ++i) + xe_mmio_write32(gt, GEN11_SOFT_SCRATCH(i).reg, + request[i]); +#undef LAST_INDEX +#define LAST_INDEX GEN11_SOFT_SCRATCH_COUNT - 1 + xe_mmio_read32(gt, GEN11_SOFT_SCRATCH(LAST_INDEX).reg); + } + + xe_guc_notify(guc); + + ret = xe_mmio_wait32(gt, reply_reg, + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, + GUC_HXG_ORIGIN_GUC), + GUC_HXG_MSG_0_ORIGIN, + 50); + if (ret) { +timeout: + drm_err(&xe->drm, "mmio request 0x%08x: no reply 0x%08x\n", + request[0], xe_mmio_read32(gt, reply_reg)); + return ret; + } + + header = xe_mmio_read32(gt, reply_reg); + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == + GUC_HXG_TYPE_NO_RESPONSE_BUSY) { +#define done ({ header = xe_mmio_read32(gt, reply_reg); \ + FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) != \ + GUC_HXG_ORIGIN_GUC || \ + FIELD_GET(GUC_HXG_MSG_0_TYPE, header) != \ + GUC_HXG_TYPE_NO_RESPONSE_BUSY; }) + + ret = wait_for(done, 1000); + if (unlikely(ret)) + goto timeout; + if (unlikely(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, header) != + GUC_HXG_ORIGIN_GUC)) + goto proto; +#undef done + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == + GUC_HXG_TYPE_NO_RESPONSE_RETRY) { + u32 reason = FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, header); + + drm_dbg(&xe->drm, "mmio request %#x: retrying, reason %u\n", + request[0], reason); + goto retry; + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) == + GUC_HXG_TYPE_RESPONSE_FAILURE) { + u32 hint = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, header); + u32 error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, header); + + drm_err(&xe->drm, "mmio request %#x: failure %x/%u\n", + request[0], error, hint); + return -ENXIO; + } + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, header) != + GUC_HXG_TYPE_RESPONSE_SUCCESS) { +proto: + drm_err(&xe->drm, "mmio request %#x: unexpected reply %#x\n", + request[0], header); + return -EPROTO; + } + + /* Use data from the GuC response as our return value */ + return FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, header); +} + +static int guc_self_cfg(struct xe_guc *guc, u16 key, u16 len, u64 val) +{ + u32 request[HOST2GUC_SELF_CFG_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, + GUC_ACTION_HOST2GUC_SELF_CFG), + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_KEY, key) | + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_1_KLV_LEN, len), + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_2_VALUE32, + lower_32_bits(val)), + FIELD_PREP(HOST2GUC_SELF_CFG_REQUEST_MSG_3_VALUE64, + upper_32_bits(val)), + }; + int ret; + + XE_BUG_ON(len > 2); + XE_BUG_ON(len == 1 && upper_32_bits(val)); + + /* Self config must go over MMIO */ + ret = xe_guc_send_mmio(guc, request, ARRAY_SIZE(request)); + + if (unlikely(ret < 0)) + return ret; + if (unlikely(ret > 1)) + return -EPROTO; + if (unlikely(!ret)) + return -ENOKEY; + + return 0; +} + +int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val) +{ + return guc_self_cfg(guc, key, 1, val); +} + +int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val) +{ + return guc_self_cfg(guc, key, 2, val); +} + +void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir) +{ + if (iir & GUC_INTR_GUC2HOST) + xe_guc_ct_irq_handler(&guc->ct); +} + +void xe_guc_sanitize(struct xe_guc *guc) +{ + xe_uc_fw_change_status(&guc->fw, XE_UC_FIRMWARE_LOADABLE); + xe_guc_ct_disable(&guc->ct); +} + +int xe_guc_reset_prepare(struct xe_guc *guc) +{ + return xe_guc_submit_reset_prepare(guc); +} + +void xe_guc_reset_wait(struct xe_guc *guc) +{ + xe_guc_submit_reset_wait(guc); +} + +void xe_guc_stop_prepare(struct xe_guc *guc) +{ + XE_WARN_ON(xe_guc_pc_stop(&guc->pc)); +} + +int xe_guc_stop(struct xe_guc *guc) +{ + int ret; + + xe_guc_ct_disable(&guc->ct); + + ret = xe_guc_submit_stop(guc); + if (ret) + return ret; + + return 0; +} + +int xe_guc_start(struct xe_guc *guc) +{ + int ret; + + ret = xe_guc_submit_start(guc); + if (ret) + return ret; + + ret = xe_guc_pc_start(&guc->pc); + XE_WARN_ON(ret); + + return 0; +} + +void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p) +{ + struct xe_gt *gt = guc_to_gt(guc); + u32 status; + int err; + int i; + + xe_uc_fw_print(&guc->fw, p); + + err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (err) + return; + + status = xe_mmio_read32(gt, GUC_STATUS.reg); + + drm_printf(p, "\nGuC status 0x%08x:\n", status); + drm_printf(p, "\tBootrom status = 0x%x\n", + (status & GS_BOOTROM_MASK) >> GS_BOOTROM_SHIFT); + drm_printf(p, "\tuKernel status = 0x%x\n", + (status & GS_UKERNEL_MASK) >> GS_UKERNEL_SHIFT); + drm_printf(p, "\tMIA Core status = 0x%x\n", + (status & GS_MIA_MASK) >> GS_MIA_SHIFT); + drm_printf(p, "\tLog level = %d\n", + xe_guc_log_get_level(&guc->log)); + + drm_puts(p, "\nScratch registers:\n"); + for (i = 0; i < SOFT_SCRATCH_COUNT; i++) { + drm_printf(p, "\t%2d: \t0x%x\n", + i, xe_mmio_read32(gt, SOFT_SCRATCH(i).reg)); + } + + xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); + + xe_guc_ct_print(&guc->ct, p); + xe_guc_submit_print(guc, p); +} diff --git a/drivers/gpu/drm/xe/xe_guc.h b/drivers/gpu/drm/xe/xe_guc.h new file mode 100644 index 000000000000..72b71d75566c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_H_ +#define _XE_GUC_H_ + +#include "xe_hw_engine_types.h" +#include "xe_guc_types.h" +#include "xe_macros.h" + +struct drm_printer; + +int xe_guc_init(struct xe_guc *guc); +int xe_guc_init_post_hwconfig(struct xe_guc *guc); +int xe_guc_post_load_init(struct xe_guc *guc); +int xe_guc_reset(struct xe_guc *guc); +int xe_guc_upload(struct xe_guc *guc); +int xe_guc_min_load_for_hwconfig(struct xe_guc *guc); +int xe_guc_enable_communication(struct xe_guc *guc); +int xe_guc_suspend(struct xe_guc *guc); +void xe_guc_notify(struct xe_guc *guc); +int xe_guc_auth_huc(struct xe_guc *guc, u32 rsa_addr); +int xe_guc_send_mmio(struct xe_guc *guc, const u32 *request, u32 len); +int xe_guc_self_cfg32(struct xe_guc *guc, u16 key, u32 val); +int xe_guc_self_cfg64(struct xe_guc *guc, u16 key, u64 val); +void xe_guc_irq_handler(struct xe_guc *guc, const u16 iir); +void xe_guc_sanitize(struct xe_guc *guc); +void xe_guc_print_info(struct xe_guc *guc, struct drm_printer *p); +int xe_guc_reset_prepare(struct xe_guc *guc); +void xe_guc_reset_wait(struct xe_guc *guc); +void xe_guc_stop_prepare(struct xe_guc *guc); +int xe_guc_stop(struct xe_guc *guc); +int xe_guc_start(struct xe_guc *guc); + +static inline u16 xe_engine_class_to_guc_class(enum xe_engine_class class) +{ + switch (class) { + case XE_ENGINE_CLASS_RENDER: + return GUC_RENDER_CLASS; + case XE_ENGINE_CLASS_VIDEO_DECODE: + return GUC_VIDEO_CLASS; + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + return GUC_VIDEOENHANCE_CLASS; + case XE_ENGINE_CLASS_COPY: + return GUC_BLITTER_CLASS; + case XE_ENGINE_CLASS_COMPUTE: + return GUC_COMPUTE_CLASS; + case XE_ENGINE_CLASS_OTHER: + default: + XE_WARN_ON(class); + return -1; + } +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c new file mode 100644 index 000000000000..0c08cecaca40 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -0,0 +1,676 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_bo.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_guc_ads.h" +#include "xe_guc_reg.h" +#include "xe_hw_engine.h" +#include "xe_lrc.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_platform_types.h" +#include "gt/intel_gt_regs.h" +#include "gt/intel_engine_regs.h" + +/* Slack of a few additional entries per engine */ +#define ADS_REGSET_EXTRA_MAX 8 + +static struct xe_guc * +ads_to_guc(struct xe_guc_ads *ads) +{ + return container_of(ads, struct xe_guc, ads); +} + +static struct xe_gt * +ads_to_gt(struct xe_guc_ads *ads) +{ + return container_of(ads, struct xe_gt, uc.guc.ads); +} + +static struct xe_device * +ads_to_xe(struct xe_guc_ads *ads) +{ + return gt_to_xe(ads_to_gt(ads)); +} + +static struct iosys_map * +ads_to_map(struct xe_guc_ads *ads) +{ + return &ads->bo->vmap; +} + +/* UM Queue parameters: */ +#define GUC_UM_QUEUE_SIZE (SZ_64K) +#define GUC_PAGE_RES_TIMEOUT_US (-1) + +/* + * The Additional Data Struct (ADS) has pointers for different buffers used by + * the GuC. One single gem object contains the ADS struct itself (guc_ads) and + * all the extra buffers indirectly linked via the ADS struct's entries. + * + * Layout of the ADS blob allocated for the GuC: + * + * +---------------------------------------+ <== base + * | guc_ads | + * +---------------------------------------+ + * | guc_policies | + * +---------------------------------------+ + * | guc_gt_system_info | + * +---------------------------------------+ + * | guc_engine_usage | + * +---------------------------------------+ + * | guc_um_init_params | + * +---------------------------------------+ <== static + * | guc_mmio_reg[countA] (engine 0.0) | + * | guc_mmio_reg[countB] (engine 0.1) | + * | guc_mmio_reg[countC] (engine 1.0) | + * | ... | + * +---------------------------------------+ <== dynamic + * | padding | + * +---------------------------------------+ <== 4K aligned + * | golden contexts | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned + * | capture lists | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned + * | UM queues | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned + * | private data | + * +---------------------------------------+ + * | padding | + * +---------------------------------------+ <== 4K aligned + */ +struct __guc_ads_blob { + struct guc_ads ads; + struct guc_policies policies; + struct guc_gt_system_info system_info; + struct guc_engine_usage engine_usage; + struct guc_um_init_params um_init_params; + /* From here on, location is dynamic! Refer to above diagram. */ + struct guc_mmio_reg regset[0]; +} __packed; + +#define ads_blob_read(ads_, field_) \ + xe_map_rd_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ + struct __guc_ads_blob, field_) + +#define ads_blob_write(ads_, field_, val_) \ + xe_map_wr_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \ + struct __guc_ads_blob, field_, val_) + +#define info_map_write(xe_, map_, field_, val_) \ + xe_map_wr_field(xe_, map_, 0, struct guc_gt_system_info, field_, val_) + +#define info_map_read(xe_, map_, field_) \ + xe_map_rd_field(xe_, map_, 0, struct guc_gt_system_info, field_) + +static size_t guc_ads_regset_size(struct xe_guc_ads *ads) +{ + XE_BUG_ON(!ads->regset_size); + + return ads->regset_size; +} + +static size_t guc_ads_golden_lrc_size(struct xe_guc_ads *ads) +{ + return PAGE_ALIGN(ads->golden_lrc_size); +} + +static size_t guc_ads_capture_size(struct xe_guc_ads *ads) +{ + /* FIXME: Allocate a proper capture list */ + return PAGE_ALIGN(PAGE_SIZE); +} + +static size_t guc_ads_um_queues_size(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + + if (!xe->info.supports_usm) + return 0; + + return GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX; +} + +static size_t guc_ads_private_data_size(struct xe_guc_ads *ads) +{ + return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size); +} + +static size_t guc_ads_regset_offset(struct xe_guc_ads *ads) +{ + return offsetof(struct __guc_ads_blob, regset); +} + +static size_t guc_ads_golden_lrc_offset(struct xe_guc_ads *ads) +{ + size_t offset; + + offset = guc_ads_regset_offset(ads) + + guc_ads_regset_size(ads); + + return PAGE_ALIGN(offset); +} + +static size_t guc_ads_capture_offset(struct xe_guc_ads *ads) +{ + size_t offset; + + offset = guc_ads_golden_lrc_offset(ads) + + guc_ads_golden_lrc_size(ads); + + return PAGE_ALIGN(offset); +} + +static size_t guc_ads_um_queues_offset(struct xe_guc_ads *ads) +{ + u32 offset; + + offset = guc_ads_capture_offset(ads) + + guc_ads_capture_size(ads); + + return PAGE_ALIGN(offset); +} + +static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads) +{ + size_t offset; + + offset = guc_ads_um_queues_offset(ads) + + guc_ads_um_queues_size(ads); + + return PAGE_ALIGN(offset); +} + +static size_t guc_ads_size(struct xe_guc_ads *ads) +{ + return guc_ads_private_data_offset(ads) + + guc_ads_private_data_size(ads); +} + +static void guc_ads_fini(struct drm_device *drm, void *arg) +{ + struct xe_guc_ads *ads = arg; + + xe_bo_unpin_map_no_vm(ads->bo); +} + +static size_t calculate_regset_size(struct xe_gt *gt) +{ + struct xe_reg_sr_entry *sr_entry; + unsigned long sr_idx; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + unsigned int count = 0; + + for_each_hw_engine(hwe, gt, id) + xa_for_each(&hwe->reg_sr.xa, sr_idx, sr_entry) + count++; + + count += (ADS_REGSET_EXTRA_MAX + LNCFCMOCS_REG_COUNT) * XE_NUM_HW_ENGINES; + + return count * sizeof(struct guc_mmio_reg); +} + +static u32 engine_enable_mask(struct xe_gt *gt, enum xe_engine_class class) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + u32 mask = 0; + + for_each_hw_engine(hwe, gt, id) + if (hwe->class == class) + mask |= BIT(hwe->instance); + + return mask; +} + +static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + size_t total_size = 0, alloc_size, real_size; + int class; + + for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { + if (class == XE_ENGINE_CLASS_OTHER) + continue; + + if (!engine_enable_mask(gt, class)) + continue; + + real_size = xe_lrc_size(xe, class); + alloc_size = PAGE_ALIGN(real_size); + total_size += alloc_size; + } + + return total_size; +} + +#define MAX_GOLDEN_LRC_SIZE (SZ_4K * 64) + +int xe_guc_ads_init(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + struct xe_bo *bo; + int err; + + ads->golden_lrc_size = calculate_golden_lrc_size(ads); + ads->regset_size = calculate_regset_size(gt); + + bo = xe_bo_create_pin_map(xe, gt, NULL, guc_ads_size(ads) + + MAX_GOLDEN_LRC_SIZE, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_GGTT_BIT); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + ads->bo = bo; + + err = drmm_add_action_or_reset(&xe->drm, guc_ads_fini, ads); + if (err) + return err; + + return 0; +} + +/** + * xe_guc_ads_init_post_hwconfig - initialize ADS post hwconfig load + * @ads: Additional data structures object + * + * Recalcuate golden_lrc_size & regset_size as the number hardware engines may + * have changed after the hwconfig was loaded. Also verify the new sizes fit in + * the already allocated ADS buffer object. + * + * Return: 0 on success, negative error code on error. + */ +int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads) +{ + struct xe_gt *gt = ads_to_gt(ads); + u32 prev_regset_size = ads->regset_size; + + XE_BUG_ON(!ads->bo); + + ads->golden_lrc_size = calculate_golden_lrc_size(ads); + ads->regset_size = calculate_regset_size(gt); + + XE_WARN_ON(ads->golden_lrc_size + + (ads->regset_size - prev_regset_size) > + MAX_GOLDEN_LRC_SIZE); + + return 0; +} + +static void guc_policies_init(struct xe_guc_ads *ads) +{ + ads_blob_write(ads, policies.dpc_promote_time, + GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US); + ads_blob_write(ads, policies.max_num_work_items, + GLOBAL_POLICY_MAX_NUM_WI); + ads_blob_write(ads, policies.global_flags, 0); + ads_blob_write(ads, policies.is_valid, 1); +} + +static void fill_engine_enable_masks(struct xe_gt *gt, + struct iosys_map *info_map) +{ + struct xe_device *xe = gt_to_xe(gt); + + info_map_write(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS], + engine_enable_mask(gt, XE_ENGINE_CLASS_RENDER)); + info_map_write(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS], + engine_enable_mask(gt, XE_ENGINE_CLASS_COPY)); + info_map_write(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS], + engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_DECODE)); + info_map_write(xe, info_map, + engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], + engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE)); + info_map_write(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], + engine_enable_mask(gt, XE_ENGINE_CLASS_COMPUTE)); +} + +static void guc_prep_golden_lrc_null(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + offsetof(struct __guc_ads_blob, system_info)); + u8 guc_class; + + for (guc_class = 0; guc_class <= GUC_MAX_ENGINE_CLASSES; ++guc_class) { + if (!info_map_read(xe, &info_map, + engine_enabled_masks[guc_class])) + continue; + + ads_blob_write(ads, ads.eng_state_size[guc_class], + guc_ads_golden_lrc_size(ads) - + xe_lrc_skip_size(xe)); + ads_blob_write(ads, ads.golden_context_lrca[guc_class], + xe_bo_ggtt_addr(ads->bo) + + guc_ads_golden_lrc_offset(ads)); + } +} + +static void guc_mapping_table_init_invalid(struct xe_gt *gt, + struct iosys_map *info_map) +{ + struct xe_device *xe = gt_to_xe(gt); + unsigned int i, j; + + /* Table must be set to invalid values for entries not used */ + for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i) + for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j) + info_map_write(xe, info_map, mapping_table[i][j], + GUC_MAX_INSTANCES_PER_CLASS); +} + +static void guc_mapping_table_init(struct xe_gt *gt, + struct iosys_map *info_map) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + guc_mapping_table_init_invalid(gt, info_map); + + for_each_hw_engine(hwe, gt, id) { + u8 guc_class; + + guc_class = xe_engine_class_to_guc_class(hwe->class); + info_map_write(xe, info_map, + mapping_table[guc_class][hwe->logical_instance], + hwe->instance); + } +} + +static void guc_capture_list_init(struct xe_guc_ads *ads) +{ + int i, j; + u32 addr = xe_bo_ggtt_addr(ads->bo) + guc_ads_capture_offset(ads); + + /* FIXME: Populate a proper capture list */ + for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) { + for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) { + ads_blob_write(ads, ads.capture_instance[i][j], addr); + ads_blob_write(ads, ads.capture_class[i][j], addr); + } + + ads_blob_write(ads, ads.capture_global[i], addr); + } +} + +static void guc_mmio_regset_write_one(struct xe_guc_ads *ads, + struct iosys_map *regset_map, + u32 reg, u32 flags, + unsigned int n_entry) +{ + struct guc_mmio_reg entry = { + .offset = reg, + .flags = flags, + /* TODO: steering */ + }; + + xe_map_memcpy_to(ads_to_xe(ads), regset_map, n_entry * sizeof(entry), + &entry, sizeof(entry)); +} + +static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads, + struct iosys_map *regset_map, + struct xe_hw_engine *hwe) +{ + struct xe_hw_engine *hwe_rcs_reset_domain = + xe_gt_any_hw_engine_by_reset_domain(hwe->gt, XE_ENGINE_CLASS_RENDER); + struct xe_reg_sr_entry *entry; + unsigned long idx; + unsigned count = 0; + const struct { + u32 reg; + u32 flags; + bool skip; + } *e, extra_regs[] = { + { .reg = RING_MODE_GEN7(hwe->mmio_base).reg, }, + { .reg = RING_HWS_PGA(hwe->mmio_base).reg, }, + { .reg = RING_IMR(hwe->mmio_base).reg, }, + { .reg = GEN12_RCU_MODE.reg, .flags = 0x3, + .skip = hwe != hwe_rcs_reset_domain }, + }; + u32 i; + + BUILD_BUG_ON(ARRAY_SIZE(extra_regs) > ADS_REGSET_EXTRA_MAX); + + xa_for_each(&hwe->reg_sr.xa, idx, entry) { + u32 flags = entry->masked_reg ? GUC_REGSET_MASKED : 0; + + guc_mmio_regset_write_one(ads, regset_map, idx, flags, count++); + } + + for (e = extra_regs; e < extra_regs + ARRAY_SIZE(extra_regs); e++) { + if (e->skip) + continue; + + guc_mmio_regset_write_one(ads, regset_map, + e->reg, e->flags, count++); + } + + for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) { + guc_mmio_regset_write_one(ads, regset_map, + GEN9_LNCFCMOCS(i).reg, 0, count++); + } + + XE_BUG_ON(ads->regset_size < (count * sizeof(struct guc_mmio_reg))); + + return count; +} + +static void guc_mmio_reg_state_init(struct xe_guc_ads *ads) +{ + size_t regset_offset = guc_ads_regset_offset(ads); + struct xe_gt *gt = ads_to_gt(ads); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + u32 addr = xe_bo_ggtt_addr(ads->bo) + regset_offset; + struct iosys_map regset_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + regset_offset); + + for_each_hw_engine(hwe, gt, id) { + unsigned int count; + u8 gc; + + /* + * 1. Write all MMIO entries for this engine to the table. No + * need to worry about fused-off engines and when there are + * entries in the regset: the reg_state_list has been zero'ed + * by xe_guc_ads_populate() + */ + count = guc_mmio_regset_write(ads, ®set_map, hwe); + if (!count) + continue; + + /* + * 2. Record in the header (ads.reg_state_list) the address + * location and number of entries + */ + gc = xe_engine_class_to_guc_class(hwe->class); + ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].address, addr); + ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].count, count); + + addr += count * sizeof(struct guc_mmio_reg); + iosys_map_incr(®set_map, count * sizeof(struct guc_mmio_reg)); + } +} + +static void guc_um_init_params(struct xe_guc_ads *ads) +{ + u32 um_queue_offset = guc_ads_um_queues_offset(ads); + u64 base_dpa; + u32 base_ggtt; + int i; + + base_ggtt = xe_bo_ggtt_addr(ads->bo) + um_queue_offset; + base_dpa = xe_bo_main_addr(ads->bo, PAGE_SIZE) + um_queue_offset; + + for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) { + ads_blob_write(ads, um_init_params.queue_params[i].base_dpa, + base_dpa + (i * GUC_UM_QUEUE_SIZE)); + ads_blob_write(ads, um_init_params.queue_params[i].base_ggtt_address, + base_ggtt + (i * GUC_UM_QUEUE_SIZE)); + ads_blob_write(ads, um_init_params.queue_params[i].size_in_bytes, + GUC_UM_QUEUE_SIZE); + } + + ads_blob_write(ads, um_init_params.page_response_timeout_in_us, + GUC_PAGE_RES_TIMEOUT_US); +} + +static void guc_doorbell_init(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + + if (GRAPHICS_VER(xe) >= 12 && !IS_DGFX(xe)) { + u32 distdbreg = + xe_mmio_read32(gt, GEN12_DIST_DBS_POPULATED.reg); + + ads_blob_write(ads, + system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI], + ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT) + & GEN12_DOORBELLS_PER_SQIDI) + 1); + } +} + +/** + * xe_guc_ads_populate_minimal - populate minimal ADS + * @ads: Additional data structures object + * + * This function populates a minimal ADS that does not support submissions but + * enough so the GuC can load and the hwconfig table can be read. + */ +void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads) +{ + struct xe_gt *gt = ads_to_gt(ads); + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + offsetof(struct __guc_ads_blob, system_info)); + u32 base = xe_bo_ggtt_addr(ads->bo); + + XE_BUG_ON(!ads->bo); + + xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, ads->bo->size); + guc_policies_init(ads); + guc_prep_golden_lrc_null(ads); + guc_mapping_table_init_invalid(gt, &info_map); + guc_doorbell_init(ads); + + ads_blob_write(ads, ads.scheduler_policies, base + + offsetof(struct __guc_ads_blob, policies)); + ads_blob_write(ads, ads.gt_system_info, base + + offsetof(struct __guc_ads_blob, system_info)); + ads_blob_write(ads, ads.private_data, base + + guc_ads_private_data_offset(ads)); +} + +void xe_guc_ads_populate(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + offsetof(struct __guc_ads_blob, system_info)); + u32 base = xe_bo_ggtt_addr(ads->bo); + + XE_BUG_ON(!ads->bo); + + xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, ads->bo->size); + guc_policies_init(ads); + fill_engine_enable_masks(gt, &info_map); + guc_mmio_reg_state_init(ads); + guc_prep_golden_lrc_null(ads); + guc_mapping_table_init(gt, &info_map); + guc_capture_list_init(ads); + guc_doorbell_init(ads); + + if (xe->info.supports_usm) { + guc_um_init_params(ads); + ads_blob_write(ads, ads.um_init_data, base + + offsetof(struct __guc_ads_blob, um_init_params)); + } + + ads_blob_write(ads, ads.scheduler_policies, base + + offsetof(struct __guc_ads_blob, policies)); + ads_blob_write(ads, ads.gt_system_info, base + + offsetof(struct __guc_ads_blob, system_info)); + ads_blob_write(ads, ads.private_data, base + + guc_ads_private_data_offset(ads)); +} + +static void guc_populate_golden_lrc(struct xe_guc_ads *ads) +{ + struct xe_device *xe = ads_to_xe(ads); + struct xe_gt *gt = ads_to_gt(ads); + struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads), + offsetof(struct __guc_ads_blob, system_info)); + size_t total_size = 0, alloc_size, real_size; + u32 addr_ggtt, offset; + int class; + + offset = guc_ads_golden_lrc_offset(ads); + addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset; + + for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { + u8 guc_class; + + if (class == XE_ENGINE_CLASS_OTHER) + continue; + + guc_class = xe_engine_class_to_guc_class(class); + + if (!info_map_read(xe, &info_map, + engine_enabled_masks[guc_class])) + continue; + + XE_BUG_ON(!gt->default_lrc[class]); + + real_size = xe_lrc_size(xe, class); + alloc_size = PAGE_ALIGN(real_size); + total_size += alloc_size; + + /* + * This interface is slightly confusing. We need to pass the + * base address of the full golden context and the size of just + * the engine state, which is the section of the context image + * that starts after the execlists LRC registers. This is + * required to allow the GuC to restore just the engine state + * when a watchdog reset occurs. + * We calculate the engine state size by removing the size of + * what comes before it in the context image (which is identical + * on all engines). + */ + ads_blob_write(ads, ads.eng_state_size[guc_class], + real_size - xe_lrc_skip_size(xe)); + ads_blob_write(ads, ads.golden_context_lrca[guc_class], + addr_ggtt); + + xe_map_memcpy_to(xe, ads_to_map(ads), offset, + gt->default_lrc[class], real_size); + + addr_ggtt += alloc_size; + offset += alloc_size; + } + + XE_BUG_ON(total_size != ads->golden_lrc_size); +} + +void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads) +{ + guc_populate_golden_lrc(ads); +} diff --git a/drivers/gpu/drm/xe/xe_guc_ads.h b/drivers/gpu/drm/xe/xe_guc_ads.h new file mode 100644 index 000000000000..138ef6267671 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ads.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_ADS_H_ +#define _XE_GUC_ADS_H_ + +#include "xe_guc_ads_types.h" + +int xe_guc_ads_init(struct xe_guc_ads *ads); +int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads); +void xe_guc_ads_populate(struct xe_guc_ads *ads); +void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads); +void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ads_types.h b/drivers/gpu/drm/xe/xe_guc_ads_types.h new file mode 100644 index 000000000000..4afe44bece4b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ads_types.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_ADS_TYPES_H_ +#define _XE_GUC_ADS_TYPES_H_ + +#include + +struct xe_bo; + +/** + * struct xe_guc_ads - GuC additional data structures (ADS) + */ +struct xe_guc_ads { + /** @bo: XE BO for GuC ads blob */ + struct xe_bo *bo; + /** @golden_lrc_size: golden LRC size */ + size_t golden_lrc_size; + /** @regset_size: size of register set passed to GuC for save/restore */ + u32 regset_size; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c new file mode 100644 index 000000000000..61a424c41779 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -0,0 +1,1196 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include +#include + +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_gt_pagefault.h" +#include "xe_guc_submit.h" +#include "xe_map.h" +#include "xe_trace.h" + +/* Used when a CT send wants to block and / or receive data */ +struct g2h_fence { + wait_queue_head_t wq; + u32 *response_buffer; + u32 seqno; + u16 response_len; + u16 error; + u16 hint; + u16 reason; + bool retry; + bool fail; + bool done; +}; + +static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer) +{ + g2h_fence->response_buffer = response_buffer; + g2h_fence->response_len = 0; + g2h_fence->fail = false; + g2h_fence->retry = false; + g2h_fence->done = false; + g2h_fence->seqno = ~0x0; +} + +static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence) +{ + return g2h_fence->seqno == ~0x0; +} + +static struct xe_guc * +ct_to_guc(struct xe_guc_ct *ct) +{ + return container_of(ct, struct xe_guc, ct); +} + +static struct xe_gt * +ct_to_gt(struct xe_guc_ct *ct) +{ + return container_of(ct, struct xe_gt, uc.guc.ct); +} + +static struct xe_device * +ct_to_xe(struct xe_guc_ct *ct) +{ + return gt_to_xe(ct_to_gt(ct)); +} + +/** + * DOC: GuC CTB Blob + * + * We allocate single blob to hold both CTB descriptors and buffers: + * + * +--------+-----------------------------------------------+------+ + * | offset | contents | size | + * +========+===============================================+======+ + * | 0x0000 | H2G CTB Descriptor (send) | | + * +--------+-----------------------------------------------+ 4K | + * | 0x0800 | G2H CTB Descriptor (g2h) | | + * +--------+-----------------------------------------------+------+ + * | 0x1000 | H2G CT Buffer (send) | n*4K | + * | | | | + * +--------+-----------------------------------------------+------+ + * | 0x1000 | G2H CT Buffer (g2h) | m*4K | + * | + n*4K | | | + * +--------+-----------------------------------------------+------+ + * + * Size of each ``CT Buffer`` must be multiple of 4K. + * We don't expect too many messages in flight at any time, unless we are + * using the GuC submission. In that case each request requires a minimum + * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this + * enough space to avoid backpressure on the driver. We increase the size + * of the receive buffer (relative to the send) to ensure a G2H response + * CTB has a landing spot. + */ + +#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K) +#define CTB_H2G_BUFFER_SIZE (SZ_4K) +#define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE) +#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4) + +static size_t guc_ct_size(void) +{ + return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + + CTB_G2H_BUFFER_SIZE; +} + +static void guc_ct_fini(struct drm_device *drm, void *arg) +{ + struct xe_guc_ct *ct = arg; + + xa_destroy(&ct->fence_lookup); + xe_bo_unpin_map_no_vm(ct->bo); +} + +static void g2h_worker_func(struct work_struct *w); + +static void primelockdep(struct xe_guc_ct *ct) +{ + if (!IS_ENABLED(CONFIG_LOCKDEP)) + return; + + fs_reclaim_acquire(GFP_KERNEL); + might_lock(&ct->lock); + fs_reclaim_release(GFP_KERNEL); +} + +int xe_guc_ct_init(struct xe_guc_ct *ct) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_gt *gt = ct_to_gt(ct); + struct xe_bo *bo; + int err; + + XE_BUG_ON(guc_ct_size() % PAGE_SIZE); + + mutex_init(&ct->lock); + spin_lock_init(&ct->fast_lock); + xa_init(&ct->fence_lookup); + ct->fence_context = dma_fence_context_alloc(1); + INIT_WORK(&ct->g2h_worker, g2h_worker_func); + init_waitqueue_head(&ct->wq); + + primelockdep(ct); + + bo = xe_bo_create_pin_map(xe, gt, NULL, guc_ct_size(), + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_GGTT_BIT); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + ct->bo = bo; + + err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); + if (err) + return err; + + return 0; +} + +#define desc_read(xe_, guc_ctb__, field_) \ + xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \ + struct guc_ct_buffer_desc, field_) + +#define desc_write(xe_, guc_ctb__, field_, val_) \ + xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \ + struct guc_ct_buffer_desc, field_, val_) + +static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g, + struct iosys_map *map) +{ + h2g->size = CTB_H2G_BUFFER_SIZE / sizeof(u32); + h2g->resv_space = 0; + h2g->tail = 0; + h2g->head = 0; + h2g->space = CIRC_SPACE(h2g->tail, h2g->head, h2g->size) - + h2g->resv_space; + h2g->broken = false; + + h2g->desc = *map; + xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); + + h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2); +} + +static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h, + struct iosys_map *map) +{ + g2h->size = CTB_G2H_BUFFER_SIZE / sizeof(u32); + g2h->resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32); + g2h->head = 0; + g2h->tail = 0; + g2h->space = CIRC_SPACE(g2h->tail, g2h->head, g2h->size) - + g2h->resv_space; + g2h->broken = false; + + g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE); + xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc)); + + g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 + + CTB_H2G_BUFFER_SIZE); +} + +static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct) +{ + struct xe_guc *guc = ct_to_guc(ct); + u32 desc_addr, ctb_addr, size; + int err; + + desc_addr = xe_bo_ggtt_addr(ct->bo); + ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2; + size = ct->ctbs.h2g.size * sizeof(u32); + + err = xe_guc_self_cfg64(guc, + GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY, + desc_addr); + if (err) + return err; + + err = xe_guc_self_cfg64(guc, + GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY, + ctb_addr); + if (err) + return err; + + return xe_guc_self_cfg32(guc, + GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY, + size); +} + +static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct) +{ + struct xe_guc *guc = ct_to_guc(ct); + u32 desc_addr, ctb_addr, size; + int err; + + desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE; + ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 + + CTB_H2G_BUFFER_SIZE; + size = ct->ctbs.g2h.size * sizeof(u32); + + err = xe_guc_self_cfg64(guc, + GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY, + desc_addr); + if (err) + return err; + + err = xe_guc_self_cfg64(guc, + GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY, + ctb_addr); + if (err) + return err; + + return xe_guc_self_cfg32(guc, + GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY, + size); +} + +static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable) +{ + u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = { + FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) | + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, + GUC_ACTION_HOST2GUC_CONTROL_CTB), + FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, + enable ? GUC_CTB_CONTROL_ENABLE : + GUC_CTB_CONTROL_DISABLE), + }; + int ret = xe_guc_send_mmio(ct_to_guc(ct), request, ARRAY_SIZE(request)); + + return ret > 0 ? -EPROTO : ret; +} + +int xe_guc_ct_enable(struct xe_guc_ct *ct) +{ + struct xe_device *xe = ct_to_xe(ct); + int err; + + XE_BUG_ON(ct->enabled); + + guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap); + guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap); + + err = guc_ct_ctb_h2g_register(ct); + if (err) + goto err_out; + + err = guc_ct_ctb_g2h_register(ct); + if (err) + goto err_out; + + err = guc_ct_control_toggle(ct, true); + if (err) + goto err_out; + + mutex_lock(&ct->lock); + ct->g2h_outstanding = 0; + ct->enabled = true; + mutex_unlock(&ct->lock); + + smp_mb(); + wake_up_all(&ct->wq); + drm_dbg(&xe->drm, "GuC CT communication channel enabled\n"); + + return 0; + +err_out: + drm_err(&xe->drm, "Failed to enabled CT (%d)\n", err); + + return err; +} + +void xe_guc_ct_disable(struct xe_guc_ct *ct) +{ + mutex_lock(&ct->lock); + ct->enabled = false; + mutex_unlock(&ct->lock); + + xa_destroy(&ct->fence_lookup); +} + +static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len) +{ + struct guc_ctb *h2g = &ct->ctbs.h2g; + + lockdep_assert_held(&ct->lock); + + if (cmd_len > h2g->space) { + h2g->head = desc_read(ct_to_xe(ct), h2g, head); + h2g->space = CIRC_SPACE(h2g->tail, h2g->head, h2g->size) - + h2g->resv_space; + if (cmd_len > h2g->space) + return false; + } + + return true; +} + +static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len) +{ + lockdep_assert_held(&ct->lock); + + return ct->ctbs.g2h.space > g2h_len; +} + +static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len) +{ + lockdep_assert_held(&ct->lock); + + if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len)) + return -EBUSY; + + return 0; +} + +static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len) +{ + lockdep_assert_held(&ct->lock); + ct->ctbs.h2g.space -= cmd_len; +} + +static void g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h) +{ + XE_BUG_ON(g2h_len > ct->ctbs.g2h.space); + + if (g2h_len) { + spin_lock_irq(&ct->fast_lock); + ct->ctbs.g2h.space -= g2h_len; + ct->g2h_outstanding += num_g2h; + spin_unlock_irq(&ct->fast_lock); + } +} + +static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) +{ + lockdep_assert_held(&ct->fast_lock); + XE_WARN_ON(ct->ctbs.g2h.space + g2h_len > + ct->ctbs.g2h.size - ct->ctbs.g2h.resv_space); + + ct->ctbs.g2h.space += g2h_len; + --ct->g2h_outstanding; +} + +static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len) +{ + spin_lock_irq(&ct->fast_lock); + __g2h_release_space(ct, g2h_len); + spin_unlock_irq(&ct->fast_lock); +} + +static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 ct_fence_value, bool want_response) +{ + struct xe_device *xe = ct_to_xe(ct); + struct guc_ctb *h2g = &ct->ctbs.h2g; + u32 cmd[GUC_CTB_MSG_MAX_LEN / sizeof(u32)]; + u32 cmd_len = len + GUC_CTB_HDR_LEN; + u32 cmd_idx = 0, i; + u32 tail = h2g->tail; + struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds, + tail * sizeof(u32)); + + lockdep_assert_held(&ct->lock); + XE_BUG_ON(len * sizeof(u32) > GUC_CTB_MSG_MAX_LEN); + XE_BUG_ON(tail > h2g->size); + + /* Command will wrap, zero fill (NOPs), return and check credits again */ + if (tail + cmd_len > h2g->size) { + xe_map_memset(xe, &map, 0, 0, (h2g->size - tail) * sizeof(u32)); + h2g_reserve_space(ct, (h2g->size - tail)); + h2g->tail = 0; + desc_write(xe, h2g, tail, h2g->tail); + + return -EAGAIN; + } + + /* + * dw0: CT header (including fence) + * dw1: HXG header (including action code) + * dw2+: action data + */ + cmd[cmd_idx++] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) | + FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) | + FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value); + if (want_response) { + cmd[cmd_idx++] = + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) | + FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | + GUC_HXG_EVENT_MSG_0_DATA0, action[0]); + } else { + cmd[cmd_idx++] = + FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_EVENT) | + FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION | + GUC_HXG_EVENT_MSG_0_DATA0, action[0]); + } + for (i = 1; i < len; ++i) + cmd[cmd_idx++] = action[i]; + + /* Write H2G ensuring visable before descriptor update */ + xe_map_memcpy_to(xe, &map, 0, cmd, cmd_len * sizeof(u32)); + xe_device_wmb(ct_to_xe(ct)); + + /* Update local copies */ + h2g->tail = (tail + cmd_len) % h2g->size; + h2g_reserve_space(ct, cmd_len); + + /* Update descriptor */ + desc_write(xe, h2g, tail, h2g->tail); + + return 0; +} + +static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, + u32 len, u32 g2h_len, u32 num_g2h, + struct g2h_fence *g2h_fence) +{ + int ret; + + XE_BUG_ON(g2h_len && g2h_fence); + XE_BUG_ON(num_g2h && g2h_fence); + XE_BUG_ON(g2h_len && !num_g2h); + XE_BUG_ON(!g2h_len && num_g2h); + lockdep_assert_held(&ct->lock); + + if (unlikely(ct->ctbs.h2g.broken)) { + ret = -EPIPE; + goto out; + } + + if (unlikely(!ct->enabled)) { + ret = -ENODEV; + goto out; + } + + if (g2h_fence) { + g2h_len = GUC_CTB_HXG_MSG_MAX_LEN; + num_g2h = 1; + + if (g2h_fence_needs_alloc(g2h_fence)) { + void *ptr; + + g2h_fence->seqno = (ct->fence_seqno++ & 0xffff); + init_waitqueue_head(&g2h_fence->wq); + ptr = xa_store(&ct->fence_lookup, + g2h_fence->seqno, + g2h_fence, GFP_ATOMIC); + if (IS_ERR(ptr)) { + ret = PTR_ERR(ptr); + goto out; + } + } + } + + xe_device_mem_access_get(ct_to_xe(ct)); +retry: + ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len); + if (unlikely(ret)) + goto put_wa; + + ret = h2g_write(ct, action, len, g2h_fence ? g2h_fence->seqno : 0, + !!g2h_fence); + if (unlikely(ret)) { + if (ret == -EAGAIN) + goto retry; + goto put_wa; + } + + g2h_reserve_space(ct, g2h_len, num_g2h); + xe_guc_notify(ct_to_guc(ct)); +put_wa: + xe_device_mem_access_put(ct_to_xe(ct)); +out: + + return ret; +} + +static void kick_reset(struct xe_guc_ct *ct) +{ + xe_gt_reset_async(ct_to_gt(ct)); +} + +static int dequeue_one_g2h(struct xe_guc_ct *ct); + +static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h, + struct g2h_fence *g2h_fence) +{ + struct drm_device *drm = &ct_to_xe(ct)->drm; + struct drm_printer p = drm_info_printer(drm->dev); + unsigned int sleep_period_ms = 1; + int ret; + + XE_BUG_ON(g2h_len && g2h_fence); + lockdep_assert_held(&ct->lock); + +try_again: + ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, + g2h_fence); + + /* + * We wait to try to restore credits for about 1 second before bailing. + * In the case of H2G credits we have no choice but just to wait for the + * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In + * the case of G2H we process any G2H in the channel, hopefully freeing + * credits as we consume the G2H messages. + */ + if (unlikely(ret == -EBUSY && + !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) { + struct guc_ctb *h2g = &ct->ctbs.h2g; + + if (sleep_period_ms == 1024) + goto broken; + + trace_xe_guc_ct_h2g_flow_control(h2g->head, h2g->tail, + h2g->size, h2g->space, + len + GUC_CTB_HDR_LEN); + msleep(sleep_period_ms); + sleep_period_ms <<= 1; + + goto try_again; + } else if (unlikely(ret == -EBUSY)) { + struct xe_device *xe = ct_to_xe(ct); + struct guc_ctb *g2h = &ct->ctbs.g2h; + + trace_xe_guc_ct_g2h_flow_control(g2h->head, + desc_read(xe, g2h, tail), + g2h->size, g2h->space, + g2h_fence ? + GUC_CTB_HXG_MSG_MAX_LEN : + g2h_len); + +#define g2h_avail(ct) \ + (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.head) + if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding || + g2h_avail(ct), HZ)) + goto broken; +#undef g2h_avail + + if (dequeue_one_g2h(ct) < 0) + goto broken; + + goto try_again; + } + + return ret; + +broken: + drm_err(drm, "No forward process on H2G, reset required"); + xe_guc_ct_print(ct, &p); + ct->ctbs.h2g.broken = true; + + return -EDEADLK; +} + +static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence) +{ + int ret; + + XE_BUG_ON(g2h_len && g2h_fence); + + mutex_lock(&ct->lock); + ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence); + mutex_unlock(&ct->lock); + + return ret; +} + +int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h) +{ + int ret; + + ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL); + if (ret == -EDEADLK) + kick_reset(ct); + + return ret; +} + +int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h) +{ + int ret; + + ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL); + if (ret == -EDEADLK) + kick_reset(ct); + + return ret; +} + +int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len) +{ + int ret; + + lockdep_assert_held(&ct->lock); + + ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL); + if (ret == -EDEADLK) + kick_reset(ct); + + return ret; +} + +/* + * Check if a GT reset is in progress or will occur and if GT reset brought the + * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset. + */ +static bool retry_failure(struct xe_guc_ct *ct, int ret) +{ + if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV)) + return false; + +#define ct_alive(ct) \ + (ct->enabled && !ct->ctbs.h2g.broken && !ct->ctbs.g2h.broken) + if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5)) + return false; +#undef ct_alive + + return true; +} + +static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 *response_buffer, bool no_fail) +{ + struct xe_device *xe = ct_to_xe(ct); + struct g2h_fence g2h_fence; + int ret = 0; + + /* + * We use a fence to implement blocking sends / receiving response data. + * The seqno of the fence is sent in the H2G, returned in the G2H, and + * an xarray is used as storage media with the seqno being to key. + * Fields in the fence hold success, failure, retry status and the + * response data. Safe to allocate on the stack as the xarray is the + * only reference and it cannot be present after this function exits. + */ +retry: + g2h_fence_init(&g2h_fence, response_buffer); +retry_same_fence: + ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence); + if (unlikely(ret == -ENOMEM)) { + void *ptr; + + /* Retry allocation /w GFP_KERNEL */ + ptr = xa_store(&ct->fence_lookup, + g2h_fence.seqno, + &g2h_fence, GFP_KERNEL); + if (IS_ERR(ptr)) { + return PTR_ERR(ptr); + } + + goto retry_same_fence; + } else if (unlikely(ret)) { + if (ret == -EDEADLK) + kick_reset(ct); + + if (no_fail && retry_failure(ct, ret)) + goto retry_same_fence; + + if (!g2h_fence_needs_alloc(&g2h_fence)) + xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno); + + return ret; + } + + ret = wait_event_timeout(g2h_fence.wq, g2h_fence.done, HZ); + if (!ret) { + drm_err(&xe->drm, "Timed out wait for G2H, fence %u, action %04x", + g2h_fence.seqno, action[0]); + xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno); + return -ETIME; + } + + if (g2h_fence.retry) { + drm_warn(&xe->drm, "Send retry, action 0x%04x, reason %d", + action[0], g2h_fence.reason); + goto retry; + } + if (g2h_fence.fail) { + drm_err(&xe->drm, "Send failed, action 0x%04x, error %d, hint %d", + action[0], g2h_fence.error, g2h_fence.hint); + ret = -EIO; + } + + return ret > 0 ? 0 : ret; +} + +int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 *response_buffer) +{ + return guc_ct_send_recv(ct, action, len, response_buffer, false); +} + +int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, + u32 len, u32 *response_buffer) +{ + return guc_ct_send_recv(ct, action, len, response_buffer, true); +} + +static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]); + + lockdep_assert_held(&ct->lock); + + switch (action) { + case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: + case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: + case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: + case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + g2h_release_space(ct, len); + } + + return 0; +} + +static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + struct xe_device *xe = ct_to_xe(ct); + u32 response_len = len - GUC_CTB_MSG_MIN_LEN; + u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]); + u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[1]); + struct g2h_fence *g2h_fence; + + lockdep_assert_held(&ct->lock); + + g2h_fence = xa_erase(&ct->fence_lookup, fence); + if (unlikely(!g2h_fence)) { + /* Don't tear down channel, as send could've timed out */ + drm_warn(&xe->drm, "G2H fence (%u) not found!\n", fence); + g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); + return 0; + } + + XE_WARN_ON(fence != g2h_fence->seqno); + + if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) { + g2h_fence->fail = true; + g2h_fence->error = + FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, msg[0]); + g2h_fence->hint = + FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, msg[0]); + } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) { + g2h_fence->retry = true; + g2h_fence->reason = + FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, msg[0]); + } else if (g2h_fence->response_buffer) { + g2h_fence->response_len = response_len; + memcpy(g2h_fence->response_buffer, msg + GUC_CTB_MSG_MIN_LEN, + response_len * sizeof(u32)); + } + + g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN); + + g2h_fence->done = true; + smp_mb(); + + wake_up(&g2h_fence->wq); + + return 0; +} + +static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + struct xe_device *xe = ct_to_xe(ct); + u32 header, hxg, origin, type; + int ret; + + lockdep_assert_held(&ct->lock); + + header = msg[0]; + hxg = msg[1]; + + origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg); + if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) { + drm_err(&xe->drm, + "G2H channel broken on read, origin=%d, reset required\n", + origin); + ct->ctbs.g2h.broken = true; + + return -EPROTO; + } + + type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg); + switch (type) { + case GUC_HXG_TYPE_EVENT: + ret = parse_g2h_event(ct, msg, len); + break; + case GUC_HXG_TYPE_RESPONSE_SUCCESS: + case GUC_HXG_TYPE_RESPONSE_FAILURE: + case GUC_HXG_TYPE_NO_RESPONSE_RETRY: + ret = parse_g2h_response(ct, msg, len); + break; + default: + drm_err(&xe->drm, + "G2H channel broken on read, type=%d, reset required\n", + type); + ct->ctbs.g2h.broken = true; + + ret = -EOPNOTSUPP; + } + + return ret; +} + +static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_guc *guc = ct_to_guc(ct); + u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]); + u32 *payload = msg + GUC_CTB_HXG_MSG_MIN_LEN; + u32 adj_len = len - GUC_CTB_HXG_MSG_MIN_LEN; + int ret = 0; + + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[1]) != GUC_HXG_TYPE_EVENT) + return 0; + + switch (action) { + case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE: + ret = xe_guc_sched_done_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE: + ret = xe_guc_deregister_done_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: + ret = xe_guc_engine_reset_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: + ret = xe_guc_engine_reset_failure_handler(guc, payload, + adj_len); + break; + case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: + /* Selftest only at the moment */ + break; + case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION: + case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE: + /* FIXME: Handle this */ + break; + case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: + ret = xe_guc_engine_memory_cat_error_handler(guc, payload, + adj_len); + break; + case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: + ret = xe_guc_pagefault_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + ret = xe_guc_tlb_invalidation_done_handler(guc, payload, + adj_len); + break; + case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY: + ret = xe_guc_access_counter_notify_handler(guc, payload, + adj_len); + break; + default: + drm_err(&xe->drm, "unexpected action 0x%04x\n", action); + } + + if (ret) + drm_err(&xe->drm, "action 0x%04x failed processing, ret=%d\n", + action, ret); + + return 0; +} + +static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path) +{ + struct xe_device *xe = ct_to_xe(ct); + struct guc_ctb *g2h = &ct->ctbs.g2h; + u32 tail, head, len; + s32 avail; + + lockdep_assert_held(&ct->fast_lock); + + if (!ct->enabled) + return -ENODEV; + + if (g2h->broken) + return -EPIPE; + + /* Calculate DW available to read */ + tail = desc_read(xe, g2h, tail); + avail = tail - g2h->head; + if (unlikely(avail == 0)) + return 0; + + if (avail < 0) + avail += g2h->size; + + /* Read header */ + xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->head, sizeof(u32)); + len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN; + if (len > avail) { + drm_err(&xe->drm, + "G2H channel broken on read, avail=%d, len=%d, reset required\n", + avail, len); + g2h->broken = true; + + return -EPROTO; + } + + head = (g2h->head + 1) % g2h->size; + avail = len - 1; + + /* Read G2H message */ + if (avail + head > g2h->size) { + u32 avail_til_wrap = g2h->size - head; + + xe_map_memcpy_from(xe, msg + 1, + &g2h->cmds, sizeof(u32) * head, + avail_til_wrap * sizeof(u32)); + xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap, + &g2h->cmds, 0, + (avail - avail_til_wrap) * sizeof(u32)); + } else { + xe_map_memcpy_from(xe, msg + 1, + &g2h->cmds, sizeof(u32) * head, + avail * sizeof(u32)); + } + + if (fast_path) { + if (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[1]) != GUC_HXG_TYPE_EVENT) + return 0; + + switch (FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1])) { + case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: + break; /* Process these in fast-path */ + default: + return 0; + } + } + + /* Update local / descriptor header */ + g2h->head = (head + avail) % g2h->size; + desc_write(xe, g2h, head, g2h->head); + + return len; +} + +static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len) +{ + struct xe_device *xe = ct_to_xe(ct); + struct xe_guc *guc = ct_to_guc(ct); + u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]); + u32 *payload = msg + GUC_CTB_HXG_MSG_MIN_LEN; + u32 adj_len = len - GUC_CTB_HXG_MSG_MIN_LEN; + int ret = 0; + + switch (action) { + case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: + ret = xe_guc_pagefault_handler(guc, payload, adj_len); + break; + case XE_GUC_ACTION_TLB_INVALIDATION_DONE: + __g2h_release_space(ct, len); + ret = xe_guc_tlb_invalidation_done_handler(guc, payload, + adj_len); + break; + default: + XE_WARN_ON("NOT_POSSIBLE"); + } + + if (ret) + drm_err(&xe->drm, "action 0x%04x failed processing, ret=%d\n", + action, ret); +} + +/** + * xe_guc_ct_fast_path - process critical G2H in the IRQ handler + * @ct: GuC CT object + * + * Anything related to page faults is critical for performance, process these + * critical G2H in the IRQ. This is safe as these handlers either just wake up + * waiters or queue another worker. + */ +void xe_guc_ct_fast_path(struct xe_guc_ct *ct) +{ + struct xe_device *xe = ct_to_xe(ct); + int len; + + if (!xe_device_in_fault_mode(xe) || !xe_device_mem_access_ongoing(xe)) + return; + + spin_lock(&ct->fast_lock); + do { + len = g2h_read(ct, ct->fast_msg, true); + if (len > 0) + g2h_fast_path(ct, ct->fast_msg, len); + } while (len > 0); + spin_unlock(&ct->fast_lock); +} + +/* Returns less than zero on error, 0 on done, 1 on more available */ +static int dequeue_one_g2h(struct xe_guc_ct *ct) +{ + int len; + int ret; + + lockdep_assert_held(&ct->lock); + + spin_lock_irq(&ct->fast_lock); + len = g2h_read(ct, ct->msg, false); + spin_unlock_irq(&ct->fast_lock); + if (len <= 0) + return len; + + ret = parse_g2h_msg(ct, ct->msg, len); + if (unlikely(ret < 0)) + return ret; + + ret = process_g2h_msg(ct, ct->msg, len); + if (unlikely(ret < 0)) + return ret; + + return 1; +} + +static void g2h_worker_func(struct work_struct *w) +{ + struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker); + int ret; + + xe_device_mem_access_get(ct_to_xe(ct)); + do { + mutex_lock(&ct->lock); + ret = dequeue_one_g2h(ct); + mutex_unlock(&ct->lock); + + if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) { + struct drm_device *drm = &ct_to_xe(ct)->drm; + struct drm_printer p = drm_info_printer(drm->dev); + + xe_guc_ct_print(ct, &p); + kick_reset(ct); + } + } while (ret == 1); + xe_device_mem_access_put(ct_to_xe(ct)); +} + +static void guc_ct_ctb_print(struct xe_device *xe, struct guc_ctb *ctb, + struct drm_printer *p) +{ + u32 head, tail; + + drm_printf(p, "\tsize: %d\n", ctb->size); + drm_printf(p, "\tresv_space: %d\n", ctb->resv_space); + drm_printf(p, "\thead: %d\n", ctb->head); + drm_printf(p, "\ttail: %d\n", ctb->tail); + drm_printf(p, "\tspace: %d\n", ctb->space); + drm_printf(p, "\tbroken: %d\n", ctb->broken); + + head = desc_read(xe, ctb, head); + tail = desc_read(xe, ctb, tail); + drm_printf(p, "\thead (memory): %d\n", head); + drm_printf(p, "\ttail (memory): %d\n", tail); + drm_printf(p, "\tstatus (memory): 0x%x\n", desc_read(xe, ctb, status)); + + if (head != tail) { + struct iosys_map map = + IOSYS_MAP_INIT_OFFSET(&ctb->cmds, head * sizeof(u32)); + + while (head != tail) { + drm_printf(p, "\tcmd[%d]: 0x%08x\n", head, + xe_map_rd(xe, &map, 0, u32)); + ++head; + if (head == ctb->size) { + head = 0; + map = ctb->cmds; + } else { + iosys_map_incr(&map, sizeof(u32)); + } + } + } +} + +void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p) +{ + if (ct->enabled) { + drm_puts(p, "\nH2G CTB (all sizes in DW):\n"); + guc_ct_ctb_print(ct_to_xe(ct), &ct->ctbs.h2g, p); + + drm_puts(p, "\nG2H CTB (all sizes in DW):\n"); + guc_ct_ctb_print(ct_to_xe(ct), &ct->ctbs.g2h, p); + drm_printf(p, "\tg2h outstanding: %d\n", ct->g2h_outstanding); + } else { + drm_puts(p, "\nCT disabled\n"); + } +} + +#ifdef XE_GUC_CT_SELFTEST +/* + * Disable G2H processing in IRQ handler to force xe_guc_ct_send to enter flow + * control if enough sent, 8k sends is enough. Verify forward process, verify + * credits expected values on exit. + */ +void xe_guc_ct_selftest(struct xe_guc_ct *ct, struct drm_printer *p) +{ + struct guc_ctb *g2h = &ct->ctbs.g2h; + u32 action[] = { XE_GUC_ACTION_SCHED_ENGINE_MODE_SET, 0, 0, 1, }; + u32 bad_action[] = { XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, 0, 0, }; + int ret; + int i; + + ct->suppress_irq_handler = true; + drm_puts(p, "Starting GuC CT selftest\n"); + + for (i = 0; i < 8192; ++i) { + ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 4, 1); + if (ret) { + drm_printf(p, "Aborted pass %d, ret %d\n", i, ret); + xe_guc_ct_print(ct, p); + break; + } + } + + ct->suppress_irq_handler = false; + if (!ret) { + xe_guc_ct_irq_handler(ct); + msleep(200); + if (g2h->space != + CIRC_SPACE(0, 0, g2h->size) - g2h->resv_space) { + drm_printf(p, "Mismatch on space %d, %d\n", + g2h->space, + CIRC_SPACE(0, 0, g2h->size) - + g2h->resv_space); + ret = -EIO; + } + if (ct->g2h_outstanding) { + drm_printf(p, "Outstanding G2H, %d\n", + ct->g2h_outstanding); + ret = -EIO; + } + } + + /* Check failure path for blocking CTs too */ + xe_guc_ct_send_block(ct, bad_action, ARRAY_SIZE(bad_action)); + if (g2h->space != + CIRC_SPACE(0, 0, g2h->size) - g2h->resv_space) { + drm_printf(p, "Mismatch on space %d, %d\n", + g2h->space, + CIRC_SPACE(0, 0, g2h->size) - + g2h->resv_space); + ret = -EIO; + } + if (ct->g2h_outstanding) { + drm_printf(p, "Outstanding G2H, %d\n", + ct->g2h_outstanding); + ret = -EIO; + } + + drm_printf(p, "GuC CT selftest done - %s\n", ret ? "FAIL" : "PASS"); +} +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ct.h b/drivers/gpu/drm/xe/xe_guc_ct.h new file mode 100644 index 000000000000..49fb74f91e4d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ct.h @@ -0,0 +1,62 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_CT_H_ +#define _XE_GUC_CT_H_ + +#include "xe_guc_ct_types.h" + +struct drm_printer; + +int xe_guc_ct_init(struct xe_guc_ct *ct); +int xe_guc_ct_enable(struct xe_guc_ct *ct); +void xe_guc_ct_disable(struct xe_guc_ct *ct); +void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p); +void xe_guc_ct_fast_path(struct xe_guc_ct *ct); + +static inline void xe_guc_ct_irq_handler(struct xe_guc_ct *ct) +{ + wake_up_all(&ct->wq); +#ifdef XE_GUC_CT_SELFTEST + if (!ct->suppress_irq_handler && ct->enabled) + queue_work(system_unbound_wq, &ct->g2h_worker); +#else + if (ct->enabled) + queue_work(system_unbound_wq, &ct->g2h_worker); +#endif + xe_guc_ct_fast_path(ct); +} + +/* Basic CT send / receives */ +int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h); +int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 g2h_len, u32 num_g2h); +int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len, + u32 *response_buffer); +static inline int +xe_guc_ct_send_block(struct xe_guc_ct *ct, const u32 *action, u32 len) +{ + return xe_guc_ct_send_recv(ct, action, len, NULL); +} + +/* This is only version of the send CT you can call from a G2H handler */ +int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, + u32 len); + +/* Can't fail because a GT reset is in progress */ +int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action, + u32 len, u32 *response_buffer); +static inline int +xe_guc_ct_send_block_no_fail(struct xe_guc_ct *ct, const u32 *action, u32 len) +{ + return xe_guc_ct_send_recv_no_fail(ct, action, len, NULL); +} + +#ifdef XE_GUC_CT_SELFTEST +void xe_guc_ct_selftest(struct xe_guc_ct *ct, struct drm_printer *p); +#endif + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_ct_types.h b/drivers/gpu/drm/xe/xe_guc_ct_types.h new file mode 100644 index 000000000000..17b148bf3735 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_ct_types.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_CT_TYPES_H_ +#define _XE_GUC_CT_TYPES_H_ + +#include +#include +#include +#include +#include + +#include "abi/guc_communication_ctb_abi.h" + +#define XE_GUC_CT_SELFTEST + +struct xe_bo; + +/** + * struct guc_ctb - GuC command transport buffer (CTB) + */ +struct guc_ctb { + /** @desc: dma buffer map for CTB descriptor */ + struct iosys_map desc; + /** @cmds: dma buffer map for CTB commands */ + struct iosys_map cmds; + /** @size: size of CTB commands (DW) */ + u32 size; + /** @resv_space: reserved space of CTB commands (DW) */ + u32 resv_space; + /** @head: head of CTB commands (DW) */ + u32 head; + /** @tail: tail of CTB commands (DW) */ + u32 tail; + /** @space: space in CTB commands (DW) */ + u32 space; + /** @broken: channel broken */ + bool broken; +}; + +/** + * struct xe_guc_ct - GuC command transport (CT) layer + * + * Includes a pair of CT buffers for bi-directional communication and tracking + * for the H2G and G2H requests sent and received through the buffers. + */ +struct xe_guc_ct { + /** @bo: XE BO for CT */ + struct xe_bo *bo; + /** @lock: protects everything in CT layer */ + struct mutex lock; + /** @fast_lock: protects G2H channel and credits */ + spinlock_t fast_lock; + /** @ctbs: buffers for sending and receiving commands */ + struct { + /** @send: Host to GuC (H2G, send) channel */ + struct guc_ctb h2g; + /** @recv: GuC to Host (G2H, receive) channel */ + struct guc_ctb g2h; + } ctbs; + /** @g2h_outstanding: number of outstanding G2H */ + u32 g2h_outstanding; + /** @g2h_worker: worker to process G2H messages */ + struct work_struct g2h_worker; + /** @enabled: CT enabled */ + bool enabled; + /** @fence_seqno: G2H fence seqno - 16 bits used by CT */ + u32 fence_seqno; + /** @fence_context: context for G2H fence */ + u64 fence_context; + /** @fence_lookup: G2H fence lookup */ + struct xarray fence_lookup; + /** @wq: wait queue used for reliable CT sends and freeing G2H credits */ + wait_queue_head_t wq; +#ifdef XE_GUC_CT_SELFTEST + /** @suppress_irq_handler: force flow control to sender */ + bool suppress_irq_handler; +#endif + /** @msg: Message buffer */ + u32 msg[GUC_CTB_MSG_MAX_LEN]; + /** @fast_msg: Message buffer */ + u32 fast_msg[GUC_CTB_MSG_MAX_LEN]; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_debugfs.c b/drivers/gpu/drm/xe/xe_guc_debugfs.c new file mode 100644 index 000000000000..916e9633b322 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_debugfs.c @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_guc_debugfs.h" +#include "xe_guc_log.h" +#include "xe_macros.h" + +static struct xe_gt * +guc_to_gt(struct xe_guc *guc) +{ + return container_of(guc, struct xe_gt, uc.guc); +} + +static struct xe_device * +guc_to_xe(struct xe_guc *guc) +{ + return gt_to_xe(guc_to_gt(guc)); +} + +static struct xe_guc *node_to_guc(struct drm_info_node *node) +{ + return node->info_ent->data; +} + +static int guc_info(struct seq_file *m, void *data) +{ + struct xe_guc *guc = node_to_guc(m->private); + struct xe_device *xe = guc_to_xe(guc); + struct drm_printer p = drm_seq_file_printer(m); + + xe_device_mem_access_get(xe); + xe_guc_print_info(guc, &p); + xe_device_mem_access_put(xe); + + return 0; +} + +static int guc_log(struct seq_file *m, void *data) +{ + struct xe_guc *guc = node_to_guc(m->private); + struct xe_device *xe = guc_to_xe(guc); + struct drm_printer p = drm_seq_file_printer(m); + + xe_device_mem_access_get(xe); + xe_guc_log_print(&guc->log, &p); + xe_device_mem_access_put(xe); + + return 0; +} + +#ifdef XE_GUC_CT_SELFTEST +static int guc_ct_selftest(struct seq_file *m, void *data) +{ + struct xe_guc *guc = node_to_guc(m->private); + struct xe_device *xe = guc_to_xe(guc); + struct drm_printer p = drm_seq_file_printer(m); + + xe_device_mem_access_get(xe); + xe_guc_ct_selftest(&guc->ct, &p); + xe_device_mem_access_put(xe); + + return 0; +} +#endif + +static const struct drm_info_list debugfs_list[] = { + {"guc_info", guc_info, 0}, + {"guc_log", guc_log, 0}, +#ifdef XE_GUC_CT_SELFTEST + {"guc_ct_selftest", guc_ct_selftest, 0}, +#endif +}; + +void xe_guc_debugfs_register(struct xe_guc *guc, struct dentry *parent) +{ + struct drm_minor *minor = guc_to_xe(guc)->drm.primary; + struct drm_info_list *local; + int i; + +#define DEBUGFS_SIZE ARRAY_SIZE(debugfs_list) * sizeof(struct drm_info_list) + local = drmm_kmalloc(&guc_to_xe(guc)->drm, DEBUGFS_SIZE, GFP_KERNEL); + if (!local) { + XE_WARN_ON("Couldn't allocate memory"); + return; + } + + memcpy(local, debugfs_list, DEBUGFS_SIZE); +#undef DEBUGFS_SIZE + + for (i = 0; i < ARRAY_SIZE(debugfs_list); ++i) + local[i].data = guc; + + drm_debugfs_create_files(local, + ARRAY_SIZE(debugfs_list), + parent, minor); +} diff --git a/drivers/gpu/drm/xe/xe_guc_debugfs.h b/drivers/gpu/drm/xe/xe_guc_debugfs.h new file mode 100644 index 000000000000..4756dff26fca --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_DEBUGFS_H_ +#define _XE_GUC_DEBUGFS_H_ + +struct dentry; +struct xe_guc; + +void xe_guc_debugfs_register(struct xe_guc *guc, struct dentry *parent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_engine_types.h b/drivers/gpu/drm/xe/xe_guc_engine_types.h new file mode 100644 index 000000000000..512615d1ce8c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_engine_types.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_ENGINE_TYPES_H_ +#define _XE_GUC_ENGINE_TYPES_H_ + +#include +#include + +#include "xe_gpu_scheduler_types.h" + +struct dma_fence; +struct xe_engine; + +/** + * struct xe_guc_engine - GuC specific state for an xe_engine + */ +struct xe_guc_engine { + /** @engine: Backpointer to parent xe_engine */ + struct xe_engine *engine; + /** @sched: GPU scheduler for this xe_engine */ + struct xe_gpu_scheduler sched; + /** @entity: Scheduler entity for this xe_engine */ + struct xe_sched_entity entity; + /** + * @static_msgs: Static messages for this xe_engine, used when a message + * needs to sent through the GPU scheduler but memory allocations are + * not allowed. + */ +#define MAX_STATIC_MSG_TYPE 3 + struct xe_sched_msg static_msgs[MAX_STATIC_MSG_TYPE]; + /** @fini_async: do final fini async from this worker */ + struct work_struct fini_async; + /** @resume_time: time of last resume */ + u64 resume_time; + /** @state: GuC specific state for this xe_engine */ + atomic_t state; + /** @wqi_head: work queue item tail */ + u32 wqi_head; + /** @wqi_tail: work queue item tail */ + u32 wqi_tail; + /** @id: GuC id for this xe_engine */ + u16 id; + /** @suspend_wait: wait queue used to wait on pending suspends */ + wait_queue_head_t suspend_wait; + /** @suspend_pending: a suspend of the engine is pending */ + bool suspend_pending; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h new file mode 100644 index 000000000000..f562404a6cf7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -0,0 +1,392 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_FWIF_H +#define _XE_GUC_FWIF_H + +#include + +#include "abi/guc_actions_abi.h" +#include "abi/guc_actions_slpc_abi.h" +#include "abi/guc_errors_abi.h" +#include "abi/guc_communication_mmio_abi.h" +#include "abi/guc_communication_ctb_abi.h" +#include "abi/guc_klvs_abi.h" +#include "abi/guc_messages_abi.h" + +#define G2H_LEN_DW_SCHED_CONTEXT_MODE_SET 4 +#define G2H_LEN_DW_DEREGISTER_CONTEXT 3 +#define G2H_LEN_DW_TLB_INVALIDATE 3 + +#define GUC_CONTEXT_DISABLE 0 +#define GUC_CONTEXT_ENABLE 1 + +#define GUC_CLIENT_PRIORITY_KMD_HIGH 0 +#define GUC_CLIENT_PRIORITY_HIGH 1 +#define GUC_CLIENT_PRIORITY_KMD_NORMAL 2 +#define GUC_CLIENT_PRIORITY_NORMAL 3 +#define GUC_CLIENT_PRIORITY_NUM 4 + +#define GUC_RENDER_ENGINE 0 +#define GUC_VIDEO_ENGINE 1 +#define GUC_BLITTER_ENGINE 2 +#define GUC_VIDEOENHANCE_ENGINE 3 +#define GUC_VIDEO_ENGINE2 4 +#define GUC_MAX_ENGINES_NUM (GUC_VIDEO_ENGINE2 + 1) + +#define GUC_RENDER_CLASS 0 +#define GUC_VIDEO_CLASS 1 +#define GUC_VIDEOENHANCE_CLASS 2 +#define GUC_BLITTER_CLASS 3 +#define GUC_COMPUTE_CLASS 4 +#define GUC_GSC_OTHER_CLASS 5 +#define GUC_LAST_ENGINE_CLASS GUC_GSC_OTHER_CLASS +#define GUC_MAX_ENGINE_CLASSES 16 +#define GUC_MAX_INSTANCES_PER_CLASS 32 + +/* Work item for submitting workloads into work queue of GuC. */ +#define WQ_STATUS_ACTIVE 1 +#define WQ_STATUS_SUSPENDED 2 +#define WQ_STATUS_CMD_ERROR 3 +#define WQ_STATUS_ENGINE_ID_NOT_USED 4 +#define WQ_STATUS_SUSPENDED_FROM_RESET 5 +#define WQ_TYPE_NOOP 0x4 +#define WQ_TYPE_MULTI_LRC 0x5 +#define WQ_TYPE_MASK GENMASK(7, 0) +#define WQ_LEN_MASK GENMASK(26, 16) + +#define WQ_GUC_ID_MASK GENMASK(15, 0) +#define WQ_RING_TAIL_MASK GENMASK(28, 18) + +struct guc_wq_item { + u32 header; + u32 context_desc; + u32 submit_element_info; + u32 fence_id; +} __packed; + +struct guc_sched_wq_desc { + u32 head; + u32 tail; + u32 error_offset; + u32 wq_status; + u32 reserved[28]; +} __packed; + +/* Helper for context registration H2G */ +struct guc_ctxt_registration_info { + u32 flags; + u32 context_idx; + u32 engine_class; + u32 engine_submit_mask; + u32 wq_desc_lo; + u32 wq_desc_hi; + u32 wq_base_lo; + u32 wq_base_hi; + u32 wq_size; + u32 hwlrca_lo; + u32 hwlrca_hi; +}; +#define CONTEXT_REGISTRATION_FLAG_KMD BIT(0) + +/* 32-bit KLV structure as used by policy updates and others */ +struct guc_klv_generic_dw_t { + u32 kl; + u32 value; +} __packed; + +/* Format of the UPDATE_CONTEXT_POLICIES H2G data packet */ +struct guc_update_engine_policy_header { + u32 action; + u32 guc_id; +} __packed; + +struct guc_update_engine_policy { + struct guc_update_engine_policy_header header; + struct guc_klv_generic_dw_t klv[GUC_CONTEXT_POLICIES_KLV_NUM_IDS]; +} __packed; + +/* GUC_CTL_* - Parameters for loading the GuC */ +#define GUC_CTL_LOG_PARAMS 0 +#define GUC_LOG_VALID BIT(0) +#define GUC_LOG_NOTIFY_ON_HALF_FULL BIT(1) +#define GUC_LOG_CAPTURE_ALLOC_UNITS BIT(2) +#define GUC_LOG_LOG_ALLOC_UNITS BIT(3) +#define GUC_LOG_CRASH_SHIFT 4 +#define GUC_LOG_CRASH_MASK (0x3 << GUC_LOG_CRASH_SHIFT) +#define GUC_LOG_DEBUG_SHIFT 6 +#define GUC_LOG_DEBUG_MASK (0xF << GUC_LOG_DEBUG_SHIFT) +#define GUC_LOG_CAPTURE_SHIFT 10 +#define GUC_LOG_CAPTURE_MASK (0x3 << GUC_LOG_CAPTURE_SHIFT) +#define GUC_LOG_BUF_ADDR_SHIFT 12 + +#define GUC_CTL_WA 1 +#define GUC_WA_GAM_CREDITS BIT(10) +#define GUC_WA_DUAL_QUEUE BIT(11) +#define GUC_WA_RCS_RESET_BEFORE_RC6 BIT(13) +#define GUC_WA_CONTEXT_ISOLATION BIT(15) +#define GUC_WA_PRE_PARSER BIT(14) +#define GUC_WA_HOLD_CCS_SWITCHOUT BIT(17) +#define GUC_WA_POLLCS BIT(18) +#define GUC_WA_RENDER_RST_RC6_EXIT BIT(19) +#define GUC_WA_RCS_REGS_IN_CCS_REGS_LIST BIT(21) + +#define GUC_CTL_FEATURE 2 +#define GUC_CTL_ENABLE_SLPC BIT(2) +#define GUC_CTL_DISABLE_SCHEDULER BIT(14) + +#define GUC_CTL_DEBUG 3 +#define GUC_LOG_VERBOSITY_SHIFT 0 +#define GUC_LOG_VERBOSITY_LOW (0 << GUC_LOG_VERBOSITY_SHIFT) +#define GUC_LOG_VERBOSITY_MED (1 << GUC_LOG_VERBOSITY_SHIFT) +#define GUC_LOG_VERBOSITY_HIGH (2 << GUC_LOG_VERBOSITY_SHIFT) +#define GUC_LOG_VERBOSITY_ULTRA (3 << GUC_LOG_VERBOSITY_SHIFT) +#define GUC_LOG_VERBOSITY_MIN 0 +#define GUC_LOG_VERBOSITY_MAX 3 +#define GUC_LOG_VERBOSITY_MASK 0x0000000f +#define GUC_LOG_DESTINATION_MASK (3 << 4) +#define GUC_LOG_DISABLED (1 << 6) +#define GUC_PROFILE_ENABLED (1 << 7) + +#define GUC_CTL_ADS 4 +#define GUC_ADS_ADDR_SHIFT 1 +#define GUC_ADS_ADDR_MASK (0xFFFFF << GUC_ADS_ADDR_SHIFT) + +#define GUC_CTL_DEVID 5 + +#define GUC_CTL_MAX_DWORDS 14 + +/* Scheduling policy settings */ + +#define GLOBAL_POLICY_MAX_NUM_WI 15 + +/* Don't reset an engine upon preemption failure */ +#define GLOBAL_POLICY_DISABLE_ENGINE_RESET BIT(0) + +#define GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US 500000 + +struct guc_policies { + u32 submission_queue_depth[GUC_MAX_ENGINE_CLASSES]; + /* In micro seconds. How much time to allow before DPC processing is + * called back via interrupt (to prevent DPC queue drain starving). + * Typically 1000s of micro seconds (example only, not granularity). */ + u32 dpc_promote_time; + + /* Must be set to take these new values. */ + u32 is_valid; + + /* Max number of WIs to process per call. A large value may keep CS + * idle. */ + u32 max_num_work_items; + + u32 global_flags; + u32 reserved[4]; +} __packed; + +/* GuC MMIO reg state struct */ +struct guc_mmio_reg { + u32 offset; + u32 value; + u32 flags; + u32 mask; +#define GUC_REGSET_MASKED BIT(0) +#define GUC_REGSET_MASKED_WITH_VALUE BIT(2) +#define GUC_REGSET_RESTORE_ONLY BIT(3) +} __packed; + +/* GuC register sets */ +struct guc_mmio_reg_set { + u32 address; + u16 count; + u16 reserved; +} __packed; + +/* Generic GT SysInfo data types */ +#define GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED 0 +#define GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK 1 +#define GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI 2 +#define GUC_GENERIC_GT_SYSINFO_MAX 16 + +/* HW info */ +struct guc_gt_system_info { + u8 mapping_table[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; + u32 engine_enabled_masks[GUC_MAX_ENGINE_CLASSES]; + u32 generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_MAX]; +} __packed; + +enum { + GUC_CAPTURE_LIST_INDEX_PF = 0, + GUC_CAPTURE_LIST_INDEX_VF = 1, + GUC_CAPTURE_LIST_INDEX_MAX = 2, +}; + +/* GuC Additional Data Struct */ +struct guc_ads { + struct guc_mmio_reg_set reg_state_list[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; + u32 reserved0; + u32 scheduler_policies; + u32 gt_system_info; + u32 reserved1; + u32 control_data; + u32 golden_context_lrca[GUC_MAX_ENGINE_CLASSES]; + u32 eng_state_size[GUC_MAX_ENGINE_CLASSES]; + u32 private_data; + u32 um_init_data; + u32 capture_instance[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES]; + u32 capture_class[GUC_CAPTURE_LIST_INDEX_MAX][GUC_MAX_ENGINE_CLASSES]; + u32 capture_global[GUC_CAPTURE_LIST_INDEX_MAX]; + u32 reserved[14]; +} __packed; + +/* Engine usage stats */ +struct guc_engine_usage_record { + u32 current_context_index; + u32 last_switch_in_stamp; + u32 reserved0; + u32 total_runtime; + u32 reserved1[4]; +} __packed; + +struct guc_engine_usage { + struct guc_engine_usage_record engines[GUC_MAX_ENGINE_CLASSES][GUC_MAX_INSTANCES_PER_CLASS]; +} __packed; + +/* This action will be programmed in C1BC - SOFT_SCRATCH_15_REG */ +enum xe_guc_recv_message { + XE_GUC_RECV_MSG_CRASH_DUMP_POSTED = BIT(1), + XE_GUC_RECV_MSG_EXCEPTION = BIT(30), +}; + +/* Page fault structures */ +struct access_counter_desc { + u32 dw0; +#define ACCESS_COUNTER_TYPE BIT(0) +#define ACCESS_COUNTER_SUBG_LO GENMASK(31, 1) + + u32 dw1; +#define ACCESS_COUNTER_SUBG_HI BIT(0) +#define ACCESS_COUNTER_RSVD0 GENMASK(2, 1) +#define ACCESS_COUNTER_ENG_INSTANCE GENMASK(8, 3) +#define ACCESS_COUNTER_ENG_CLASS GENMASK(11, 9) +#define ACCESS_COUNTER_ASID GENMASK(31, 12) + + u32 dw2; +#define ACCESS_COUNTER_VFID GENMASK(5, 0) +#define ACCESS_COUNTER_RSVD1 GENMASK(7, 6) +#define ACCESS_COUNTER_GRANULARITY GENMASK(10, 8) +#define ACCESS_COUNTER_RSVD2 GENMASK(16, 11) +#define ACCESS_COUNTER_VIRTUAL_ADDR_RANGE_LO GENMASK(31, 17) + + u32 dw3; +#define ACCESS_COUNTER_VIRTUAL_ADDR_RANGE_HI GENMASK(31, 0) +} __packed; + +enum guc_um_queue_type { + GUC_UM_HW_QUEUE_PAGE_FAULT = 0, + GUC_UM_HW_QUEUE_PAGE_FAULT_RESPONSE, + GUC_UM_HW_QUEUE_ACCESS_COUNTER, + GUC_UM_HW_QUEUE_MAX +}; + +struct guc_um_queue_params { + u64 base_dpa; + u32 base_ggtt_address; + u32 size_in_bytes; + u32 rsvd[4]; +} __packed; + +struct guc_um_init_params { + u64 page_response_timeout_in_us; + u32 rsvd[6]; + struct guc_um_queue_params queue_params[GUC_UM_HW_QUEUE_MAX]; +} __packed; + +enum xe_guc_fault_reply_type { + PFR_ACCESS = 0, + PFR_ENGINE, + PFR_VFID, + PFR_ALL, + PFR_INVALID +}; + +enum xe_guc_response_desc_type { + TLB_INVALIDATION_DESC = 0, + FAULT_RESPONSE_DESC +}; + +struct xe_guc_pagefault_desc { + u32 dw0; +#define PFD_FAULT_LEVEL GENMASK(2, 0) +#define PFD_SRC_ID GENMASK(10, 3) +#define PFD_RSVD_0 GENMASK(17, 11) +#define XE2_PFD_TRVA_FAULT BIT(18) +#define PFD_ENG_INSTANCE GENMASK(24, 19) +#define PFD_ENG_CLASS GENMASK(27, 25) +#define PFD_PDATA_LO GENMASK(31, 28) + + u32 dw1; +#define PFD_PDATA_HI GENMASK(11, 0) +#define PFD_PDATA_HI_SHIFT 4 +#define PFD_ASID GENMASK(31, 12) + + u32 dw2; +#define PFD_ACCESS_TYPE GENMASK(1, 0) +#define PFD_FAULT_TYPE GENMASK(3, 2) +#define PFD_VFID GENMASK(9, 4) +#define PFD_RSVD_1 GENMASK(11, 10) +#define PFD_VIRTUAL_ADDR_LO GENMASK(31, 12) +#define PFD_VIRTUAL_ADDR_LO_SHIFT 12 + + u32 dw3; +#define PFD_VIRTUAL_ADDR_HI GENMASK(31, 0) +#define PFD_VIRTUAL_ADDR_HI_SHIFT 32 +} __packed; + +struct xe_guc_pagefault_reply { + u32 dw0; +#define PFR_VALID BIT(0) +#define PFR_SUCCESS BIT(1) +#define PFR_REPLY GENMASK(4, 2) +#define PFR_RSVD_0 GENMASK(9, 5) +#define PFR_DESC_TYPE GENMASK(11, 10) +#define PFR_ASID GENMASK(31, 12) + + u32 dw1; +#define PFR_VFID GENMASK(5, 0) +#define PFR_RSVD_1 BIT(6) +#define PFR_ENG_INSTANCE GENMASK(12, 7) +#define PFR_ENG_CLASS GENMASK(15, 13) +#define PFR_PDATA GENMASK(31, 16) + + u32 dw2; +#define PFR_RSVD_2 GENMASK(31, 0) +} __packed; + +struct xe_guc_acc_desc { + u32 dw0; +#define ACC_TYPE BIT(0) +#define ACC_TRIGGER 0 +#define ACC_NOTIFY 1 +#define ACC_SUBG_LO GENMASK(31, 1) + + u32 dw1; +#define ACC_SUBG_HI BIT(0) +#define ACC_RSVD0 GENMASK(2, 1) +#define ACC_ENG_INSTANCE GENMASK(8, 3) +#define ACC_ENG_CLASS GENMASK(11, 9) +#define ACC_ASID GENMASK(31, 12) + + u32 dw2; +#define ACC_VFID GENMASK(5, 0) +#define ACC_RSVD1 GENMASK(7, 6) +#define ACC_GRANULARITY GENMASK(10, 8) +#define ACC_RSVD2 GENMASK(16, 11) +#define ACC_VIRTUAL_ADDR_RANGE_LO GENMASK(31, 17) + + u32 dw3; +#define ACC_VIRTUAL_ADDR_RANGE_HI GENMASK(31, 0) +} __packed; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_hwconfig.c b/drivers/gpu/drm/xe/xe_guc_hwconfig.c new file mode 100644 index 000000000000..8dfd48f71a7c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_hwconfig.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_guc_hwconfig.h" +#include "xe_map.h" + +static struct xe_gt * +guc_to_gt(struct xe_guc *guc) +{ + return container_of(guc, struct xe_gt, uc.guc); +} + +static struct xe_device * +guc_to_xe(struct xe_guc *guc) +{ + return gt_to_xe(guc_to_gt(guc)); +} + +static int send_get_hwconfig(struct xe_guc *guc, u32 ggtt_addr, u32 size) +{ + u32 action[] = { + XE_GUC_ACTION_GET_HWCONFIG, + lower_32_bits(ggtt_addr), + upper_32_bits(ggtt_addr), + size, + }; + + return xe_guc_send_mmio(guc, action, ARRAY_SIZE(action)); +} + +static int guc_hwconfig_size(struct xe_guc *guc, u32 *size) +{ + int ret = send_get_hwconfig(guc, 0, 0); + + if (ret < 0) + return ret; + + *size = ret; + return 0; +} + +static int guc_hwconfig_copy(struct xe_guc *guc) +{ + int ret = send_get_hwconfig(guc, xe_bo_ggtt_addr(guc->hwconfig.bo), + guc->hwconfig.size); + + if (ret < 0) + return ret; + + return 0; +} + +static void guc_hwconfig_fini(struct drm_device *drm, void *arg) +{ + struct xe_guc *guc = arg; + + xe_bo_unpin_map_no_vm(guc->hwconfig.bo); +} + +int xe_guc_hwconfig_init(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + struct xe_bo *bo; + u32 size; + int err; + + /* Initialization already done */ + if (guc->hwconfig.bo) + return 0; + + /* + * All hwconfig the same across GTs so only GT0 needs to be configured + */ + if (gt->info.id != XE_GT0) + return 0; + + /* ADL_P, DG2+ supports hwconfig table */ + if (GRAPHICS_VERx100(xe) < 1255 && xe->info.platform != XE_ALDERLAKE_P) + return 0; + + err = guc_hwconfig_size(guc, &size); + if (err) + return err; + if (!size) + return -EINVAL; + + bo = xe_bo_create_pin_map(xe, gt, NULL, PAGE_ALIGN(size), + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_GGTT_BIT); + if (IS_ERR(bo)) + return PTR_ERR(bo); + guc->hwconfig.bo = bo; + guc->hwconfig.size = size; + + err = drmm_add_action_or_reset(&xe->drm, guc_hwconfig_fini, guc); + if (err) + return err; + + return guc_hwconfig_copy(guc); +} + +u32 xe_guc_hwconfig_size(struct xe_guc *guc) +{ + return !guc->hwconfig.bo ? 0 : guc->hwconfig.size; +} + +void xe_guc_hwconfig_copy(struct xe_guc *guc, void *dst) +{ + struct xe_device *xe = guc_to_xe(guc); + + XE_BUG_ON(!guc->hwconfig.bo); + + xe_map_memcpy_from(xe, dst, &guc->hwconfig.bo->vmap, 0, + guc->hwconfig.size); +} diff --git a/drivers/gpu/drm/xe/xe_guc_hwconfig.h b/drivers/gpu/drm/xe/xe_guc_hwconfig.h new file mode 100644 index 000000000000..b5794d641900 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_hwconfig.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_HWCONFIG_H_ +#define _XE_GUC_HWCONFIG_H_ + +#include + +struct xe_guc; + +int xe_guc_hwconfig_init(struct xe_guc *guc); +u32 xe_guc_hwconfig_size(struct xe_guc *guc); +void xe_guc_hwconfig_copy(struct xe_guc *guc, void *dst); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c new file mode 100644 index 000000000000..7ec1b2bb1f8e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -0,0 +1,109 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_bo.h" +#include "xe_gt.h" +#include "xe_guc_log.h" +#include "xe_map.h" +#include "xe_module.h" + +static struct xe_gt * +log_to_gt(struct xe_guc_log *log) +{ + return container_of(log, struct xe_gt, uc.guc.log); +} + +static struct xe_device * +log_to_xe(struct xe_guc_log *log) +{ + return gt_to_xe(log_to_gt(log)); +} + +static size_t guc_log_size(void) +{ + /* + * GuC Log buffer Layout + * + * +===============================+ 00B + * | Crash dump state header | + * +-------------------------------+ 32B + * | Debug state header | + * +-------------------------------+ 64B + * | Capture state header | + * +-------------------------------+ 96B + * | | + * +===============================+ PAGE_SIZE (4KB) + * | Crash Dump logs | + * +===============================+ + CRASH_SIZE + * | Debug logs | + * +===============================+ + DEBUG_SIZE + * | Capture logs | + * +===============================+ + CAPTURE_SIZE + */ + return PAGE_SIZE + CRASH_BUFFER_SIZE + DEBUG_BUFFER_SIZE + + CAPTURE_BUFFER_SIZE; +} + +void xe_guc_log_print(struct xe_guc_log *log, struct drm_printer *p) +{ + struct xe_device *xe = log_to_xe(log); + size_t size; + int i, j; + + XE_BUG_ON(!log->bo); + + size = log->bo->size; + +#define DW_PER_READ 128 + XE_BUG_ON(size % (DW_PER_READ * sizeof(u32))); + for (i = 0; i < size / sizeof(u32); i += DW_PER_READ) { + u32 read[DW_PER_READ]; + + xe_map_memcpy_from(xe, read, &log->bo->vmap, i * sizeof(u32), + DW_PER_READ * sizeof(u32)); +#define DW_PER_PRINT 4 + for (j = 0; j < DW_PER_READ / DW_PER_PRINT; ++j) { + u32 *print = read + j * DW_PER_PRINT; + + drm_printf(p, "0x%08x 0x%08x 0x%08x 0x%08x\n", + *(print + 0), *(print + 1), + *(print + 2), *(print + 3)); + } + } +} + +static void guc_log_fini(struct drm_device *drm, void *arg) +{ + struct xe_guc_log *log = arg; + + xe_bo_unpin_map_no_vm(log->bo); +} + +int xe_guc_log_init(struct xe_guc_log *log) +{ + struct xe_device *xe = log_to_xe(log); + struct xe_gt *gt = log_to_gt(log); + struct xe_bo *bo; + int err; + + bo = xe_bo_create_pin_map(xe, gt, NULL, guc_log_size(), + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_GGTT_BIT); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + xe_map_memset(xe, &bo->vmap, 0, 0, guc_log_size()); + log->bo = bo; + log->level = xe_guc_log_level; + + err = drmm_add_action_or_reset(&xe->drm, guc_log_fini, log); + if (err) + return err; + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_guc_log.h b/drivers/gpu/drm/xe/xe_guc_log.h new file mode 100644 index 000000000000..2d25ab28b4b3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_log.h @@ -0,0 +1,48 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_LOG_H_ +#define _XE_GUC_LOG_H_ + +#include "xe_guc_log_types.h" + +struct drm_printer; + +#if IS_ENABLED(CONFIG_DRM_XE_LARGE_GUC_BUFFER) +#define CRASH_BUFFER_SIZE SZ_1M +#define DEBUG_BUFFER_SIZE SZ_8M +#define CAPTURE_BUFFER_SIZE SZ_2M +#else +#define CRASH_BUFFER_SIZE SZ_8K +#define DEBUG_BUFFER_SIZE SZ_64K +#define CAPTURE_BUFFER_SIZE SZ_16K +#endif +/* + * While we're using plain log level in i915, GuC controls are much more... + * "elaborate"? We have a couple of bits for verbosity, separate bit for actual + * log enabling, and separate bit for default logging - which "conveniently" + * ignores the enable bit. + */ +#define GUC_LOG_LEVEL_DISABLED 0 +#define GUC_LOG_LEVEL_NON_VERBOSE 1 +#define GUC_LOG_LEVEL_IS_ENABLED(x) ((x) > GUC_LOG_LEVEL_DISABLED) +#define GUC_LOG_LEVEL_IS_VERBOSE(x) ((x) > GUC_LOG_LEVEL_NON_VERBOSE) +#define GUC_LOG_LEVEL_TO_VERBOSITY(x) ({ \ + typeof(x) _x = (x); \ + GUC_LOG_LEVEL_IS_VERBOSE(_x) ? _x - 2 : 0; \ +}) +#define GUC_VERBOSITY_TO_LOG_LEVEL(x) ((x) + 2) +#define GUC_LOG_LEVEL_MAX GUC_VERBOSITY_TO_LOG_LEVEL(GUC_LOG_VERBOSITY_MAX) + +int xe_guc_log_init(struct xe_guc_log *log); +void xe_guc_log_print(struct xe_guc_log *log, struct drm_printer *p); + +static inline u32 +xe_guc_log_get_level(struct xe_guc_log *log) +{ + return log->level; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_log_types.h b/drivers/gpu/drm/xe/xe_guc_log_types.h new file mode 100644 index 000000000000..125080d138a7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_log_types.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_LOG_TYPES_H_ +#define _XE_GUC_LOG_TYPES_H_ + +#include + +struct xe_bo; + +/** + * struct xe_guc_log - GuC log + */ +struct xe_guc_log { + /** @level: GuC log level */ + u32 level; + /** @bo: XE BO for GuC log */ + struct xe_bo *bo; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c new file mode 100644 index 000000000000..227e30a482e3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -0,0 +1,843 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_types.h" +#include "xe_gt_sysfs.h" +#include "xe_guc_ct.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_pcode.h" +#include "i915_reg_defs.h" +#include "i915_reg.h" + +#include "intel_mchbar_regs.h" + +/* For GEN6_RP_STATE_CAP.reg to be merged when the definition moves to Xe */ +#define RP0_MASK REG_GENMASK(7, 0) +#define RP1_MASK REG_GENMASK(15, 8) +#define RPN_MASK REG_GENMASK(23, 16) + +#define GEN10_FREQ_INFO_REC _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5ef0) +#define RPE_MASK REG_GENMASK(15, 8) + +#include "gt/intel_gt_regs.h" +/* For GEN6_RPNSWREQ.reg to be merged when the definition moves to Xe */ +#define REQ_RATIO_MASK REG_GENMASK(31, 23) + +/* For GEN6_GT_CORE_STATUS.reg to be merged when the definition moves to Xe */ +#define RCN_MASK REG_GENMASK(2, 0) + +#define GEN12_RPSTAT1 _MMIO(0x1381b4) +#define GEN12_CAGF_MASK REG_GENMASK(19, 11) + +#define GT_FREQUENCY_MULTIPLIER 50 +#define GEN9_FREQ_SCALER 3 + +/** + * DOC: GuC Power Conservation (PC) + * + * GuC Power Conservation (PC) supports multiple features for the most + * efficient and performing use of the GT when GuC submission is enabled, + * including frequency management, Render-C states management, and various + * algorithms for power balancing. + * + * Single Loop Power Conservation (SLPC) is the name given to the suite of + * connected power conservation features in the GuC firmware. The firmware + * exposes a programming interface to the host for the control of SLPC. + * + * Frequency management: + * ===================== + * + * Xe driver enables SLPC with all of its defaults features and frequency + * selection, which varies per platform. + * Xe's GuC PC provides a sysfs API for frequency management: + * + * device/gt#/freq_* *read-only* files: + * - freq_act: The actual resolved frequency decided by PCODE. + * - freq_cur: The current one requested by GuC PC to the Hardware. + * - freq_rpn: The Render Performance (RP) N level, which is the minimal one. + * - freq_rpe: The Render Performance (RP) E level, which is the efficient one. + * - freq_rp0: The Render Performance (RP) 0 level, which is the maximum one. + * + * device/gt#/freq_* *read-write* files: + * - freq_min: GuC PC min request. + * - freq_max: GuC PC max request. + * If max <= min, then freq_min becomes a fixed frequency request. + * + * Render-C States: + * ================ + * + * Render-C states is also a GuC PC feature that is now enabled in Xe for + * all platforms. + * Xe's GuC PC provides a sysfs API for Render-C States: + * + * device/gt#/rc* *read-only* files: + * - rc_status: Provide the actual immediate status of Render-C: (rc0 or rc6) + * - rc6_residency: Provide the rc6_residency counter in units of 1.28 uSec. + * Prone to overflows. + */ + +static struct xe_guc * +pc_to_guc(struct xe_guc_pc *pc) +{ + return container_of(pc, struct xe_guc, pc); +} + +static struct xe_device * +pc_to_xe(struct xe_guc_pc *pc) +{ + struct xe_guc *guc = pc_to_guc(pc); + struct xe_gt *gt = container_of(guc, struct xe_gt, uc.guc); + + return gt_to_xe(gt); +} + +static struct xe_gt * +pc_to_gt(struct xe_guc_pc *pc) +{ + return container_of(pc, struct xe_gt, uc.guc.pc); +} + +static struct xe_guc_pc * +dev_to_pc(struct device *dev) +{ + return &kobj_to_gt(&dev->kobj)->uc.guc.pc; +} + +static struct iosys_map * +pc_to_maps(struct xe_guc_pc *pc) +{ + return &pc->bo->vmap; +} + +#define slpc_shared_data_read(pc_, field_) \ + xe_map_rd_field(pc_to_xe(pc_), pc_to_maps(pc_), 0, \ + struct slpc_shared_data, field_) + +#define slpc_shared_data_write(pc_, field_, val_) \ + xe_map_wr_field(pc_to_xe(pc_), pc_to_maps(pc_), 0, \ + struct slpc_shared_data, field_, val_) + +#define SLPC_EVENT(id, count) \ + (FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ID, id) | \ + FIELD_PREP(HOST2GUC_PC_SLPC_REQUEST_MSG_1_EVENT_ARGC, count)) + +static bool pc_is_in_state(struct xe_guc_pc *pc, enum slpc_global_state state) +{ + xe_device_assert_mem_access(pc_to_xe(pc)); + return slpc_shared_data_read(pc, header.global_state) == state; +} + +static int pc_action_reset(struct xe_guc_pc *pc) +{ + struct xe_guc_ct *ct = &pc_to_guc(pc)->ct; + int ret; + u32 action[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_RESET, 2), + xe_bo_ggtt_addr(pc->bo), + 0, + }; + + ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); + if (ret) + drm_err(&pc_to_xe(pc)->drm, "GuC PC reset: %pe", ERR_PTR(ret)); + + return ret; +} + +static int pc_action_shutdown(struct xe_guc_pc *pc) +{ + struct xe_guc_ct *ct = &pc_to_guc(pc)->ct; + int ret; + u32 action[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_SHUTDOWN, 2), + xe_bo_ggtt_addr(pc->bo), + 0, + }; + + ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); + if (ret) + drm_err(&pc_to_xe(pc)->drm, "GuC PC shutdown %pe", + ERR_PTR(ret)); + + return ret; +} + +static int pc_action_query_task_state(struct xe_guc_pc *pc) +{ + struct xe_guc_ct *ct = &pc_to_guc(pc)->ct; + int ret; + u32 action[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_QUERY_TASK_STATE, 2), + xe_bo_ggtt_addr(pc->bo), + 0, + }; + + if (!pc_is_in_state(pc, SLPC_GLOBAL_STATE_RUNNING)) + return -EAGAIN; + + /* Blocking here to ensure the results are ready before reading them */ + ret = xe_guc_ct_send_block(ct, action, ARRAY_SIZE(action)); + if (ret) + drm_err(&pc_to_xe(pc)->drm, + "GuC PC query task state failed: %pe", ERR_PTR(ret)); + + return ret; +} + +static int pc_action_set_param(struct xe_guc_pc *pc, u8 id, u32 value) +{ + struct xe_guc_ct *ct = &pc_to_guc(pc)->ct; + int ret; + u32 action[] = { + GUC_ACTION_HOST2GUC_PC_SLPC_REQUEST, + SLPC_EVENT(SLPC_EVENT_PARAMETER_SET, 2), + id, + value, + }; + + if (!pc_is_in_state(pc, SLPC_GLOBAL_STATE_RUNNING)) + return -EAGAIN; + + ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); + if (ret) + drm_err(&pc_to_xe(pc)->drm, "GuC PC set param failed: %pe", + ERR_PTR(ret)); + + return ret; +} + +static int pc_action_setup_gucrc(struct xe_guc_pc *pc, u32 mode) +{ + struct xe_guc_ct *ct = &pc_to_guc(pc)->ct; + u32 action[] = { + XE_GUC_ACTION_SETUP_PC_GUCRC, + mode, + }; + int ret; + + ret = xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0); + if (ret) + drm_err(&pc_to_xe(pc)->drm, "GuC RC enable failed: %pe", + ERR_PTR(ret)); + return ret; +} + +static u32 decode_freq(u32 raw) +{ + return DIV_ROUND_CLOSEST(raw * GT_FREQUENCY_MULTIPLIER, + GEN9_FREQ_SCALER); +} + +static u32 pc_get_min_freq(struct xe_guc_pc *pc) +{ + u32 freq; + + freq = FIELD_GET(SLPC_MIN_UNSLICE_FREQ_MASK, + slpc_shared_data_read(pc, task_state_data.freq)); + + return decode_freq(freq); +} + +static int pc_set_min_freq(struct xe_guc_pc *pc, u32 freq) +{ + /* + * Let's only check for the rpn-rp0 range. If max < min, + * min becomes a fixed request. + */ + if (freq < pc->rpn_freq || freq > pc->rp0_freq) + return -EINVAL; + + /* + * GuC policy is to elevate minimum frequency to the efficient levels + * Our goal is to have the admin choices respected. + */ + pc_action_set_param(pc, SLPC_PARAM_IGNORE_EFFICIENT_FREQUENCY, + freq < pc->rpe_freq); + + return pc_action_set_param(pc, + SLPC_PARAM_GLOBAL_MIN_GT_UNSLICE_FREQ_MHZ, + freq); +} + +static int pc_get_max_freq(struct xe_guc_pc *pc) +{ + u32 freq; + + freq = FIELD_GET(SLPC_MAX_UNSLICE_FREQ_MASK, + slpc_shared_data_read(pc, task_state_data.freq)); + + return decode_freq(freq); +} + +static int pc_set_max_freq(struct xe_guc_pc *pc, u32 freq) +{ + /* + * Let's only check for the rpn-rp0 range. If max < min, + * min becomes a fixed request. + * Also, overclocking is not supported. + */ + if (freq < pc->rpn_freq || freq > pc->rp0_freq) + return -EINVAL; + + return pc_action_set_param(pc, + SLPC_PARAM_GLOBAL_MAX_GT_UNSLICE_FREQ_MHZ, + freq); +} + +static void pc_update_rp_values(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + struct xe_device *xe = gt_to_xe(gt); + u32 reg; + + /* + * For PVC we still need to use fused RP1 as the approximation for RPe + * For other platforms than PVC we get the resolved RPe directly from + * PCODE at a different register + */ + if (xe->info.platform == XE_PVC) + reg = xe_mmio_read32(gt, PVC_RP_STATE_CAP.reg); + else + reg = xe_mmio_read32(gt, GEN10_FREQ_INFO_REC.reg); + + pc->rpe_freq = REG_FIELD_GET(RPE_MASK, reg) * GT_FREQUENCY_MULTIPLIER; + + /* + * RPe is decided at runtime by PCODE. In the rare case where that's + * smaller than the fused min, we will trust the PCODE and use that + * as our minimum one. + */ + pc->rpn_freq = min(pc->rpn_freq, pc->rpe_freq); +} + +static ssize_t freq_act_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct kobject *kobj = &dev->kobj; + struct xe_gt *gt = kobj_to_gt(kobj); + u32 freq; + ssize_t ret; + + /* + * When in RC6, actual frequency is 0. Let's block RC6 so we are able + * to verify that our freq requests are really happening. + */ + ret = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (ret) + return ret; + + xe_device_mem_access_get(gt_to_xe(gt)); + freq = xe_mmio_read32(gt, GEN12_RPSTAT1.reg); + xe_device_mem_access_put(gt_to_xe(gt)); + + freq = REG_FIELD_GET(GEN12_CAGF_MASK, freq); + ret = sysfs_emit(buf, "%d\n", decode_freq(freq)); + + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + return ret; +} +static DEVICE_ATTR_RO(freq_act); + +static ssize_t freq_cur_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct kobject *kobj = &dev->kobj; + struct xe_gt *gt = kobj_to_gt(kobj); + u32 freq; + ssize_t ret; + + /* + * GuC SLPC plays with cur freq request when GuCRC is enabled + * Block RC6 for a more reliable read. + */ + ret = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (ret) + return ret; + + xe_device_mem_access_get(gt_to_xe(gt)); + freq = xe_mmio_read32(gt, GEN6_RPNSWREQ.reg); + xe_device_mem_access_put(gt_to_xe(gt)); + + freq = REG_FIELD_GET(REQ_RATIO_MASK, freq); + ret = sysfs_emit(buf, "%d\n", decode_freq(freq)); + + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + return ret; +} +static DEVICE_ATTR_RO(freq_cur); + +static ssize_t freq_rp0_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct xe_guc_pc *pc = dev_to_pc(dev); + + return sysfs_emit(buf, "%d\n", pc->rp0_freq); +} +static DEVICE_ATTR_RO(freq_rp0); + +static ssize_t freq_rpe_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct xe_guc_pc *pc = dev_to_pc(dev); + + pc_update_rp_values(pc); + return sysfs_emit(buf, "%d\n", pc->rpe_freq); +} +static DEVICE_ATTR_RO(freq_rpe); + +static ssize_t freq_rpn_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct xe_guc_pc *pc = dev_to_pc(dev); + + return sysfs_emit(buf, "%d\n", pc->rpn_freq); +} +static DEVICE_ATTR_RO(freq_rpn); + +static ssize_t freq_min_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct xe_guc_pc *pc = dev_to_pc(dev); + struct xe_gt *gt = pc_to_gt(pc); + ssize_t ret; + + xe_device_mem_access_get(pc_to_xe(pc)); + mutex_lock(&pc->freq_lock); + if (!pc->freq_ready) { + /* Might be in the middle of a gt reset */ + ret = -EAGAIN; + goto out; + } + + /* + * GuC SLPC plays with min freq request when GuCRC is enabled + * Block RC6 for a more reliable read. + */ + ret = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (ret) + goto out; + + ret = pc_action_query_task_state(pc); + if (ret) + goto fw; + + ret = sysfs_emit(buf, "%d\n", pc_get_min_freq(pc)); + +fw: + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); +out: + mutex_unlock(&pc->freq_lock); + xe_device_mem_access_put(pc_to_xe(pc)); + return ret; +} + +static ssize_t freq_min_store(struct device *dev, struct device_attribute *attr, + const char *buff, size_t count) +{ + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + ssize_t ret; + + ret = kstrtou32(buff, 0, &freq); + if (ret) + return ret; + + xe_device_mem_access_get(pc_to_xe(pc)); + mutex_lock(&pc->freq_lock); + if (!pc->freq_ready) { + /* Might be in the middle of a gt reset */ + ret = -EAGAIN; + goto out; + } + + ret = pc_set_min_freq(pc, freq); + if (ret) + goto out; + + pc->user_requested_min = freq; + +out: + mutex_unlock(&pc->freq_lock); + xe_device_mem_access_put(pc_to_xe(pc)); + return ret ?: count; +} +static DEVICE_ATTR_RW(freq_min); + +static ssize_t freq_max_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct xe_guc_pc *pc = dev_to_pc(dev); + ssize_t ret; + + xe_device_mem_access_get(pc_to_xe(pc)); + mutex_lock(&pc->freq_lock); + if (!pc->freq_ready) { + /* Might be in the middle of a gt reset */ + ret = -EAGAIN; + goto out; + } + + ret = pc_action_query_task_state(pc); + if (ret) + goto out; + + ret = sysfs_emit(buf, "%d\n", pc_get_max_freq(pc)); + +out: + mutex_unlock(&pc->freq_lock); + xe_device_mem_access_put(pc_to_xe(pc)); + return ret; +} + +static ssize_t freq_max_store(struct device *dev, struct device_attribute *attr, + const char *buff, size_t count) +{ + struct xe_guc_pc *pc = dev_to_pc(dev); + u32 freq; + ssize_t ret; + + ret = kstrtou32(buff, 0, &freq); + if (ret) + return ret; + + xe_device_mem_access_get(pc_to_xe(pc)); + mutex_lock(&pc->freq_lock); + if (!pc->freq_ready) { + /* Might be in the middle of a gt reset */ + ret = -EAGAIN; + goto out; + } + + ret = pc_set_max_freq(pc, freq); + if (ret) + goto out; + + pc->user_requested_max = freq; + +out: + mutex_unlock(&pc->freq_lock); + xe_device_mem_access_put(pc_to_xe(pc)); + return ret ?: count; +} +static DEVICE_ATTR_RW(freq_max); + +static ssize_t rc_status_show(struct device *dev, + struct device_attribute *attr, char *buff) +{ + struct xe_guc_pc *pc = dev_to_pc(dev); + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + + xe_device_mem_access_get(gt_to_xe(gt)); + reg = xe_mmio_read32(gt, GEN6_GT_CORE_STATUS.reg); + xe_device_mem_access_put(gt_to_xe(gt)); + + switch (REG_FIELD_GET(RCN_MASK, reg)) { + case GEN6_RC6: + return sysfs_emit(buff, "rc6\n"); + case GEN6_RC0: + return sysfs_emit(buff, "rc0\n"); + default: + return -ENOENT; + } +} +static DEVICE_ATTR_RO(rc_status); + +static ssize_t rc6_residency_show(struct device *dev, + struct device_attribute *attr, char *buff) +{ + struct xe_guc_pc *pc = dev_to_pc(dev); + struct xe_gt *gt = pc_to_gt(pc); + u32 reg; + ssize_t ret; + + ret = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (ret) + return ret; + + xe_device_mem_access_get(pc_to_xe(pc)); + reg = xe_mmio_read32(gt, GEN6_GT_GFX_RC6.reg); + xe_device_mem_access_put(pc_to_xe(pc)); + + ret = sysfs_emit(buff, "%u\n", reg); + + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + return ret; +} +static DEVICE_ATTR_RO(rc6_residency); + +static const struct attribute *pc_attrs[] = { + &dev_attr_freq_act.attr, + &dev_attr_freq_cur.attr, + &dev_attr_freq_rp0.attr, + &dev_attr_freq_rpe.attr, + &dev_attr_freq_rpn.attr, + &dev_attr_freq_min.attr, + &dev_attr_freq_max.attr, + &dev_attr_rc_status.attr, + &dev_attr_rc6_residency.attr, + NULL +}; + +static void pc_init_fused_rp_values(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + struct xe_device *xe = gt_to_xe(gt); + u32 reg; + + xe_device_assert_mem_access(pc_to_xe(pc)); + + if (xe->info.platform == XE_PVC) + reg = xe_mmio_read32(gt, PVC_RP_STATE_CAP.reg); + else + reg = xe_mmio_read32(gt, GEN6_RP_STATE_CAP.reg); + pc->rp0_freq = REG_FIELD_GET(RP0_MASK, reg) * GT_FREQUENCY_MULTIPLIER; + pc->rpn_freq = REG_FIELD_GET(RPN_MASK, reg) * GT_FREQUENCY_MULTIPLIER; +} + +static int pc_adjust_freq_bounds(struct xe_guc_pc *pc) +{ + int ret; + + lockdep_assert_held(&pc->freq_lock); + + ret = pc_action_query_task_state(pc); + if (ret) + return ret; + + /* + * GuC defaults to some RPmax that is not actually achievable without + * overclocking. Let's adjust it to the Hardware RP0, which is the + * regular maximum + */ + if (pc_get_max_freq(pc) > pc->rp0_freq) + pc_set_max_freq(pc, pc->rp0_freq); + + /* + * Same thing happens for Server platforms where min is listed as + * RPMax + */ + if (pc_get_min_freq(pc) > pc->rp0_freq) + pc_set_min_freq(pc, pc->rp0_freq); + + return 0; +} + +static int pc_adjust_requested_freq(struct xe_guc_pc *pc) +{ + int ret = 0; + + lockdep_assert_held(&pc->freq_lock); + + if (pc->user_requested_min != 0) { + ret = pc_set_min_freq(pc, pc->user_requested_min); + if (ret) + return ret; + } + + if (pc->user_requested_max != 0) { + ret = pc_set_max_freq(pc, pc->user_requested_max); + if (ret) + return ret; + } + + return ret; +} + +static int pc_gucrc_disable(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + int ret; + + xe_device_assert_mem_access(pc_to_xe(pc)); + + ret = pc_action_setup_gucrc(pc, XE_GUCRC_HOST_CONTROL); + if (ret) + return ret; + + ret = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (ret) + return ret; + + xe_mmio_write32(gt, GEN9_PG_ENABLE.reg, 0); + xe_mmio_write32(gt, GEN6_RC_CONTROL.reg, 0); + xe_mmio_write32(gt, GEN6_RC_STATE.reg, 0); + + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + return 0; +} + +static void pc_init_pcode_freq(struct xe_guc_pc *pc) +{ + u32 min = DIV_ROUND_CLOSEST(pc->rpn_freq, GT_FREQUENCY_MULTIPLIER); + u32 max = DIV_ROUND_CLOSEST(pc->rp0_freq, GT_FREQUENCY_MULTIPLIER); + + XE_WARN_ON(xe_pcode_init_min_freq_table(pc_to_gt(pc), min, max)); +} + +static int pc_init_freqs(struct xe_guc_pc *pc) +{ + int ret; + + mutex_lock(&pc->freq_lock); + + ret = pc_adjust_freq_bounds(pc); + if (ret) + goto out; + + ret = pc_adjust_requested_freq(pc); + if (ret) + goto out; + + pc_update_rp_values(pc); + + pc_init_pcode_freq(pc); + + /* + * The frequencies are really ready for use only after the user + * requested ones got restored. + */ + pc->freq_ready = true; + +out: + mutex_unlock(&pc->freq_lock); + return ret; +} + +/** + * xe_guc_pc_start - Start GuC's Power Conservation component + * @pc: Xe_GuC_PC instance + */ +int xe_guc_pc_start(struct xe_guc_pc *pc) +{ + struct xe_device *xe = pc_to_xe(pc); + struct xe_gt *gt = pc_to_gt(pc); + u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data)); + int ret; + + XE_WARN_ON(!xe_device_guc_submission_enabled(xe)); + + xe_device_mem_access_get(pc_to_xe(pc)); + + memset(pc->bo->vmap.vaddr, 0, size); + slpc_shared_data_write(pc, header.size, size); + + ret = xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + if (ret) + return ret; + + ret = pc_action_reset(pc); + if (ret) + goto out; + + if (wait_for(pc_is_in_state(pc, SLPC_GLOBAL_STATE_RUNNING), 5)) { + drm_err(&pc_to_xe(pc)->drm, "GuC PC Start failed\n"); + ret = -EIO; + goto out; + } + + ret = pc_init_freqs(pc); + if (ret) + goto out; + + if (xe->info.platform == XE_PVC) { + pc_gucrc_disable(pc); + ret = 0; + goto out; + } + + ret = pc_action_setup_gucrc(pc, XE_GUCRC_FIRMWARE_CONTROL); + +out: + xe_device_mem_access_put(pc_to_xe(pc)); + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL)); + return ret; +} + +/** + * xe_guc_pc_stop - Stop GuC's Power Conservation component + * @pc: Xe_GuC_PC instance + */ +int xe_guc_pc_stop(struct xe_guc_pc *pc) +{ + int ret; + + xe_device_mem_access_get(pc_to_xe(pc)); + + ret = pc_gucrc_disable(pc); + if (ret) + goto out; + + mutex_lock(&pc->freq_lock); + pc->freq_ready = false; + mutex_unlock(&pc->freq_lock); + + ret = pc_action_shutdown(pc); + if (ret) + goto out; + + if (wait_for(pc_is_in_state(pc, SLPC_GLOBAL_STATE_NOT_RUNNING), 5)) { + drm_err(&pc_to_xe(pc)->drm, "GuC PC Shutdown failed\n"); + ret = -EIO; + } + +out: + xe_device_mem_access_put(pc_to_xe(pc)); + return ret; +} + +static void pc_fini(struct drm_device *drm, void *arg) +{ + struct xe_guc_pc *pc = arg; + + XE_WARN_ON(xe_guc_pc_stop(pc)); + sysfs_remove_files(pc_to_gt(pc)->sysfs, pc_attrs); + xe_bo_unpin_map_no_vm(pc->bo); +} + +/** + * xe_guc_pc_init - Initialize GuC's Power Conservation component + * @pc: Xe_GuC_PC instance + */ +int xe_guc_pc_init(struct xe_guc_pc *pc) +{ + struct xe_gt *gt = pc_to_gt(pc); + struct xe_device *xe = gt_to_xe(gt); + struct xe_bo *bo; + u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data)); + int err; + + mutex_init(&pc->freq_lock); + + bo = xe_bo_create_pin_map(xe, gt, NULL, size, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_GGTT_BIT); + + if (IS_ERR(bo)) + return PTR_ERR(bo); + + pc->bo = bo; + + pc_init_fused_rp_values(pc); + + err = sysfs_create_files(gt->sysfs, pc_attrs); + if (err) + return err; + + err = drmm_add_action_or_reset(&xe->drm, pc_fini, pc); + if (err) + return err; + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_guc_pc.h b/drivers/gpu/drm/xe/xe_guc_pc.h new file mode 100644 index 000000000000..da29e4934868 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_pc.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_PC_H_ +#define _XE_GUC_PC_H_ + +#include "xe_guc_pc_types.h" + +int xe_guc_pc_init(struct xe_guc_pc *pc); +int xe_guc_pc_start(struct xe_guc_pc *pc); +int xe_guc_pc_stop(struct xe_guc_pc *pc); + +#endif /* _XE_GUC_PC_H_ */ diff --git a/drivers/gpu/drm/xe/xe_guc_pc_types.h b/drivers/gpu/drm/xe/xe_guc_pc_types.h new file mode 100644 index 000000000000..39548e03acf4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_pc_types.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_PC_TYPES_H_ +#define _XE_GUC_PC_TYPES_H_ + +#include +#include + +/** + * struct xe_guc_pc - GuC Power Conservation (PC) + */ +struct xe_guc_pc { + /** @bo: GGTT buffer object that is shared with GuC PC */ + struct xe_bo *bo; + /** @rp0_freq: HW RP0 frequency - The Maximum one */ + u32 rp0_freq; + /** @rpe_freq: HW RPe frequency - The Efficient one */ + u32 rpe_freq; + /** @rpn_freq: HW RPN frequency - The Minimum one */ + u32 rpn_freq; + /** @user_requested_min: Stash the minimum requested freq by user */ + u32 user_requested_min; + /** @user_requested_max: Stash the maximum requested freq by user */ + u32 user_requested_max; + /** @freq_lock: Let's protect the frequencies */ + struct mutex freq_lock; + /** @freq_ready: Only handle freq changes, if they are really ready */ + bool freq_ready; +}; + +#endif /* _XE_GUC_PC_TYPES_H_ */ diff --git a/drivers/gpu/drm/xe/xe_guc_reg.h b/drivers/gpu/drm/xe/xe_guc_reg.h new file mode 100644 index 000000000000..1e16a9b76ddc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_reg.h @@ -0,0 +1,147 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_REG_H_ +#define _XE_GUC_REG_H_ + +#include +#include + +#include "i915_reg.h" + +/* Definitions of GuC H/W registers, bits, etc */ + +#define GUC_STATUS _MMIO(0xc000) +#define GS_RESET_SHIFT 0 +#define GS_MIA_IN_RESET (0x01 << GS_RESET_SHIFT) +#define GS_BOOTROM_SHIFT 1 +#define GS_BOOTROM_MASK (0x7F << GS_BOOTROM_SHIFT) +#define GS_BOOTROM_RSA_FAILED (0x50 << GS_BOOTROM_SHIFT) +#define GS_BOOTROM_JUMP_PASSED (0x76 << GS_BOOTROM_SHIFT) +#define GS_UKERNEL_SHIFT 8 +#define GS_UKERNEL_MASK (0xFF << GS_UKERNEL_SHIFT) +#define GS_MIA_SHIFT 16 +#define GS_MIA_MASK (0x07 << GS_MIA_SHIFT) +#define GS_MIA_CORE_STATE (0x01 << GS_MIA_SHIFT) +#define GS_MIA_HALT_REQUESTED (0x02 << GS_MIA_SHIFT) +#define GS_MIA_ISR_ENTRY (0x04 << GS_MIA_SHIFT) +#define GS_AUTH_STATUS_SHIFT 30 +#define GS_AUTH_STATUS_MASK (0x03 << GS_AUTH_STATUS_SHIFT) +#define GS_AUTH_STATUS_BAD (0x01 << GS_AUTH_STATUS_SHIFT) +#define GS_AUTH_STATUS_GOOD (0x02 << GS_AUTH_STATUS_SHIFT) + +#define SOFT_SCRATCH(n) _MMIO(0xc180 + (n) * 4) +#define SOFT_SCRATCH_COUNT 16 + +#define GEN11_SOFT_SCRATCH(n) _MMIO(0x190240 + (n) * 4) +#define GEN11_SOFT_SCRATCH_COUNT 4 + +#define UOS_RSA_SCRATCH(i) _MMIO(0xc200 + (i) * 4) +#define UOS_RSA_SCRATCH_COUNT 64 + +#define DMA_ADDR_0_LOW _MMIO(0xc300) +#define DMA_ADDR_0_HIGH _MMIO(0xc304) +#define DMA_ADDR_1_LOW _MMIO(0xc308) +#define DMA_ADDR_1_HIGH _MMIO(0xc30c) +#define DMA_ADDRESS_SPACE_WOPCM (7 << 16) +#define DMA_ADDRESS_SPACE_GTT (8 << 16) +#define DMA_COPY_SIZE _MMIO(0xc310) +#define DMA_CTRL _MMIO(0xc314) +#define HUC_UKERNEL (1<<9) +#define UOS_MOVE (1<<4) +#define START_DMA (1<<0) +#define DMA_GUC_WOPCM_OFFSET _MMIO(0xc340) +#define GUC_WOPCM_OFFSET_VALID (1<<0) +#define HUC_LOADING_AGENT_VCR (0<<1) +#define HUC_LOADING_AGENT_GUC (1<<1) +#define GUC_WOPCM_OFFSET_SHIFT 14 +#define GUC_WOPCM_OFFSET_MASK (0x3ffff << GUC_WOPCM_OFFSET_SHIFT) +#define GUC_MAX_IDLE_COUNT _MMIO(0xC3E4) + +#define HUC_STATUS2 _MMIO(0xD3B0) +#define HUC_FW_VERIFIED (1<<7) + +#define GEN11_HUC_KERNEL_LOAD_INFO _MMIO(0xC1DC) +#define HUC_LOAD_SUCCESSFUL (1 << 0) + +#define GUC_WOPCM_SIZE _MMIO(0xc050) +#define GUC_WOPCM_SIZE_LOCKED (1<<0) +#define GUC_WOPCM_SIZE_SHIFT 12 +#define GUC_WOPCM_SIZE_MASK (0xfffff << GUC_WOPCM_SIZE_SHIFT) + +#define GEN8_GT_PM_CONFIG _MMIO(0x138140) +#define GEN9LP_GT_PM_CONFIG _MMIO(0x138140) +#define GEN9_GT_PM_CONFIG _MMIO(0x13816c) +#define GT_DOORBELL_ENABLE (1<<0) + +#define GEN8_GTCR _MMIO(0x4274) +#define GEN8_GTCR_INVALIDATE (1<<0) + +#define GEN12_GUC_TLB_INV_CR _MMIO(0xcee8) +#define GEN12_GUC_TLB_INV_CR_INVALIDATE (1 << 0) + +#define GUC_ARAT_C6DIS _MMIO(0xA178) + +#define GUC_SHIM_CONTROL _MMIO(0xc064) +#define GUC_DISABLE_SRAM_INIT_TO_ZEROES (1<<0) +#define GUC_ENABLE_READ_CACHE_LOGIC (1<<1) +#define GUC_ENABLE_MIA_CACHING (1<<2) +#define GUC_GEN10_MSGCH_ENABLE (1<<4) +#define GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA (1<<9) +#define GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA (1<<10) +#define GUC_ENABLE_MIA_CLOCK_GATING (1<<15) +#define GUC_GEN10_SHIM_WC_ENABLE (1<<21) + +#define GUC_SEND_INTERRUPT _MMIO(0xc4c8) +#define GUC_SEND_TRIGGER (1<<0) +#define GEN11_GUC_HOST_INTERRUPT _MMIO(0x1901f0) + +#define GUC_NUM_DOORBELLS 256 + +/* format of the HW-monitored doorbell cacheline */ +struct guc_doorbell_info { + u32 db_status; +#define GUC_DOORBELL_DISABLED 0 +#define GUC_DOORBELL_ENABLED 1 + + u32 cookie; + u32 reserved[14]; +} __packed; + +#define GEN8_DRBREGL(x) _MMIO(0x1000 + (x) * 8) +#define GEN8_DRB_VALID (1<<0) +#define GEN8_DRBREGU(x) _MMIO(0x1000 + (x) * 8 + 4) + +#define GEN12_DIST_DBS_POPULATED _MMIO(0xd08) +#define GEN12_DOORBELLS_PER_SQIDI_SHIFT 16 +#define GEN12_DOORBELLS_PER_SQIDI (0xff) +#define GEN12_SQIDIS_DOORBELL_EXIST (0xffff) + +#define DE_GUCRMR _MMIO(0x44054) + +#define GUC_BCS_RCS_IER _MMIO(0xC550) +#define GUC_VCS2_VCS1_IER _MMIO(0xC554) +#define GUC_WD_VECS_IER _MMIO(0xC558) +#define GUC_PM_P24C_IER _MMIO(0xC55C) + +/* GuC Interrupt Vector */ +#define GUC_INTR_GUC2HOST BIT(15) +#define GUC_INTR_EXEC_ERROR BIT(14) +#define GUC_INTR_DISPLAY_EVENT BIT(13) +#define GUC_INTR_SEM_SIG BIT(12) +#define GUC_INTR_IOMMU2GUC BIT(11) +#define GUC_INTR_DOORBELL_RANG BIT(10) +#define GUC_INTR_DMA_DONE BIT(9) +#define GUC_INTR_FATAL_ERROR BIT(8) +#define GUC_INTR_NOTIF_ERROR BIT(7) +#define GUC_INTR_SW_INT_6 BIT(6) +#define GUC_INTR_SW_INT_5 BIT(5) +#define GUC_INTR_SW_INT_4 BIT(4) +#define GUC_INTR_SW_INT_3 BIT(3) +#define GUC_INTR_SW_INT_2 BIT(2) +#define GUC_INTR_SW_INT_1 BIT(1) +#define GUC_INTR_SW_INT_0 BIT(0) + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c new file mode 100644 index 000000000000..e0d424c2b78c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -0,0 +1,1695 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include +#include +#include +#include + +#include + +#include "xe_device.h" +#include "xe_engine.h" +#include "xe_guc.h" +#include "xe_guc_ct.h" +#include "xe_guc_engine_types.h" +#include "xe_guc_submit.h" +#include "xe_gt.h" +#include "xe_force_wake.h" +#include "xe_gpu_scheduler.h" +#include "xe_hw_engine.h" +#include "xe_hw_fence.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_map.h" +#include "xe_mocs.h" +#include "xe_ring_ops_types.h" +#include "xe_sched_job.h" +#include "xe_trace.h" +#include "xe_vm.h" + +#include "gt/intel_lrc_reg.h" + +static struct xe_gt * +guc_to_gt(struct xe_guc *guc) +{ + return container_of(guc, struct xe_gt, uc.guc); +} + +static struct xe_device * +guc_to_xe(struct xe_guc *guc) +{ + return gt_to_xe(guc_to_gt(guc)); +} + +static struct xe_guc * +engine_to_guc(struct xe_engine *e) +{ + return &e->gt->uc.guc; +} + +/* + * Helpers for engine state, using an atomic as some of the bits can transition + * as the same time (e.g. a suspend can be happning at the same time as schedule + * engine done being processed). + */ +#define ENGINE_STATE_REGISTERED (1 << 0) +#define ENGINE_STATE_ENABLED (1 << 1) +#define ENGINE_STATE_PENDING_ENABLE (1 << 2) +#define ENGINE_STATE_PENDING_DISABLE (1 << 3) +#define ENGINE_STATE_DESTROYED (1 << 4) +#define ENGINE_STATE_SUSPENDED (1 << 5) +#define ENGINE_STATE_RESET (1 << 6) +#define ENGINE_STATE_KILLED (1 << 7) + +static bool engine_registered(struct xe_engine *e) +{ + return atomic_read(&e->guc->state) & ENGINE_STATE_REGISTERED; +} + +static void set_engine_registered(struct xe_engine *e) +{ + atomic_or(ENGINE_STATE_REGISTERED, &e->guc->state); +} + +static void clear_engine_registered(struct xe_engine *e) +{ + atomic_and(~ENGINE_STATE_REGISTERED, &e->guc->state); +} + +static bool engine_enabled(struct xe_engine *e) +{ + return atomic_read(&e->guc->state) & ENGINE_STATE_ENABLED; +} + +static void set_engine_enabled(struct xe_engine *e) +{ + atomic_or(ENGINE_STATE_ENABLED, &e->guc->state); +} + +static void clear_engine_enabled(struct xe_engine *e) +{ + atomic_and(~ENGINE_STATE_ENABLED, &e->guc->state); +} + +static bool engine_pending_enable(struct xe_engine *e) +{ + return atomic_read(&e->guc->state) & ENGINE_STATE_PENDING_ENABLE; +} + +static void set_engine_pending_enable(struct xe_engine *e) +{ + atomic_or(ENGINE_STATE_PENDING_ENABLE, &e->guc->state); +} + +static void clear_engine_pending_enable(struct xe_engine *e) +{ + atomic_and(~ENGINE_STATE_PENDING_ENABLE, &e->guc->state); +} + +static bool engine_pending_disable(struct xe_engine *e) +{ + return atomic_read(&e->guc->state) & ENGINE_STATE_PENDING_DISABLE; +} + +static void set_engine_pending_disable(struct xe_engine *e) +{ + atomic_or(ENGINE_STATE_PENDING_DISABLE, &e->guc->state); +} + +static void clear_engine_pending_disable(struct xe_engine *e) +{ + atomic_and(~ENGINE_STATE_PENDING_DISABLE, &e->guc->state); +} + +static bool engine_destroyed(struct xe_engine *e) +{ + return atomic_read(&e->guc->state) & ENGINE_STATE_DESTROYED; +} + +static void set_engine_destroyed(struct xe_engine *e) +{ + atomic_or(ENGINE_STATE_DESTROYED, &e->guc->state); +} + +static bool engine_banned(struct xe_engine *e) +{ + return (e->flags & ENGINE_FLAG_BANNED); +} + +static void set_engine_banned(struct xe_engine *e) +{ + e->flags |= ENGINE_FLAG_BANNED; +} + +static bool engine_suspended(struct xe_engine *e) +{ + return atomic_read(&e->guc->state) & ENGINE_STATE_SUSPENDED; +} + +static void set_engine_suspended(struct xe_engine *e) +{ + atomic_or(ENGINE_STATE_SUSPENDED, &e->guc->state); +} + +static void clear_engine_suspended(struct xe_engine *e) +{ + atomic_and(~ENGINE_STATE_SUSPENDED, &e->guc->state); +} + +static bool engine_reset(struct xe_engine *e) +{ + return atomic_read(&e->guc->state) & ENGINE_STATE_RESET; +} + +static void set_engine_reset(struct xe_engine *e) +{ + atomic_or(ENGINE_STATE_RESET, &e->guc->state); +} + +static bool engine_killed(struct xe_engine *e) +{ + return atomic_read(&e->guc->state) & ENGINE_STATE_KILLED; +} + +static void set_engine_killed(struct xe_engine *e) +{ + atomic_or(ENGINE_STATE_KILLED, &e->guc->state); +} + +static bool engine_killed_or_banned(struct xe_engine *e) +{ + return engine_killed(e) || engine_banned(e); +} + +static void guc_submit_fini(struct drm_device *drm, void *arg) +{ + struct xe_guc *guc = arg; + + xa_destroy(&guc->submission_state.engine_lookup); + ida_destroy(&guc->submission_state.guc_ids); + bitmap_free(guc->submission_state.guc_ids_bitmap); +} + +#define GUC_ID_MAX 65535 +#define GUC_ID_NUMBER_MLRC 4096 +#define GUC_ID_NUMBER_SLRC (GUC_ID_MAX - GUC_ID_NUMBER_MLRC) +#define GUC_ID_START_MLRC GUC_ID_NUMBER_SLRC + +static const struct xe_engine_ops guc_engine_ops; + +static void primelockdep(struct xe_guc *guc) +{ + if (!IS_ENABLED(CONFIG_LOCKDEP)) + return; + + fs_reclaim_acquire(GFP_KERNEL); + + mutex_lock(&guc->submission_state.lock); + might_lock(&guc->submission_state.suspend.lock); + mutex_unlock(&guc->submission_state.lock); + + fs_reclaim_release(GFP_KERNEL); +} + +int xe_guc_submit_init(struct xe_guc *guc) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_gt *gt = guc_to_gt(guc); + int err; + + guc->submission_state.guc_ids_bitmap = + bitmap_zalloc(GUC_ID_NUMBER_MLRC, GFP_KERNEL); + if (!guc->submission_state.guc_ids_bitmap) + return -ENOMEM; + + gt->engine_ops = &guc_engine_ops; + + mutex_init(&guc->submission_state.lock); + xa_init(&guc->submission_state.engine_lookup); + ida_init(&guc->submission_state.guc_ids); + + spin_lock_init(&guc->submission_state.suspend.lock); + guc->submission_state.suspend.context = dma_fence_context_alloc(1); + + primelockdep(guc); + + err = drmm_add_action_or_reset(&xe->drm, guc_submit_fini, guc); + if (err) + return err; + + return 0; +} + +static int alloc_guc_id(struct xe_guc *guc, struct xe_engine *e) +{ + int ret; + void *ptr; + + /* + * Must use GFP_NOWAIT as this lock is in the dma fence signalling path, + * worse case user gets -ENOMEM on engine create and has to try again. + * + * FIXME: Have caller pre-alloc or post-alloc /w GFP_KERNEL to prevent + * failure. + */ + lockdep_assert_held(&guc->submission_state.lock); + + if (xe_engine_is_parallel(e)) { + void *bitmap = guc->submission_state.guc_ids_bitmap; + + ret = bitmap_find_free_region(bitmap, GUC_ID_NUMBER_MLRC, + order_base_2(e->width)); + } else { + ret = ida_simple_get(&guc->submission_state.guc_ids, 0, + GUC_ID_NUMBER_SLRC, GFP_NOWAIT); + } + if (ret < 0) + return ret; + + e->guc->id = ret; + if (xe_engine_is_parallel(e)) + e->guc->id += GUC_ID_START_MLRC; + + ptr = xa_store(&guc->submission_state.engine_lookup, + e->guc->id, e, GFP_NOWAIT); + if (IS_ERR(ptr)) { + ret = PTR_ERR(ptr); + goto err_release; + } + + return 0; + +err_release: + ida_simple_remove(&guc->submission_state.guc_ids, e->guc->id); + return ret; +} + +static void release_guc_id(struct xe_guc *guc, struct xe_engine *e) +{ + mutex_lock(&guc->submission_state.lock); + xa_erase(&guc->submission_state.engine_lookup, e->guc->id); + if (xe_engine_is_parallel(e)) + bitmap_release_region(guc->submission_state.guc_ids_bitmap, + e->guc->id - GUC_ID_START_MLRC, + order_base_2(e->width)); + else + ida_simple_remove(&guc->submission_state.guc_ids, e->guc->id); + mutex_unlock(&guc->submission_state.lock); +} + +struct engine_policy { + u32 count; + struct guc_update_engine_policy h2g; +}; + +static u32 __guc_engine_policy_action_size(struct engine_policy *policy) +{ + size_t bytes = sizeof(policy->h2g.header) + + (sizeof(policy->h2g.klv[0]) * policy->count); + + return bytes / sizeof(u32); +} + +static void __guc_engine_policy_start_klv(struct engine_policy *policy, + u16 guc_id) +{ + policy->h2g.header.action = + XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; + policy->h2g.header.guc_id = guc_id; + policy->count = 0; +} + +#define MAKE_ENGINE_POLICY_ADD(func, id) \ +static void __guc_engine_policy_add_##func(struct engine_policy *policy, \ + u32 data) \ +{ \ + XE_BUG_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ + \ + policy->h2g.klv[policy->count].kl = \ + FIELD_PREP(GUC_KLV_0_KEY, \ + GUC_CONTEXT_POLICIES_KLV_ID_##id) | \ + FIELD_PREP(GUC_KLV_0_LEN, 1); \ + policy->h2g.klv[policy->count].value = data; \ + policy->count++; \ +} + +MAKE_ENGINE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) +MAKE_ENGINE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) +MAKE_ENGINE_POLICY_ADD(priority, SCHEDULING_PRIORITY) +#undef MAKE_ENGINE_POLICY_ADD + +static const int xe_engine_prio_to_guc[] = { + [XE_ENGINE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, + [XE_ENGINE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, + [XE_ENGINE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, + [XE_ENGINE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, +}; + +static void init_policies(struct xe_guc *guc, struct xe_engine *e) +{ + struct engine_policy policy; + enum xe_engine_priority prio = e->priority; + u32 timeslice_us = e->sched_props.timeslice_us; + u32 preempt_timeout_us = e->sched_props.preempt_timeout_us; + + XE_BUG_ON(!engine_registered(e)); + + __guc_engine_policy_start_klv(&policy, e->guc->id); + __guc_engine_policy_add_priority(&policy, xe_engine_prio_to_guc[prio]); + __guc_engine_policy_add_execution_quantum(&policy, timeslice_us); + __guc_engine_policy_add_preemption_timeout(&policy, preempt_timeout_us); + + xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, + __guc_engine_policy_action_size(&policy), 0, 0); +} + +static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_engine *e) +{ + struct engine_policy policy; + + __guc_engine_policy_start_klv(&policy, e->guc->id); + __guc_engine_policy_add_preemption_timeout(&policy, 1); + + xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, + __guc_engine_policy_action_size(&policy), 0, 0); +} + +#define PARALLEL_SCRATCH_SIZE 2048 +#define WQ_SIZE (PARALLEL_SCRATCH_SIZE / 2) +#define WQ_OFFSET (PARALLEL_SCRATCH_SIZE - WQ_SIZE) +#define CACHELINE_BYTES 64 + +struct sync_semaphore { + u32 semaphore; + u8 unused[CACHELINE_BYTES - sizeof(u32)]; +}; + +struct parallel_scratch { + struct guc_sched_wq_desc wq_desc; + + struct sync_semaphore go; + struct sync_semaphore join[XE_HW_ENGINE_MAX_INSTANCE]; + + u8 unused[WQ_OFFSET - sizeof(struct guc_sched_wq_desc) - + sizeof(struct sync_semaphore) * (XE_HW_ENGINE_MAX_INSTANCE + 1)]; + + u32 wq[WQ_SIZE / sizeof(u32)]; +}; + +#define parallel_read(xe_, map_, field_) \ + xe_map_rd_field(xe_, &map_, 0, struct parallel_scratch, field_) +#define parallel_write(xe_, map_, field_, val_) \ + xe_map_wr_field(xe_, &map_, 0, struct parallel_scratch, field_, val_) + +static void __register_mlrc_engine(struct xe_guc *guc, + struct xe_engine *e, + struct guc_ctxt_registration_info *info) +{ +#define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) + u32 action[MAX_MLRC_REG_SIZE]; + int len = 0; + int i; + + XE_BUG_ON(!xe_engine_is_parallel(e)); + + action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; + action[len++] = info->flags; + action[len++] = info->context_idx; + action[len++] = info->engine_class; + action[len++] = info->engine_submit_mask; + action[len++] = info->wq_desc_lo; + action[len++] = info->wq_desc_hi; + action[len++] = info->wq_base_lo; + action[len++] = info->wq_base_hi; + action[len++] = info->wq_size; + action[len++] = e->width; + action[len++] = info->hwlrca_lo; + action[len++] = info->hwlrca_hi; + + for (i = 1; i < e->width; ++i) { + struct xe_lrc *lrc = e->lrc + i; + + action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); + action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); + } + + XE_BUG_ON(len > MAX_MLRC_REG_SIZE); +#undef MAX_MLRC_REG_SIZE + + xe_guc_ct_send(&guc->ct, action, len, 0, 0); +} + +static void __register_engine(struct xe_guc *guc, + struct guc_ctxt_registration_info *info) +{ + u32 action[] = { + XE_GUC_ACTION_REGISTER_CONTEXT, + info->flags, + info->context_idx, + info->engine_class, + info->engine_submit_mask, + info->wq_desc_lo, + info->wq_desc_hi, + info->wq_base_lo, + info->wq_base_hi, + info->wq_size, + info->hwlrca_lo, + info->hwlrca_hi, + }; + + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); +} + +static void register_engine(struct xe_engine *e) +{ + struct xe_guc *guc = engine_to_guc(e); + struct xe_device *xe = guc_to_xe(guc); + struct xe_lrc *lrc = e->lrc; + struct guc_ctxt_registration_info info; + + XE_BUG_ON(engine_registered(e)); + + memset(&info, 0, sizeof(info)); + info.context_idx = e->guc->id; + info.engine_class = xe_engine_class_to_guc_class(e->class); + info.engine_submit_mask = e->logical_mask; + info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); + info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); + info.flags = CONTEXT_REGISTRATION_FLAG_KMD; + + if (xe_engine_is_parallel(e)) { + u32 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); + struct iosys_map map = xe_lrc_parallel_map(lrc); + + info.wq_desc_lo = lower_32_bits(ggtt_addr + + offsetof(struct parallel_scratch, wq_desc)); + info.wq_desc_hi = upper_32_bits(ggtt_addr + + offsetof(struct parallel_scratch, wq_desc)); + info.wq_base_lo = lower_32_bits(ggtt_addr + + offsetof(struct parallel_scratch, wq[0])); + info.wq_base_hi = upper_32_bits(ggtt_addr + + offsetof(struct parallel_scratch, wq[0])); + info.wq_size = WQ_SIZE; + + e->guc->wqi_head = 0; + e->guc->wqi_tail = 0; + xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); + parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); + } + + set_engine_registered(e); + trace_xe_engine_register(e); + if (xe_engine_is_parallel(e)) + __register_mlrc_engine(guc, e, &info); + else + __register_engine(guc, &info); + init_policies(guc, e); +} + +static u32 wq_space_until_wrap(struct xe_engine *e) +{ + return (WQ_SIZE - e->guc->wqi_tail); +} + +static int wq_wait_for_space(struct xe_engine *e, u32 wqi_size) +{ + struct xe_guc *guc = engine_to_guc(e); + struct xe_device *xe = guc_to_xe(guc); + struct iosys_map map = xe_lrc_parallel_map(e->lrc); + unsigned int sleep_period_ms = 1; + +#define AVAILABLE_SPACE \ + CIRC_SPACE(e->guc->wqi_tail, e->guc->wqi_head, WQ_SIZE) + if (wqi_size > AVAILABLE_SPACE) { +try_again: + e->guc->wqi_head = parallel_read(xe, map, wq_desc.head); + if (wqi_size > AVAILABLE_SPACE) { + if (sleep_period_ms == 1024) { + xe_gt_reset_async(e->gt); + return -ENODEV; + } + + msleep(sleep_period_ms); + sleep_period_ms <<= 1; + goto try_again; + } + } +#undef AVAILABLE_SPACE + + return 0; +} + +static int wq_noop_append(struct xe_engine *e) +{ + struct xe_guc *guc = engine_to_guc(e); + struct xe_device *xe = guc_to_xe(guc); + struct iosys_map map = xe_lrc_parallel_map(e->lrc); + u32 len_dw = wq_space_until_wrap(e) / sizeof(u32) - 1; + + if (wq_wait_for_space(e, wq_space_until_wrap(e))) + return -ENODEV; + + XE_BUG_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw)); + + parallel_write(xe, map, wq[e->guc->wqi_tail / sizeof(u32)], + FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | + FIELD_PREP(WQ_LEN_MASK, len_dw)); + e->guc->wqi_tail = 0; + + return 0; +} + +static void wq_item_append(struct xe_engine *e) +{ + struct xe_guc *guc = engine_to_guc(e); + struct xe_device *xe = guc_to_xe(guc); + struct iosys_map map = xe_lrc_parallel_map(e->lrc); + u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + 3]; + u32 wqi_size = (e->width + 3) * sizeof(u32); + u32 len_dw = (wqi_size / sizeof(u32)) - 1; + int i = 0, j; + + if (wqi_size > wq_space_until_wrap(e)) { + if (wq_noop_append(e)) + return; + } + if (wq_wait_for_space(e, wqi_size)) + return; + + wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | + FIELD_PREP(WQ_LEN_MASK, len_dw); + wqi[i++] = xe_lrc_descriptor(e->lrc); + wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, e->guc->id) | + FIELD_PREP(WQ_RING_TAIL_MASK, e->lrc->ring.tail / sizeof(u64)); + wqi[i++] = 0; + for (j = 1; j < e->width; ++j) { + struct xe_lrc *lrc = e->lrc + j; + + wqi[i++] = lrc->ring.tail / sizeof(u64); + } + + XE_BUG_ON(i != wqi_size / sizeof(u32)); + + iosys_map_incr(&map, offsetof(struct parallel_scratch, + wq[e->guc->wqi_tail / sizeof(u32)])); + xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); + e->guc->wqi_tail += wqi_size; + XE_BUG_ON(e->guc->wqi_tail > WQ_SIZE); + + xe_device_wmb(xe); + + map = xe_lrc_parallel_map(e->lrc); + parallel_write(xe, map, wq_desc.tail, e->guc->wqi_tail); +} + +#define RESUME_PENDING ~0x0ull +static void submit_engine(struct xe_engine *e) +{ + struct xe_guc *guc = engine_to_guc(e); + struct xe_lrc *lrc = e->lrc; + u32 action[3]; + u32 g2h_len = 0; + u32 num_g2h = 0; + int len = 0; + bool extra_submit = false; + + XE_BUG_ON(!engine_registered(e)); + + if (xe_engine_is_parallel(e)) + wq_item_append(e); + else + xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); + + if (engine_suspended(e) && !xe_engine_is_parallel(e)) + return; + + if (!engine_enabled(e) && !engine_suspended(e)) { + action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; + action[len++] = e->guc->id; + action[len++] = GUC_CONTEXT_ENABLE; + g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; + num_g2h = 1; + if (xe_engine_is_parallel(e)) + extra_submit = true; + + e->guc->resume_time = RESUME_PENDING; + set_engine_pending_enable(e); + set_engine_enabled(e); + trace_xe_engine_scheduling_enable(e); + } else { + action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; + action[len++] = e->guc->id; + trace_xe_engine_submit(e); + } + + xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); + + if (extra_submit) { + len = 0; + action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; + action[len++] = e->guc->id; + trace_xe_engine_submit(e); + + xe_guc_ct_send(&guc->ct, action, len, 0, 0); + } +} + +static struct dma_fence * +guc_engine_run_job(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + struct xe_engine *e = job->engine; + + XE_BUG_ON((engine_destroyed(e) || engine_pending_disable(e)) && + !engine_banned(e) && !engine_suspended(e)); + + trace_xe_sched_job_run(job); + + if (!engine_killed_or_banned(e) && !xe_sched_job_is_error(job)) { + if (!engine_registered(e)) + register_engine(e); + e->ring_ops->emit_job(job); + submit_engine(e); + } + + if (test_and_set_bit(JOB_FLAG_SUBMIT, &job->fence->flags)) + return job->fence; + else + return dma_fence_get(job->fence); +} + +static void guc_engine_free_job(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + + trace_xe_sched_job_free(job); + xe_sched_job_put(job); +} + +static int guc_read_stopped(struct xe_guc *guc) +{ + return atomic_read(&guc->submission_state.stopped); +} + +#define MAKE_SCHED_CONTEXT_ACTION(e, enable_disable) \ + u32 action[] = { \ + XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ + e->guc->id, \ + GUC_CONTEXT_##enable_disable, \ + } + +static void disable_scheduling_deregister(struct xe_guc *guc, + struct xe_engine *e) +{ + MAKE_SCHED_CONTEXT_ACTION(e, DISABLE); + int ret; + + set_min_preemption_timeout(guc, e); + smp_rmb(); + ret = wait_event_timeout(guc->ct.wq, !engine_pending_enable(e) || + guc_read_stopped(guc), HZ * 5); + if (!ret) { + struct xe_gpu_scheduler *sched = &e->guc->sched; + + XE_WARN_ON("Pending enable failed to respond"); + xe_sched_submission_start(sched); + xe_gt_reset_async(e->gt); + xe_sched_tdr_queue_imm(sched); + return; + } + + clear_engine_enabled(e); + set_engine_pending_disable(e); + set_engine_destroyed(e); + trace_xe_engine_scheduling_disable(e); + + /* + * Reserve space for both G2H here as the 2nd G2H is sent from a G2H + * handler and we are not allowed to reserved G2H space in handlers. + */ + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET + + G2H_LEN_DW_DEREGISTER_CONTEXT, 2); +} + +static void guc_engine_print(struct xe_engine *e, struct drm_printer *p); + +#if IS_ENABLED(CONFIG_DRM_XE_SIMPLE_ERROR_CAPTURE) +static void simple_error_capture(struct xe_engine *e) +{ + struct xe_guc *guc = engine_to_guc(e); + struct drm_printer p = drm_err_printer(""); + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + u32 adj_logical_mask = e->logical_mask; + u32 width_mask = (0x1 << e->width) - 1; + int i; + bool cookie; + + if (e->vm && !e->vm->error_capture.capture_once) { + e->vm->error_capture.capture_once = true; + cookie = dma_fence_begin_signalling(); + for (i = 0; e->width > 1 && i < XE_HW_ENGINE_MAX_INSTANCE;) { + if (adj_logical_mask & BIT(i)) { + adj_logical_mask |= width_mask << i; + i += e->width; + } else { + ++i; + } + } + + xe_force_wake_get(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL); + xe_guc_ct_print(&guc->ct, &p); + guc_engine_print(e, &p); + for_each_hw_engine(hwe, guc_to_gt(guc), id) { + if (hwe->class != e->hwe->class || + !(BIT(hwe->logical_instance) & adj_logical_mask)) + continue; + xe_hw_engine_print_state(hwe, &p); + } + xe_analyze_vm(&p, e->vm, e->gt->info.id); + xe_force_wake_put(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL); + dma_fence_end_signalling(cookie); + } +} +#else +static void simple_error_capture(struct xe_engine *e) +{ +} +#endif + +static enum drm_gpu_sched_stat +guc_engine_timedout_job(struct drm_sched_job *drm_job) +{ + struct xe_sched_job *job = to_xe_sched_job(drm_job); + struct xe_sched_job *tmp_job; + struct xe_engine *e = job->engine; + struct xe_gpu_scheduler *sched = &e->guc->sched; + struct xe_device *xe = guc_to_xe(engine_to_guc(e)); + int err = -ETIME; + int i = 0; + + if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) { + XE_WARN_ON(e->flags & ENGINE_FLAG_KERNEL); + XE_WARN_ON(e->flags & ENGINE_FLAG_VM && !engine_killed(e)); + + drm_notice(&xe->drm, "Timedout job: seqno=%u, guc_id=%d, flags=0x%lx", + xe_sched_job_seqno(job), e->guc->id, e->flags); + simple_error_capture(e); + } else { + drm_dbg(&xe->drm, "Timedout signaled job: seqno=%u, guc_id=%d, flags=0x%lx", + xe_sched_job_seqno(job), e->guc->id, e->flags); + } + trace_xe_sched_job_timedout(job); + + /* Kill the run_job entry point */ + xe_sched_submission_stop(sched); + + /* + * Kernel jobs should never fail, nor should VM jobs if they do + * somethings has gone wrong and the GT needs a reset + */ + if (e->flags & ENGINE_FLAG_KERNEL || + (e->flags & ENGINE_FLAG_VM && !engine_killed(e))) { + if (!xe_sched_invalidate_job(job, 2)) { + xe_sched_add_pending_job(sched, job); + xe_sched_submission_start(sched); + xe_gt_reset_async(e->gt); + goto out; + } + } + + /* Engine state now stable, disable scheduling if needed */ + if (engine_enabled(e)) { + struct xe_guc *guc = engine_to_guc(e); + int ret; + + if (engine_reset(e)) + err = -EIO; + set_engine_banned(e); + xe_engine_get(e); + disable_scheduling_deregister(engine_to_guc(e), e); + + /* + * Must wait for scheduling to be disabled before signalling + * any fences, if GT broken the GT reset code should signal us. + * + * FIXME: Tests can generate a ton of 0x6000 (IOMMU CAT fault + * error) messages which can cause the schedule disable to get + * lost. If this occurs, trigger a GT reset to recover. + */ + smp_rmb(); + ret = wait_event_timeout(guc->ct.wq, + !engine_pending_disable(e) || + guc_read_stopped(guc), HZ * 5); + if (!ret) { + XE_WARN_ON("Schedule disable failed to respond"); + xe_sched_add_pending_job(sched, job); + xe_sched_submission_start(sched); + xe_gt_reset_async(e->gt); + xe_sched_tdr_queue_imm(sched); + goto out; + } + } + + /* Stop fence signaling */ + xe_hw_fence_irq_stop(e->fence_irq); + + /* + * Fence state now stable, stop / start scheduler which cleans up any + * fences that are complete + */ + xe_sched_add_pending_job(sched, job); + xe_sched_submission_start(sched); + xe_sched_tdr_queue_imm(&e->guc->sched); + + /* Mark all outstanding jobs as bad, thus completing them */ + spin_lock(&sched->base.job_list_lock); + list_for_each_entry(tmp_job, &sched->base.pending_list, drm.list) + xe_sched_job_set_error(tmp_job, !i++ ? err : -ECANCELED); + spin_unlock(&sched->base.job_list_lock); + + /* Start fence signaling */ + xe_hw_fence_irq_start(e->fence_irq); + +out: + return DRM_GPU_SCHED_STAT_NOMINAL; +} + +static void __guc_engine_fini_async(struct work_struct *w) +{ + struct xe_guc_engine *ge = + container_of(w, struct xe_guc_engine, fini_async); + struct xe_engine *e = ge->engine; + struct xe_guc *guc = engine_to_guc(e); + + trace_xe_engine_destroy(e); + + if (e->flags & ENGINE_FLAG_PERSISTENT) + xe_device_remove_persitent_engines(gt_to_xe(e->gt), e); + release_guc_id(guc, e); + xe_sched_entity_fini(&ge->entity); + xe_sched_fini(&ge->sched); + + if (!(e->flags & ENGINE_FLAG_KERNEL)) { + kfree(ge); + xe_engine_fini(e); + } +} + +static void guc_engine_fini_async(struct xe_engine *e) +{ + bool kernel = e->flags & ENGINE_FLAG_KERNEL; + + INIT_WORK(&e->guc->fini_async, __guc_engine_fini_async); + queue_work(system_unbound_wq, &e->guc->fini_async); + + /* We must block on kernel engines so slabs are empty on driver unload */ + if (kernel) { + struct xe_guc_engine *ge = e->guc; + + flush_work(&ge->fini_async); + kfree(ge); + xe_engine_fini(e); + } +} + +static void __guc_engine_fini(struct xe_guc *guc, struct xe_engine *e) +{ + /* + * Might be done from within the GPU scheduler, need to do async as we + * fini the scheduler when the engine is fini'd, the scheduler can't + * complete fini within itself (circular dependency). Async resolves + * this we and don't really care when everything is fini'd, just that it + * is. + */ + guc_engine_fini_async(e); +} + +static void __guc_engine_process_msg_cleanup(struct xe_sched_msg *msg) +{ + struct xe_engine *e = msg->private_data; + struct xe_guc *guc = engine_to_guc(e); + + XE_BUG_ON(e->flags & ENGINE_FLAG_KERNEL); + trace_xe_engine_cleanup_entity(e); + + if (engine_registered(e)) + disable_scheduling_deregister(guc, e); + else + __guc_engine_fini(guc, e); +} + +static bool guc_engine_allowed_to_change_state(struct xe_engine *e) +{ + return !engine_killed_or_banned(e) && engine_registered(e); +} + +static void __guc_engine_process_msg_set_sched_props(struct xe_sched_msg *msg) +{ + struct xe_engine *e = msg->private_data; + struct xe_guc *guc = engine_to_guc(e); + + if (guc_engine_allowed_to_change_state(e)) + init_policies(guc, e); + kfree(msg); +} + +static void suspend_fence_signal(struct xe_engine *e) +{ + struct xe_guc *guc = engine_to_guc(e); + + XE_BUG_ON(!engine_suspended(e) && !engine_killed(e) && + !guc_read_stopped(guc)); + XE_BUG_ON(!e->guc->suspend_pending); + + e->guc->suspend_pending = false; + smp_wmb(); + wake_up(&e->guc->suspend_wait); +} + +static void __guc_engine_process_msg_suspend(struct xe_sched_msg *msg) +{ + struct xe_engine *e = msg->private_data; + struct xe_guc *guc = engine_to_guc(e); + + if (guc_engine_allowed_to_change_state(e) && !engine_suspended(e) && + engine_enabled(e)) { + wait_event(guc->ct.wq, e->guc->resume_time != RESUME_PENDING || + guc_read_stopped(guc)); + + if (!guc_read_stopped(guc)) { + MAKE_SCHED_CONTEXT_ACTION(e, DISABLE); + s64 since_resume_ms = + ktime_ms_delta(ktime_get(), + e->guc->resume_time); + s64 wait_ms = e->vm->preempt.min_run_period_ms - + since_resume_ms; + + if (wait_ms > 0 && e->guc->resume_time) + msleep(wait_ms); + + set_engine_suspended(e); + clear_engine_enabled(e); + set_engine_pending_disable(e); + trace_xe_engine_scheduling_disable(e); + + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); + } + } else if (e->guc->suspend_pending) { + set_engine_suspended(e); + suspend_fence_signal(e); + } +} + +static void __guc_engine_process_msg_resume(struct xe_sched_msg *msg) +{ + struct xe_engine *e = msg->private_data; + struct xe_guc *guc = engine_to_guc(e); + + if (guc_engine_allowed_to_change_state(e)) { + MAKE_SCHED_CONTEXT_ACTION(e, ENABLE); + + e->guc->resume_time = RESUME_PENDING; + clear_engine_suspended(e); + set_engine_pending_enable(e); + set_engine_enabled(e); + trace_xe_engine_scheduling_enable(e); + + xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), + G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); + } else { + clear_engine_suspended(e); + } +} + +#define CLEANUP 1 /* Non-zero values to catch uninitialized msg */ +#define SET_SCHED_PROPS 2 +#define SUSPEND 3 +#define RESUME 4 + +static void guc_engine_process_msg(struct xe_sched_msg *msg) +{ + trace_xe_sched_msg_recv(msg); + + switch (msg->opcode) { + case CLEANUP: + __guc_engine_process_msg_cleanup(msg); + break; + case SET_SCHED_PROPS: + __guc_engine_process_msg_set_sched_props(msg); + break; + case SUSPEND: + __guc_engine_process_msg_suspend(msg); + break; + case RESUME: + __guc_engine_process_msg_resume(msg); + break; + default: + XE_BUG_ON("Unknown message type"); + } +} + +static const struct drm_sched_backend_ops drm_sched_ops = { + .run_job = guc_engine_run_job, + .free_job = guc_engine_free_job, + .timedout_job = guc_engine_timedout_job, +}; + +static const struct xe_sched_backend_ops xe_sched_ops = { + .process_msg = guc_engine_process_msg, +}; + +static int guc_engine_init(struct xe_engine *e) +{ + struct xe_gpu_scheduler *sched; + struct xe_guc *guc = engine_to_guc(e); + struct xe_guc_engine *ge; + long timeout; + int err; + + XE_BUG_ON(!xe_device_guc_submission_enabled(guc_to_xe(guc))); + + ge = kzalloc(sizeof(*ge), GFP_KERNEL); + if (!ge) + return -ENOMEM; + + e->guc = ge; + ge->engine = e; + init_waitqueue_head(&ge->suspend_wait); + + timeout = xe_vm_no_dma_fences(e->vm) ? MAX_SCHEDULE_TIMEOUT : HZ * 5; + err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, NULL, + e->lrc[0].ring.size / MAX_JOB_SIZE_BYTES, + 64, timeout, guc_to_gt(guc)->ordered_wq, NULL, + e->name, gt_to_xe(e->gt)->drm.dev); + if (err) + goto err_free; + + sched = &ge->sched; + err = xe_sched_entity_init(&ge->entity, sched); + if (err) + goto err_sched; + e->priority = XE_ENGINE_PRIORITY_NORMAL; + + mutex_lock(&guc->submission_state.lock); + + err = alloc_guc_id(guc, e); + if (err) + goto err_entity; + + e->entity = &ge->entity; + + if (guc_read_stopped(guc)) + xe_sched_stop(sched); + + mutex_unlock(&guc->submission_state.lock); + + switch (e->class) { + case XE_ENGINE_CLASS_RENDER: + sprintf(e->name, "rcs%d", e->guc->id); + break; + case XE_ENGINE_CLASS_VIDEO_DECODE: + sprintf(e->name, "vcs%d", e->guc->id); + break; + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + sprintf(e->name, "vecs%d", e->guc->id); + break; + case XE_ENGINE_CLASS_COPY: + sprintf(e->name, "bcs%d", e->guc->id); + break; + case XE_ENGINE_CLASS_COMPUTE: + sprintf(e->name, "ccs%d", e->guc->id); + break; + default: + XE_WARN_ON(e->class); + } + + trace_xe_engine_create(e); + + return 0; + +err_entity: + xe_sched_entity_fini(&ge->entity); +err_sched: + xe_sched_fini(&ge->sched); +err_free: + kfree(ge); + + return err; +} + +static void guc_engine_kill(struct xe_engine *e) +{ + trace_xe_engine_kill(e); + set_engine_killed(e); + xe_sched_tdr_queue_imm(&e->guc->sched); +} + +static void guc_engine_add_msg(struct xe_engine *e, struct xe_sched_msg *msg, + u32 opcode) +{ + INIT_LIST_HEAD(&msg->link); + msg->opcode = opcode; + msg->private_data = e; + + trace_xe_sched_msg_add(msg); + xe_sched_add_msg(&e->guc->sched, msg); +} + +#define STATIC_MSG_CLEANUP 0 +#define STATIC_MSG_SUSPEND 1 +#define STATIC_MSG_RESUME 2 +static void guc_engine_fini(struct xe_engine *e) +{ + struct xe_sched_msg *msg = e->guc->static_msgs + STATIC_MSG_CLEANUP; + + if (!(e->flags & ENGINE_FLAG_KERNEL)) + guc_engine_add_msg(e, msg, CLEANUP); + else + __guc_engine_fini(engine_to_guc(e), e); +} + +static int guc_engine_set_priority(struct xe_engine *e, + enum xe_engine_priority priority) +{ + struct xe_sched_msg *msg; + + if (e->priority == priority || engine_killed_or_banned(e)) + return 0; + + msg = kmalloc(sizeof(*msg), GFP_KERNEL); + if (!msg) + return -ENOMEM; + + guc_engine_add_msg(e, msg, SET_SCHED_PROPS); + e->priority = priority; + + return 0; +} + +static int guc_engine_set_timeslice(struct xe_engine *e, u32 timeslice_us) +{ + struct xe_sched_msg *msg; + + if (e->sched_props.timeslice_us == timeslice_us || + engine_killed_or_banned(e)) + return 0; + + msg = kmalloc(sizeof(*msg), GFP_KERNEL); + if (!msg) + return -ENOMEM; + + e->sched_props.timeslice_us = timeslice_us; + guc_engine_add_msg(e, msg, SET_SCHED_PROPS); + + return 0; +} + +static int guc_engine_set_preempt_timeout(struct xe_engine *e, + u32 preempt_timeout_us) +{ + struct xe_sched_msg *msg; + + if (e->sched_props.preempt_timeout_us == preempt_timeout_us || + engine_killed_or_banned(e)) + return 0; + + msg = kmalloc(sizeof(*msg), GFP_KERNEL); + if (!msg) + return -ENOMEM; + + e->sched_props.preempt_timeout_us = preempt_timeout_us; + guc_engine_add_msg(e, msg, SET_SCHED_PROPS); + + return 0; +} + +static int guc_engine_set_job_timeout(struct xe_engine *e, u32 job_timeout_ms) +{ + struct xe_gpu_scheduler *sched = &e->guc->sched; + + XE_BUG_ON(engine_registered(e)); + XE_BUG_ON(engine_banned(e)); + XE_BUG_ON(engine_killed(e)); + + sched->base.timeout = job_timeout_ms; + + return 0; +} + +static int guc_engine_suspend(struct xe_engine *e) +{ + struct xe_sched_msg *msg = e->guc->static_msgs + STATIC_MSG_SUSPEND; + + if (engine_killed_or_banned(e) || e->guc->suspend_pending) + return -EINVAL; + + e->guc->suspend_pending = true; + guc_engine_add_msg(e, msg, SUSPEND); + + return 0; +} + +static void guc_engine_suspend_wait(struct xe_engine *e) +{ + struct xe_guc *guc = engine_to_guc(e); + + wait_event(e->guc->suspend_wait, !e->guc->suspend_pending || + guc_read_stopped(guc)); +} + +static void guc_engine_resume(struct xe_engine *e) +{ + struct xe_sched_msg *msg = e->guc->static_msgs + STATIC_MSG_RESUME; + + XE_BUG_ON(e->guc->suspend_pending); + + xe_mocs_init_engine(e); + guc_engine_add_msg(e, msg, RESUME); +} + +/* + * All of these functions are an abstraction layer which other parts of XE can + * use to trap into the GuC backend. All of these functions, aside from init, + * really shouldn't do much other than trap into the DRM scheduler which + * synchronizes these operations. + */ +static const struct xe_engine_ops guc_engine_ops = { + .init = guc_engine_init, + .kill = guc_engine_kill, + .fini = guc_engine_fini, + .set_priority = guc_engine_set_priority, + .set_timeslice = guc_engine_set_timeslice, + .set_preempt_timeout = guc_engine_set_preempt_timeout, + .set_job_timeout = guc_engine_set_job_timeout, + .suspend = guc_engine_suspend, + .suspend_wait = guc_engine_suspend_wait, + .resume = guc_engine_resume, +}; + +static void guc_engine_stop(struct xe_guc *guc, struct xe_engine *e) +{ + struct xe_gpu_scheduler *sched = &e->guc->sched; + + /* Stop scheduling + flush any DRM scheduler operations */ + xe_sched_submission_stop(sched); + + /* Clean up lost G2H + reset engine state */ + if (engine_destroyed(e) && engine_registered(e)) { + if (engine_banned(e)) + xe_engine_put(e); + else + __guc_engine_fini(guc, e); + } + if (e->guc->suspend_pending) { + set_engine_suspended(e); + suspend_fence_signal(e); + } + atomic_and(ENGINE_STATE_DESTROYED | ENGINE_STATE_SUSPENDED, + &e->guc->state); + e->guc->resume_time = 0; + trace_xe_engine_stop(e); + + /* + * Ban any engine (aside from kernel and engines used for VM ops) with a + * started but not complete job or if a job has gone through a GT reset + * more than twice. + */ + if (!(e->flags & (ENGINE_FLAG_KERNEL | ENGINE_FLAG_VM))) { + struct xe_sched_job *job = xe_sched_first_pending_job(sched); + + if (job) { + if ((xe_sched_job_started(job) && + !xe_sched_job_completed(job)) || + xe_sched_invalidate_job(job, 2)) { + trace_xe_sched_job_ban(job); + xe_sched_tdr_queue_imm(&e->guc->sched); + set_engine_banned(e); + } + } + } +} + +int xe_guc_submit_reset_prepare(struct xe_guc *guc) +{ + int ret; + + /* + * Using an atomic here rather than submission_state.lock as this + * function can be called while holding the CT lock (engine reset + * failure). submission_state.lock needs the CT lock to resubmit jobs. + * Atomic is not ideal, but it works to prevent against concurrent reset + * and releasing any TDRs waiting on guc->submission_state.stopped. + */ + ret = atomic_fetch_or(1, &guc->submission_state.stopped); + smp_wmb(); + wake_up_all(&guc->ct.wq); + + return ret; +} + +void xe_guc_submit_reset_wait(struct xe_guc *guc) +{ + wait_event(guc->ct.wq, !guc_read_stopped(guc)); +} + +int xe_guc_submit_stop(struct xe_guc *guc) +{ + struct xe_engine *e; + unsigned long index; + + XE_BUG_ON(guc_read_stopped(guc) != 1); + + mutex_lock(&guc->submission_state.lock); + + xa_for_each(&guc->submission_state.engine_lookup, index, e) + guc_engine_stop(guc, e); + + mutex_unlock(&guc->submission_state.lock); + + /* + * No one can enter the backend at this point, aside from new engine + * creation which is protected by guc->submission_state.lock. + */ + + return 0; +} + +static void guc_engine_start(struct xe_engine *e) +{ + struct xe_gpu_scheduler *sched = &e->guc->sched; + + if (!engine_killed_or_banned(e)) { + int i; + + trace_xe_engine_resubmit(e); + for (i = 0; i < e->width; ++i) + xe_lrc_set_ring_head(e->lrc + i, e->lrc[i].ring.tail); + xe_sched_resubmit_jobs(sched); + } + + xe_sched_submission_start(sched); +} + +int xe_guc_submit_start(struct xe_guc *guc) +{ + struct xe_engine *e; + unsigned long index; + + XE_BUG_ON(guc_read_stopped(guc) != 1); + + mutex_lock(&guc->submission_state.lock); + atomic_dec(&guc->submission_state.stopped); + xa_for_each(&guc->submission_state.engine_lookup, index, e) + guc_engine_start(e); + mutex_unlock(&guc->submission_state.lock); + + wake_up_all(&guc->ct.wq); + + return 0; +} + +static struct xe_engine * +g2h_engine_lookup(struct xe_guc *guc, u32 guc_id) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_engine *e; + + if (unlikely(guc_id >= GUC_ID_MAX)) { + drm_err(&xe->drm, "Invalid guc_id %u", guc_id); + return NULL; + } + + e = xa_load(&guc->submission_state.engine_lookup, guc_id); + if (unlikely(!e)) { + drm_err(&xe->drm, "Not engine present for guc_id %u", guc_id); + return NULL; + } + + XE_BUG_ON(e->guc->id != guc_id); + + return e; +} + +static void deregister_engine(struct xe_guc *guc, struct xe_engine *e) +{ + u32 action[] = { + XE_GUC_ACTION_DEREGISTER_CONTEXT, + e->guc->id, + }; + + trace_xe_engine_deregister(e); + + xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); +} + +int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_engine *e; + u32 guc_id = msg[0]; + + if (unlikely(len < 2)) { + drm_err(&xe->drm, "Invalid length %u", len); + return -EPROTO; + } + + e = g2h_engine_lookup(guc, guc_id); + if (unlikely(!e)) + return -EPROTO; + + if (unlikely(!engine_pending_enable(e) && + !engine_pending_disable(e))) { + drm_err(&xe->drm, "Unexpected engine state 0x%04x", + atomic_read(&e->guc->state)); + return -EPROTO; + } + + trace_xe_engine_scheduling_done(e); + + if (engine_pending_enable(e)) { + e->guc->resume_time = ktime_get(); + clear_engine_pending_enable(e); + smp_wmb(); + wake_up_all(&guc->ct.wq); + } else { + clear_engine_pending_disable(e); + if (e->guc->suspend_pending) { + suspend_fence_signal(e); + } else { + if (engine_banned(e)) { + smp_wmb(); + wake_up_all(&guc->ct.wq); + } + deregister_engine(guc, e); + } + } + + return 0; +} + +int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_engine *e; + u32 guc_id = msg[0]; + + if (unlikely(len < 1)) { + drm_err(&xe->drm, "Invalid length %u", len); + return -EPROTO; + } + + e = g2h_engine_lookup(guc, guc_id); + if (unlikely(!e)) + return -EPROTO; + + if (!engine_destroyed(e) || engine_pending_disable(e) || + engine_pending_enable(e) || engine_enabled(e)) { + drm_err(&xe->drm, "Unexpected engine state 0x%04x", + atomic_read(&e->guc->state)); + return -EPROTO; + } + + trace_xe_engine_deregister_done(e); + + clear_engine_registered(e); + if (engine_banned(e)) + xe_engine_put(e); + else + __guc_engine_fini(guc, e); + + return 0; +} + +int xe_guc_engine_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_engine *e; + u32 guc_id = msg[0]; + + if (unlikely(len < 1)) { + drm_err(&xe->drm, "Invalid length %u", len); + return -EPROTO; + } + + e = g2h_engine_lookup(guc, guc_id); + if (unlikely(!e)) + return -EPROTO; + + drm_info(&xe->drm, "Engine reset: guc_id=%d", guc_id); + + /* FIXME: Do error capture, most likely async */ + + trace_xe_engine_reset(e); + + /* + * A banned engine is a NOP at this point (came from + * guc_engine_timedout_job). Otherwise, kick drm scheduler to cancel + * jobs by setting timeout of the job to the minimum value kicking + * guc_engine_timedout_job. + */ + set_engine_reset(e); + if (!engine_banned(e)) + xe_sched_tdr_queue_imm(&e->guc->sched); + + return 0; +} + +int xe_guc_engine_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, + u32 len) +{ + struct xe_device *xe = guc_to_xe(guc); + struct xe_engine *e; + u32 guc_id = msg[0]; + + if (unlikely(len < 1)) { + drm_err(&xe->drm, "Invalid length %u", len); + return -EPROTO; + } + + e = g2h_engine_lookup(guc, guc_id); + if (unlikely(!e)) + return -EPROTO; + + drm_warn(&xe->drm, "Engine memory cat error: guc_id=%d", guc_id); + trace_xe_engine_memory_cat_error(e); + + /* Treat the same as engine reset */ + set_engine_reset(e); + if (!engine_banned(e)) + xe_sched_tdr_queue_imm(&e->guc->sched); + + return 0; +} + +int xe_guc_engine_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) +{ + struct xe_device *xe = guc_to_xe(guc); + u8 guc_class, instance; + u32 reason; + + if (unlikely(len != 3)) { + drm_err(&xe->drm, "Invalid length %u", len); + return -EPROTO; + } + + guc_class = msg[0]; + instance = msg[1]; + reason = msg[2]; + + /* Unexpected failure of a hardware feature, log an actual error */ + drm_err(&xe->drm, "GuC engine reset request failed on %d:%d because 0x%08X", + guc_class, instance, reason); + + xe_gt_reset_async(guc_to_gt(guc)); + + return 0; +} + +static void guc_engine_wq_print(struct xe_engine *e, struct drm_printer *p) +{ + struct xe_guc *guc = engine_to_guc(e); + struct xe_device *xe = guc_to_xe(guc); + struct iosys_map map = xe_lrc_parallel_map(e->lrc); + int i; + + drm_printf(p, "\tWQ head: %u (internal), %d (memory)\n", + e->guc->wqi_head, parallel_read(xe, map, wq_desc.head)); + drm_printf(p, "\tWQ tail: %u (internal), %d (memory)\n", + e->guc->wqi_tail, parallel_read(xe, map, wq_desc.tail)); + drm_printf(p, "\tWQ status: %u\n", + parallel_read(xe, map, wq_desc.wq_status)); + if (parallel_read(xe, map, wq_desc.head) != + parallel_read(xe, map, wq_desc.tail)) { + for (i = parallel_read(xe, map, wq_desc.head); + i != parallel_read(xe, map, wq_desc.tail); + i = (i + sizeof(u32)) % WQ_SIZE) + drm_printf(p, "\tWQ[%ld]: 0x%08x\n", i / sizeof(u32), + parallel_read(xe, map, wq[i / sizeof(u32)])); + } +} + +static void guc_engine_print(struct xe_engine *e, struct drm_printer *p) +{ + struct xe_gpu_scheduler *sched = &e->guc->sched; + struct xe_sched_job *job; + int i; + + drm_printf(p, "\nGuC ID: %d\n", e->guc->id); + drm_printf(p, "\tName: %s\n", e->name); + drm_printf(p, "\tClass: %d\n", e->class); + drm_printf(p, "\tLogical mask: 0x%x\n", e->logical_mask); + drm_printf(p, "\tWidth: %d\n", e->width); + drm_printf(p, "\tRef: %d\n", kref_read(&e->refcount)); + drm_printf(p, "\tTimeout: %ld (ms)\n", sched->base.timeout); + drm_printf(p, "\tTimeslice: %u (us)\n", e->sched_props.timeslice_us); + drm_printf(p, "\tPreempt timeout: %u (us)\n", + e->sched_props.preempt_timeout_us); + for (i = 0; i < e->width; ++i ) { + struct xe_lrc *lrc = e->lrc + i; + + drm_printf(p, "\tHW Context Desc: 0x%08x\n", + lower_32_bits(xe_lrc_ggtt_addr(lrc))); + drm_printf(p, "\tLRC Head: (memory) %u\n", + xe_lrc_ring_head(lrc)); + drm_printf(p, "\tLRC Tail: (internal) %u, (memory) %u\n", + lrc->ring.tail, + xe_lrc_read_ctx_reg(lrc, CTX_RING_TAIL)); + drm_printf(p, "\tStart seqno: (memory) %d\n", + xe_lrc_start_seqno(lrc)); + drm_printf(p, "\tSeqno: (memory) %d\n", xe_lrc_seqno(lrc)); + } + drm_printf(p, "\tSchedule State: 0x%x\n", atomic_read(&e->guc->state)); + drm_printf(p, "\tFlags: 0x%lx\n", e->flags); + if (xe_engine_is_parallel(e)) + guc_engine_wq_print(e, p); + + spin_lock(&sched->base.job_list_lock); + list_for_each_entry(job, &sched->base.pending_list, drm.list) + drm_printf(p, "\tJob: seqno=%d, fence=%d, finished=%d\n", + xe_sched_job_seqno(job), + dma_fence_is_signaled(job->fence) ? 1 : 0, + dma_fence_is_signaled(&job->drm.s_fence->finished) ? + 1 : 0); + spin_unlock(&sched->base.job_list_lock); +} + +void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) +{ + struct xe_engine *e; + unsigned long index; + + if (!xe_device_guc_submission_enabled(guc_to_xe(guc))) + return; + + mutex_lock(&guc->submission_state.lock); + xa_for_each(&guc->submission_state.engine_lookup, index, e) + guc_engine_print(e, p); + mutex_unlock(&guc->submission_state.lock); +} diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h new file mode 100644 index 000000000000..8002734d6f24 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_SUBMIT_H_ +#define _XE_GUC_SUBMIT_H_ + +#include + +struct drm_printer; +struct xe_engine; +struct xe_guc; + +int xe_guc_submit_init(struct xe_guc *guc); +void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p); + +int xe_guc_submit_reset_prepare(struct xe_guc *guc); +void xe_guc_submit_reset_wait(struct xe_guc *guc); +int xe_guc_submit_stop(struct xe_guc *guc); +int xe_guc_submit_start(struct xe_guc *guc); + +int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_engine_reset_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_engine_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, + u32 len); +int xe_guc_engine_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len); + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h new file mode 100644 index 000000000000..ca177853cc12 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_types.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_TYPES_H_ +#define _XE_GUC_TYPES_H_ + +#include +#include + +#include "xe_guc_ads_types.h" +#include "xe_guc_ct_types.h" +#include "xe_guc_fwif.h" +#include "xe_guc_log_types.h" +#include "xe_guc_pc_types.h" +#include "xe_uc_fw_types.h" + +/** + * struct xe_guc - Graphic micro controller + */ +struct xe_guc { + /** @fw: Generic uC firmware management */ + struct xe_uc_fw fw; + /** @log: GuC log */ + struct xe_guc_log log; + /** @ads: GuC ads */ + struct xe_guc_ads ads; + /** @ct: GuC ct */ + struct xe_guc_ct ct; + /** @pc: GuC Power Conservation */ + struct xe_guc_pc pc; + /** @submission_state: GuC submission state */ + struct { + /** @engine_lookup: Lookup an xe_engine from guc_id */ + struct xarray engine_lookup; + /** @guc_ids: used to allocate new guc_ids, single-lrc */ + struct ida guc_ids; + /** @guc_ids_bitmap: used to allocate new guc_ids, multi-lrc */ + unsigned long *guc_ids_bitmap; + /** @stopped: submissions are stopped */ + atomic_t stopped; + /** @lock: protects submission state */ + struct mutex lock; + /** @suspend: suspend fence state */ + struct { + /** @lock: suspend fences lock */ + spinlock_t lock; + /** @context: suspend fences context */ + u64 context; + /** @seqno: suspend fences seqno */ + u32 seqno; + } suspend; + } submission_state; + /** @hwconfig: Hardware config state */ + struct { + /** @bo: buffer object of the hardware config */ + struct xe_bo *bo; + /** @size: size of the hardware config */ + u32 size; + } hwconfig; + + /** + * @notify_reg: Register which is written to notify GuC of H2G messages + */ + u32 notify_reg; + /** @params: Control params for fw initialization */ + u32 params[GUC_CTL_MAX_DWORDS]; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_huc.c b/drivers/gpu/drm/xe/xe_huc.c new file mode 100644 index 000000000000..93b22fac6e14 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc.c @@ -0,0 +1,131 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_guc_reg.h" +#include "xe_huc.h" +#include "xe_mmio.h" +#include "xe_uc_fw.h" + +static struct xe_gt * +huc_to_gt(struct xe_huc *huc) +{ + return container_of(huc, struct xe_gt, uc.huc); +} + +static struct xe_device * +huc_to_xe(struct xe_huc *huc) +{ + return gt_to_xe(huc_to_gt(huc)); +} + +static struct xe_guc * +huc_to_guc(struct xe_huc *huc) +{ + return &container_of(huc, struct xe_uc, huc)->guc; +} + +int xe_huc_init(struct xe_huc *huc) +{ + struct xe_device *xe = huc_to_xe(huc); + int ret; + + huc->fw.type = XE_UC_FW_TYPE_HUC; + ret = xe_uc_fw_init(&huc->fw); + if (ret) + goto out; + + xe_uc_fw_change_status(&huc->fw, XE_UC_FIRMWARE_LOADABLE); + + return 0; + +out: + if (xe_uc_fw_is_disabled(&huc->fw)) { + drm_info(&xe->drm, "HuC disabled\n"); + return 0; + } + drm_err(&xe->drm, "HuC init failed with %d", ret); + return ret; +} + +int xe_huc_upload(struct xe_huc *huc) +{ + if (xe_uc_fw_is_disabled(&huc->fw)) + return 0; + return xe_uc_fw_upload(&huc->fw, 0, HUC_UKERNEL); +} + +int xe_huc_auth(struct xe_huc *huc) +{ + struct xe_device *xe = huc_to_xe(huc); + struct xe_gt *gt = huc_to_gt(huc); + struct xe_guc *guc = huc_to_guc(huc); + int ret; + if (xe_uc_fw_is_disabled(&huc->fw)) + return 0; + + XE_BUG_ON(xe_uc_fw_is_running(&huc->fw)); + + if (!xe_uc_fw_is_loaded(&huc->fw)) + return -ENOEXEC; + + ret = xe_guc_auth_huc(guc, xe_bo_ggtt_addr(huc->fw.bo) + + xe_uc_fw_rsa_offset(&huc->fw)); + if (ret) { + drm_err(&xe->drm, "HuC: GuC did not ack Auth request %d\n", + ret); + goto fail; + } + + ret = xe_mmio_wait32(gt, GEN11_HUC_KERNEL_LOAD_INFO.reg, + HUC_LOAD_SUCCESSFUL, + HUC_LOAD_SUCCESSFUL, 100); + if (ret) { + drm_err(&xe->drm, "HuC: Firmware not verified %d\n", ret); + goto fail; + } + + xe_uc_fw_change_status(&huc->fw, XE_UC_FIRMWARE_RUNNING); + drm_dbg(&xe->drm, "HuC authenticated\n"); + + return 0; + +fail: + drm_err(&xe->drm, "HuC authentication failed %d\n", ret); + xe_uc_fw_change_status(&huc->fw, XE_UC_FIRMWARE_LOAD_FAIL); + + return ret; +} + +void xe_huc_sanitize(struct xe_huc *huc) +{ + if (xe_uc_fw_is_disabled(&huc->fw)) + return; + xe_uc_fw_change_status(&huc->fw, XE_UC_FIRMWARE_LOADABLE); +} + +void xe_huc_print_info(struct xe_huc *huc, struct drm_printer *p) +{ + struct xe_gt *gt = huc_to_gt(huc); + int err; + + xe_uc_fw_print(&huc->fw, p); + + if (xe_uc_fw_is_disabled(&huc->fw)) + return; + + err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (err) + return; + + drm_printf(p, "\nHuC status: 0x%08x\n", + xe_mmio_read32(gt, GEN11_HUC_KERNEL_LOAD_INFO.reg)); + + xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); +} diff --git a/drivers/gpu/drm/xe/xe_huc.h b/drivers/gpu/drm/xe/xe_huc.h new file mode 100644 index 000000000000..5802c43b6ce2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HUC_H_ +#define _XE_HUC_H_ + +#include "xe_huc_types.h" + +struct drm_printer; + +int xe_huc_init(struct xe_huc *huc); +int xe_huc_upload(struct xe_huc *huc); +int xe_huc_auth(struct xe_huc *huc); +void xe_huc_sanitize(struct xe_huc *huc); +void xe_huc_print_info(struct xe_huc *huc, struct drm_printer *p); + +#endif diff --git a/drivers/gpu/drm/xe/xe_huc_debugfs.c b/drivers/gpu/drm/xe/xe_huc_debugfs.c new file mode 100644 index 000000000000..268bac36336a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc_debugfs.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_huc.h" +#include "xe_huc_debugfs.h" +#include "xe_macros.h" + +static struct xe_gt * +huc_to_gt(struct xe_huc *huc) +{ + return container_of(huc, struct xe_gt, uc.huc); +} + +static struct xe_device * +huc_to_xe(struct xe_huc *huc) +{ + return gt_to_xe(huc_to_gt(huc)); +} + +static struct xe_huc *node_to_huc(struct drm_info_node *node) +{ + return node->info_ent->data; +} + +static int huc_info(struct seq_file *m, void *data) +{ + struct xe_huc *huc = node_to_huc(m->private); + struct xe_device *xe = huc_to_xe(huc); + struct drm_printer p = drm_seq_file_printer(m); + + xe_device_mem_access_get(xe); + xe_huc_print_info(huc, &p); + xe_device_mem_access_put(xe); + + return 0; +} + +static const struct drm_info_list debugfs_list[] = { + {"huc_info", huc_info, 0}, +}; + +void xe_huc_debugfs_register(struct xe_huc *huc, struct dentry *parent) +{ + struct drm_minor *minor = huc_to_xe(huc)->drm.primary; + struct drm_info_list *local; + int i; + +#define DEBUGFS_SIZE ARRAY_SIZE(debugfs_list) * sizeof(struct drm_info_list) + local = drmm_kmalloc(&huc_to_xe(huc)->drm, DEBUGFS_SIZE, GFP_KERNEL); + if (!local) { + XE_WARN_ON("Couldn't allocate memory"); + return; + } + + memcpy(local, debugfs_list, DEBUGFS_SIZE); +#undef DEBUGFS_SIZE + + for (i = 0; i < ARRAY_SIZE(debugfs_list); ++i) + local[i].data = huc; + + drm_debugfs_create_files(local, + ARRAY_SIZE(debugfs_list), + parent, minor); +} diff --git a/drivers/gpu/drm/xe/xe_huc_debugfs.h b/drivers/gpu/drm/xe/xe_huc_debugfs.h new file mode 100644 index 000000000000..ec58f1818804 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HUC_DEBUGFS_H_ +#define _XE_HUC_DEBUGFS_H_ + +struct dentry; +struct xe_huc; + +void xe_huc_debugfs_register(struct xe_huc *huc, struct dentry *parent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_huc_types.h b/drivers/gpu/drm/xe/xe_huc_types.h new file mode 100644 index 000000000000..cae6d19097df --- /dev/null +++ b/drivers/gpu/drm/xe/xe_huc_types.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HUC_TYPES_H_ +#define _XE_HUC_TYPES_H_ + +#include "xe_uc_fw_types.h" + +/** + * struct xe_huc - HuC + */ +struct xe_huc { + /** @fw: Generic uC firmware management */ + struct xe_uc_fw fw; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c new file mode 100644 index 000000000000..fd89dd90131c --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_hw_engine.h" + +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_execlist.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_topology.h" +#include "xe_hw_fence.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_mmio.h" +#include "xe_reg_sr.h" +#include "xe_sched_job.h" +#include "xe_wa.h" + +#include "gt/intel_engine_regs.h" +#include "i915_reg.h" +#include "gt/intel_gt_regs.h" + +#define MAX_MMIO_BASES 3 +struct engine_info { + const char *name; + unsigned int class : 8; + unsigned int instance : 8; + enum xe_force_wake_domains domain; + /* mmio bases table *must* be sorted in reverse graphics_ver order */ + struct engine_mmio_base { + unsigned int graphics_ver : 8; + unsigned int base : 24; + } mmio_bases[MAX_MMIO_BASES]; +}; + +static const struct engine_info engine_infos[] = { + [XE_HW_ENGINE_RCS0] = { + .name = "rcs0", + .class = XE_ENGINE_CLASS_RENDER, + .instance = 0, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 1, .base = RENDER_RING_BASE } + }, + }, + [XE_HW_ENGINE_BCS0] = { + .name = "bcs0", + .class = XE_ENGINE_CLASS_COPY, + .instance = 0, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 6, .base = BLT_RING_BASE } + }, + }, + [XE_HW_ENGINE_BCS1] = { + .name = "bcs1", + .class = XE_ENGINE_CLASS_COPY, + .instance = 1, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHPC_BCS1_RING_BASE } + }, + }, + [XE_HW_ENGINE_BCS2] = { + .name = "bcs2", + .class = XE_ENGINE_CLASS_COPY, + .instance = 2, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHPC_BCS2_RING_BASE } + }, + }, + [XE_HW_ENGINE_BCS3] = { + .name = "bcs3", + .class = XE_ENGINE_CLASS_COPY, + .instance = 3, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHPC_BCS3_RING_BASE } + }, + }, + [XE_HW_ENGINE_BCS4] = { + .name = "bcs4", + .class = XE_ENGINE_CLASS_COPY, + .instance = 4, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHPC_BCS4_RING_BASE } + }, + }, + [XE_HW_ENGINE_BCS5] = { + .name = "bcs5", + .class = XE_ENGINE_CLASS_COPY, + .instance = 5, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHPC_BCS5_RING_BASE } + }, + }, + [XE_HW_ENGINE_BCS6] = { + .name = "bcs6", + .class = XE_ENGINE_CLASS_COPY, + .instance = 6, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHPC_BCS6_RING_BASE } + }, + }, + [XE_HW_ENGINE_BCS7] = { + .name = "bcs7", + .class = XE_ENGINE_CLASS_COPY, + .instance = 7, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHPC_BCS7_RING_BASE } + }, + }, + [XE_HW_ENGINE_BCS8] = { + .name = "bcs8", + .class = XE_ENGINE_CLASS_COPY, + .instance = 8, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHPC_BCS8_RING_BASE } + }, + }, + + [XE_HW_ENGINE_VCS0] = { + .name = "vcs0", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 0, + .domain = XE_FW_MEDIA_VDBOX0, + .mmio_bases = { + { .graphics_ver = 11, .base = GEN11_BSD_RING_BASE }, + { .graphics_ver = 6, .base = GEN6_BSD_RING_BASE }, + { .graphics_ver = 4, .base = BSD_RING_BASE } + }, + }, + [XE_HW_ENGINE_VCS1] = { + .name = "vcs1", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 1, + .domain = XE_FW_MEDIA_VDBOX1, + .mmio_bases = { + { .graphics_ver = 11, .base = GEN11_BSD2_RING_BASE }, + { .graphics_ver = 8, .base = GEN8_BSD2_RING_BASE } + }, + }, + [XE_HW_ENGINE_VCS2] = { + .name = "vcs2", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 2, + .domain = XE_FW_MEDIA_VDBOX2, + .mmio_bases = { + { .graphics_ver = 11, .base = GEN11_BSD3_RING_BASE } + }, + }, + [XE_HW_ENGINE_VCS3] = { + .name = "vcs3", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 3, + .domain = XE_FW_MEDIA_VDBOX3, + .mmio_bases = { + { .graphics_ver = 11, .base = GEN11_BSD4_RING_BASE } + }, + }, + [XE_HW_ENGINE_VCS4] = { + .name = "vcs4", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 4, + .domain = XE_FW_MEDIA_VDBOX4, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_BSD5_RING_BASE } + }, + }, + [XE_HW_ENGINE_VCS5] = { + .name = "vcs5", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 5, + .domain = XE_FW_MEDIA_VDBOX5, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_BSD6_RING_BASE } + }, + }, + [XE_HW_ENGINE_VCS6] = { + .name = "vcs6", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 6, + .domain = XE_FW_MEDIA_VDBOX6, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_BSD7_RING_BASE } + }, + }, + [XE_HW_ENGINE_VCS7] = { + .name = "vcs7", + .class = XE_ENGINE_CLASS_VIDEO_DECODE, + .instance = 7, + .domain = XE_FW_MEDIA_VDBOX7, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_BSD8_RING_BASE } + }, + }, + [XE_HW_ENGINE_VECS0] = { + .name = "vecs0", + .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, + .instance = 0, + .domain = XE_FW_MEDIA_VEBOX0, + .mmio_bases = { + { .graphics_ver = 11, .base = GEN11_VEBOX_RING_BASE }, + { .graphics_ver = 7, .base = VEBOX_RING_BASE } + }, + }, + [XE_HW_ENGINE_VECS1] = { + .name = "vecs1", + .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, + .instance = 1, + .domain = XE_FW_MEDIA_VEBOX1, + .mmio_bases = { + { .graphics_ver = 11, .base = GEN11_VEBOX2_RING_BASE } + }, + }, + [XE_HW_ENGINE_VECS2] = { + .name = "vecs2", + .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, + .instance = 2, + .domain = XE_FW_MEDIA_VEBOX2, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_VEBOX3_RING_BASE } + }, + }, + [XE_HW_ENGINE_VECS3] = { + .name = "vecs3", + .class = XE_ENGINE_CLASS_VIDEO_ENHANCE, + .instance = 3, + .domain = XE_FW_MEDIA_VEBOX3, + .mmio_bases = { + { .graphics_ver = 12, .base = XEHP_VEBOX4_RING_BASE } + }, + }, + [XE_HW_ENGINE_CCS0] = { + .name = "ccs0", + .class = XE_ENGINE_CLASS_COMPUTE, + .instance = 0, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = GEN12_COMPUTE0_RING_BASE }, + }, + }, + [XE_HW_ENGINE_CCS1] = { + .name = "ccs1", + .class = XE_ENGINE_CLASS_COMPUTE, + .instance = 1, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = GEN12_COMPUTE1_RING_BASE }, + }, + }, + [XE_HW_ENGINE_CCS2] = { + .name = "ccs2", + .class = XE_ENGINE_CLASS_COMPUTE, + .instance = 2, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = GEN12_COMPUTE2_RING_BASE }, + }, + }, + [XE_HW_ENGINE_CCS3] = { + .name = "ccs3", + .class = XE_ENGINE_CLASS_COMPUTE, + .instance = 3, + .domain = XE_FW_RENDER, + .mmio_bases = { + { .graphics_ver = 12, .base = GEN12_COMPUTE3_RING_BASE }, + }, + }, +}; + +static u32 engine_info_mmio_base(const struct engine_info *info, + unsigned int graphics_ver) +{ + int i; + + for (i = 0; i < MAX_MMIO_BASES; i++) + if (graphics_ver >= info->mmio_bases[i].graphics_ver) + break; + + XE_BUG_ON(i == MAX_MMIO_BASES); + XE_BUG_ON(!info->mmio_bases[i].base); + + return info->mmio_bases[i].base; +} + +static void hw_engine_fini(struct drm_device *drm, void *arg) +{ + struct xe_hw_engine *hwe = arg; + + if (hwe->exl_port) + xe_execlist_port_destroy(hwe->exl_port); + xe_lrc_finish(&hwe->kernel_lrc); + + xe_bo_unpin_map_no_vm(hwe->hwsp); + + hwe->gt = NULL; +} + +static void hw_engine_mmio_write32(struct xe_hw_engine *hwe, u32 reg, u32 val) +{ + XE_BUG_ON(reg & hwe->mmio_base); + xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain); + + xe_mmio_write32(hwe->gt, reg + hwe->mmio_base, val); +} + +static u32 hw_engine_mmio_read32(struct xe_hw_engine *hwe, u32 reg) +{ + XE_BUG_ON(reg & hwe->mmio_base); + xe_force_wake_assert_held(gt_to_fw(hwe->gt), hwe->domain); + + return xe_mmio_read32(hwe->gt, reg + hwe->mmio_base); +} + +void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe) +{ + u32 ccs_mask = + xe_hw_engine_mask_per_class(hwe->gt, XE_ENGINE_CLASS_COMPUTE); + + if (hwe->class == XE_ENGINE_CLASS_COMPUTE && ccs_mask & BIT(0)) + xe_mmio_write32(hwe->gt, GEN12_RCU_MODE.reg, + _MASKED_BIT_ENABLE(GEN12_RCU_MODE_CCS_ENABLE)); + + hw_engine_mmio_write32(hwe, RING_HWSTAM(0).reg, ~0x0); + hw_engine_mmio_write32(hwe, RING_HWS_PGA(0).reg, + xe_bo_ggtt_addr(hwe->hwsp)); + hw_engine_mmio_write32(hwe, RING_MODE_GEN7(0).reg, + _MASKED_BIT_ENABLE(GEN11_GFX_DISABLE_LEGACY_MODE)); + hw_engine_mmio_write32(hwe, RING_MI_MODE(0).reg, + _MASKED_BIT_DISABLE(STOP_RING)); + hw_engine_mmio_read32(hwe, RING_MI_MODE(0).reg); +} + +static void hw_engine_init_early(struct xe_gt *gt, struct xe_hw_engine *hwe, + enum xe_hw_engine_id id) +{ + struct xe_device *xe = gt_to_xe(gt); + const struct engine_info *info; + + if (WARN_ON(id >= ARRAY_SIZE(engine_infos) || !engine_infos[id].name)) + return; + + if (!(gt->info.engine_mask & BIT(id))) + return; + + info = &engine_infos[id]; + + XE_BUG_ON(hwe->gt); + + hwe->gt = gt; + hwe->class = info->class; + hwe->instance = info->instance; + hwe->mmio_base = engine_info_mmio_base(info, GRAPHICS_VER(xe)); + hwe->domain = info->domain; + hwe->name = info->name; + hwe->fence_irq = >->fence_irq[info->class]; + hwe->engine_id = id; + + xe_reg_sr_init(&hwe->reg_sr, hwe->name, gt_to_xe(gt)); + xe_wa_process_engine(hwe); + + xe_reg_sr_init(&hwe->reg_whitelist, hwe->name, gt_to_xe(gt)); + xe_reg_whitelist_process_engine(hwe); +} + +static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, + enum xe_hw_engine_id id) +{ + struct xe_device *xe = gt_to_xe(gt); + int err; + + XE_BUG_ON(id >= ARRAY_SIZE(engine_infos) || !engine_infos[id].name); + XE_BUG_ON(!(gt->info.engine_mask & BIT(id))); + + xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); + xe_reg_sr_apply_whitelist(&hwe->reg_whitelist, hwe->mmio_base, gt); + + hwe->hwsp = xe_bo_create_locked(xe, gt, NULL, SZ_4K, ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_GGTT_BIT); + if (IS_ERR(hwe->hwsp)) { + err = PTR_ERR(hwe->hwsp); + goto err_name; + } + + err = xe_bo_pin(hwe->hwsp); + if (err) + goto err_unlock_put_hwsp; + + err = xe_bo_vmap(hwe->hwsp); + if (err) + goto err_unpin_hwsp; + + xe_bo_unlock_no_vm(hwe->hwsp); + + err = xe_lrc_init(&hwe->kernel_lrc, hwe, NULL, NULL, SZ_16K); + if (err) + goto err_hwsp; + + if (!xe_device_guc_submission_enabled(xe)) { + hwe->exl_port = xe_execlist_port_create(xe, hwe); + if (IS_ERR(hwe->exl_port)) { + err = PTR_ERR(hwe->exl_port); + goto err_kernel_lrc; + } + } + + if (xe_device_guc_submission_enabled(xe)) + xe_hw_engine_enable_ring(hwe); + + /* We reserve the highest BCS instance for USM */ + if (xe->info.supports_usm && hwe->class == XE_ENGINE_CLASS_COPY) + gt->usm.reserved_bcs_instance = hwe->instance; + + err = drmm_add_action_or_reset(&xe->drm, hw_engine_fini, hwe); + if (err) + return err; + + return 0; + +err_unpin_hwsp: + xe_bo_unpin(hwe->hwsp); +err_unlock_put_hwsp: + xe_bo_unlock_no_vm(hwe->hwsp); + xe_bo_put(hwe->hwsp); +err_kernel_lrc: + xe_lrc_finish(&hwe->kernel_lrc); +err_hwsp: + xe_bo_put(hwe->hwsp); +err_name: + hwe->name = NULL; + + return err; +} + +static void hw_engine_setup_logical_mapping(struct xe_gt *gt) +{ + int class; + + /* FIXME: Doing a simple logical mapping that works for most hardware */ + for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) { + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + int logical_instance = 0; + + for_each_hw_engine(hwe, gt, id) + if (hwe->class == class) + hwe->logical_instance = logical_instance++; + } +} + +static void read_fuses(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 media_fuse; + u16 vdbox_mask; + u16 vebox_mask; + u32 bcs_mask; + int i, j; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + /* + * FIXME: Hack job, thinking we should have table of vfuncs for each + * class which picks the correct vfunc based on IP version. + */ + + media_fuse = xe_mmio_read32(gt, GEN11_GT_VEBOX_VDBOX_DISABLE.reg); + if (GRAPHICS_VERx100(xe) < 1250) + media_fuse = ~media_fuse; + + vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK; + vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >> + GEN11_GT_VEBOX_DISABLE_SHIFT; + + for (i = XE_HW_ENGINE_VCS0, j = 0; i <= XE_HW_ENGINE_VCS7; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + if (!(BIT(j) & vdbox_mask)) { + gt->info.engine_mask &= ~BIT(i); + drm_info(&xe->drm, "vcs%u fused off\n", j); + } + } + + for (i = XE_HW_ENGINE_VECS0, j = 0; i <= XE_HW_ENGINE_VECS3; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + if (!(BIT(j) & vebox_mask)) { + gt->info.engine_mask &= ~BIT(i); + drm_info(&xe->drm, "vecs%u fused off\n", j); + } + } + + bcs_mask = xe_mmio_read32(gt, GEN10_MIRROR_FUSE3.reg); + bcs_mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK, bcs_mask); + + for (i = XE_HW_ENGINE_BCS1, j = 0; i <= XE_HW_ENGINE_BCS8; ++i, ++j) { + if (!(gt->info.engine_mask & BIT(i))) + continue; + + if (!(BIT(j/2) & bcs_mask)) { + gt->info.engine_mask &= ~BIT(i); + drm_info(&xe->drm, "bcs%u fused off\n", j); + } + } + + /* TODO: compute engines */ +} + +int xe_hw_engines_init_early(struct xe_gt *gt) +{ + int i; + + read_fuses(gt); + + for (i = 0; i < ARRAY_SIZE(gt->hw_engines); i++) + hw_engine_init_early(gt, >->hw_engines[i], i); + + return 0; +} + +int xe_hw_engines_init(struct xe_gt *gt) +{ + int err; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(hwe, gt, id) { + err = hw_engine_init(gt, hwe, id); + if (err) + return err; + } + + hw_engine_setup_logical_mapping(gt); + + return 0; +} + +void xe_hw_engine_handle_irq(struct xe_hw_engine *hwe, u16 intr_vec) +{ + wake_up_all(>_to_xe(hwe->gt)->ufence_wq); + + if (hwe->irq_handler) + hwe->irq_handler(hwe, intr_vec); + + if (intr_vec & GT_RENDER_USER_INTERRUPT) + xe_hw_fence_irq_run(hwe->fence_irq); +} + +void xe_hw_engine_print_state(struct xe_hw_engine *hwe, struct drm_printer *p) +{ + if (!xe_hw_engine_is_valid(hwe)) + return; + + drm_printf(p, "%s (physical), logical instance=%d\n", hwe->name, + hwe->logical_instance); + drm_printf(p, "\tForcewake: domain 0x%x, ref %d\n", + hwe->domain, + xe_force_wake_ref(gt_to_fw(hwe->gt), hwe->domain)); + drm_printf(p, "\tMMIO base: 0x%08x\n", hwe->mmio_base); + + drm_printf(p, "\tHWSTAM: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_HWSTAM(0).reg)); + drm_printf(p, "\tRING_HWS_PGA: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_HWS_PGA(0).reg)); + + drm_printf(p, "\tRING_EXECLIST_STATUS_LO: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_EXECLIST_STATUS_LO(0).reg)); + drm_printf(p, "\tRING_EXECLIST_STATUS_HI: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_EXECLIST_STATUS_HI(0).reg)); + drm_printf(p, "\tRING_EXECLIST_SQ_CONTENTS_LO: 0x%08x\n", + hw_engine_mmio_read32(hwe, + RING_EXECLIST_SQ_CONTENTS(0).reg)); + drm_printf(p, "\tRING_EXECLIST_SQ_CONTENTS_HI: 0x%08x\n", + hw_engine_mmio_read32(hwe, + RING_EXECLIST_SQ_CONTENTS(0).reg) + 4); + drm_printf(p, "\tRING_EXECLIST_CONTROL: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_EXECLIST_CONTROL(0).reg)); + + drm_printf(p, "\tRING_START: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_START(0).reg)); + drm_printf(p, "\tRING_HEAD: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_HEAD(0).reg) & HEAD_ADDR); + drm_printf(p, "\tRING_TAIL: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_TAIL(0).reg) & TAIL_ADDR); + drm_printf(p, "\tRING_CTL: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_CTL(0).reg)); + drm_printf(p, "\tRING_MODE: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_MI_MODE(0).reg)); + drm_printf(p, "\tRING_MODE_GEN7: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_MODE_GEN7(0).reg)); + + drm_printf(p, "\tRING_IMR: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_IMR(0).reg)); + drm_printf(p, "\tRING_ESR: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_ESR(0).reg)); + drm_printf(p, "\tRING_EMR: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_EMR(0).reg)); + drm_printf(p, "\tRING_EIR: 0x%08x\n", + hw_engine_mmio_read32(hwe, RING_EIR(0).reg)); + + drm_printf(p, "\tACTHD: 0x%08x_%08x\n", + hw_engine_mmio_read32(hwe, RING_ACTHD_UDW(0).reg), + hw_engine_mmio_read32(hwe, RING_ACTHD(0).reg)); + drm_printf(p, "\tBBADDR: 0x%08x_%08x\n", + hw_engine_mmio_read32(hwe, RING_BBADDR_UDW(0).reg), + hw_engine_mmio_read32(hwe, RING_BBADDR(0).reg)); + drm_printf(p, "\tDMA_FADDR: 0x%08x_%08x\n", + hw_engine_mmio_read32(hwe, RING_DMA_FADD_UDW(0).reg), + hw_engine_mmio_read32(hwe, RING_DMA_FADD(0).reg)); + + drm_printf(p, "\tIPEIR: 0x%08x\n", + hw_engine_mmio_read32(hwe, IPEIR(0).reg)); + drm_printf(p, "\tIPEHR: 0x%08x\n\n", + hw_engine_mmio_read32(hwe, IPEHR(0).reg)); + + if (hwe->class == XE_ENGINE_CLASS_COMPUTE) + drm_printf(p, "\tGEN12_RCU_MODE: 0x%08x\n", + xe_mmio_read32(hwe->gt, GEN12_RCU_MODE.reg)); + +} + +u32 xe_hw_engine_mask_per_class(struct xe_gt *gt, + enum xe_engine_class engine_class) +{ + u32 mask = 0; + enum xe_hw_engine_id id; + + for (id = 0; id < XE_NUM_HW_ENGINES; ++id) { + if (engine_infos[id].class == engine_class && + gt->info.engine_mask & BIT(id)) + mask |= BIT(engine_infos[id].instance); + } + return mask; +} + +bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe) +{ + struct xe_gt *gt = hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + + return xe->info.supports_usm && hwe->class == XE_ENGINE_CLASS_COPY && + hwe->instance == gt->usm.reserved_bcs_instance; +} diff --git a/drivers/gpu/drm/xe/xe_hw_engine.h b/drivers/gpu/drm/xe/xe_hw_engine.h new file mode 100644 index 000000000000..ceab65397256 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_HW_ENGINE_H_ +#define _XE_HW_ENGINE_H_ + +#include "xe_hw_engine_types.h" + +struct drm_printer; + +int xe_hw_engines_init_early(struct xe_gt *gt); +int xe_hw_engines_init(struct xe_gt *gt); +void xe_hw_engine_handle_irq(struct xe_hw_engine *hwe, u16 intr_vec); +void xe_hw_engine_enable_ring(struct xe_hw_engine *hwe); +void xe_hw_engine_print_state(struct xe_hw_engine *hwe, struct drm_printer *p); +u32 xe_hw_engine_mask_per_class(struct xe_gt *gt, + enum xe_engine_class engine_class); + +bool xe_hw_engine_is_reserved(struct xe_hw_engine *hwe); +static inline bool xe_hw_engine_is_valid(struct xe_hw_engine *hwe) +{ + return hwe->name; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_engine_types.h b/drivers/gpu/drm/xe/xe_hw_engine_types.h new file mode 100644 index 000000000000..05a2fdc381d7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_engine_types.h @@ -0,0 +1,107 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HW_ENGINE_TYPES_H_ +#define _XE_HW_ENGINE_TYPES_H_ + +#include "xe_force_wake_types.h" +#include "xe_lrc_types.h" +#include "xe_reg_sr_types.h" + +/* See "Engine ID Definition" struct in the Icelake PRM */ +enum xe_engine_class { + XE_ENGINE_CLASS_RENDER = 0, + XE_ENGINE_CLASS_VIDEO_DECODE = 1, + XE_ENGINE_CLASS_VIDEO_ENHANCE = 2, + XE_ENGINE_CLASS_COPY = 3, + XE_ENGINE_CLASS_OTHER = 4, + XE_ENGINE_CLASS_COMPUTE = 5, + XE_ENGINE_CLASS_MAX = 6, +}; + +enum xe_hw_engine_id { + XE_HW_ENGINE_RCS0, + XE_HW_ENGINE_BCS0, + XE_HW_ENGINE_BCS1, + XE_HW_ENGINE_BCS2, + XE_HW_ENGINE_BCS3, + XE_HW_ENGINE_BCS4, + XE_HW_ENGINE_BCS5, + XE_HW_ENGINE_BCS6, + XE_HW_ENGINE_BCS7, + XE_HW_ENGINE_BCS8, + XE_HW_ENGINE_VCS0, + XE_HW_ENGINE_VCS1, + XE_HW_ENGINE_VCS2, + XE_HW_ENGINE_VCS3, + XE_HW_ENGINE_VCS4, + XE_HW_ENGINE_VCS5, + XE_HW_ENGINE_VCS6, + XE_HW_ENGINE_VCS7, + XE_HW_ENGINE_VECS0, + XE_HW_ENGINE_VECS1, + XE_HW_ENGINE_VECS2, + XE_HW_ENGINE_VECS3, + XE_HW_ENGINE_CCS0, + XE_HW_ENGINE_CCS1, + XE_HW_ENGINE_CCS2, + XE_HW_ENGINE_CCS3, + XE_NUM_HW_ENGINES, +}; + +/* FIXME: s/XE_HW_ENGINE_MAX_INSTANCE/XE_HW_ENGINE_MAX_COUNT */ +#define XE_HW_ENGINE_MAX_INSTANCE 9 + +struct xe_bo; +struct xe_execlist_port; +struct xe_gt; + +/** + * struct xe_hw_engine - Hardware engine + * + * Contains all the hardware engine state for physical instances. + */ +struct xe_hw_engine { + /** @gt: graphics tile this hw engine belongs to */ + struct xe_gt *gt; + /** @name: name of this hw engine */ + const char *name; + /** @class: class of this hw engine */ + enum xe_engine_class class; + /** @instance: physical instance of this hw engine */ + u16 instance; + /** @logical_instance: logical instance of this hw engine */ + u16 logical_instance; + /** @mmio_base: MMIO base address of this hw engine*/ + u32 mmio_base; + /** + * @reg_sr: table with registers to be restored on GT init/resume/reset + */ + struct xe_reg_sr reg_sr; + /** + * @reg_whitelist: table with registers to be whitelisted + */ + struct xe_reg_sr reg_whitelist; + /** + * @reg_lrc: LRC workaround registers + */ + struct xe_reg_sr reg_lrc; + /** @domain: force wake domain of this hw engine */ + enum xe_force_wake_domains domain; + /** @hwsp: hardware status page buffer object */ + struct xe_bo *hwsp; + /** @kernel_lrc: Kernel LRC (should be replaced /w an xe_engine) */ + struct xe_lrc kernel_lrc; + /** @exl_port: execlists port */ + struct xe_execlist_port *exl_port; + /** @fence_irq: fence IRQ to run when a hw engine IRQ is received */ + struct xe_hw_fence_irq *fence_irq; + /** @irq_handler: IRQ handler to run when hw engine IRQ is received */ + void (*irq_handler)(struct xe_hw_engine *, u16); + /** @engine_id: id for this hw engine */ + enum xe_hw_engine_id engine_id; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_fence.c b/drivers/gpu/drm/xe/xe_hw_fence.c new file mode 100644 index 000000000000..e56ca2867545 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_fence.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_hw_fence.h" + +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_hw_engine.h" +#include "xe_macros.h" +#include "xe_map.h" +#include "xe_trace.h" + +static struct kmem_cache *xe_hw_fence_slab; + +int __init xe_hw_fence_module_init(void) +{ + xe_hw_fence_slab = kmem_cache_create("xe_hw_fence", + sizeof(struct xe_hw_fence), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!xe_hw_fence_slab) + return -ENOMEM; + + return 0; +} + +void xe_hw_fence_module_exit(void) +{ + rcu_barrier(); + kmem_cache_destroy(xe_hw_fence_slab); +} + +static struct xe_hw_fence *fence_alloc(void) +{ + return kmem_cache_zalloc(xe_hw_fence_slab, GFP_KERNEL); +} + +static void fence_free(struct rcu_head *rcu) +{ + struct xe_hw_fence *fence = + container_of(rcu, struct xe_hw_fence, dma.rcu); + + if (!WARN_ON_ONCE(!fence)) + kmem_cache_free(xe_hw_fence_slab, fence); +} + +static void hw_fence_irq_run_cb(struct irq_work *work) +{ + struct xe_hw_fence_irq *irq = container_of(work, typeof(*irq), work); + struct xe_hw_fence *fence, *next; + bool tmp; + + tmp = dma_fence_begin_signalling(); + spin_lock(&irq->lock); + if (irq->enabled) { + list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { + struct dma_fence *dma_fence = &fence->dma; + + trace_xe_hw_fence_try_signal(fence); + if (dma_fence_is_signaled_locked(dma_fence)) { + trace_xe_hw_fence_signal(fence); + list_del_init(&fence->irq_link); + dma_fence_put(dma_fence); + } + } + } + spin_unlock(&irq->lock); + dma_fence_end_signalling(tmp); +} + +void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq) +{ + spin_lock_init(&irq->lock); + init_irq_work(&irq->work, hw_fence_irq_run_cb); + INIT_LIST_HEAD(&irq->pending); + irq->enabled = true; +} + +void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq) +{ + struct xe_hw_fence *fence, *next; + unsigned long flags; + int err; + bool tmp; + + if (XE_WARN_ON(!list_empty(&irq->pending))) { + tmp = dma_fence_begin_signalling(); + spin_lock_irqsave(&irq->lock, flags); + list_for_each_entry_safe(fence, next, &irq->pending, irq_link) { + list_del_init(&fence->irq_link); + err = dma_fence_signal_locked(&fence->dma); + dma_fence_put(&fence->dma); + XE_WARN_ON(err); + } + spin_unlock_irqrestore(&irq->lock, flags); + dma_fence_end_signalling(tmp); + } +} + +void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq) +{ + irq_work_queue(&irq->work); +} + +void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq) +{ + spin_lock_irq(&irq->lock); + irq->enabled = false; + spin_unlock_irq(&irq->lock); +} + +void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq) +{ + spin_lock_irq(&irq->lock); + irq->enabled = true; + spin_unlock_irq(&irq->lock); + + irq_work_queue(&irq->work); +} + +void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt, + struct xe_hw_fence_irq *irq, const char *name) +{ + ctx->gt = gt; + ctx->irq = irq; + ctx->dma_fence_ctx = dma_fence_context_alloc(1); + ctx->next_seqno = 1; + sprintf(ctx->name, "%s", name); +} + +void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx) +{ +} + +static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence); + +static struct xe_hw_fence_irq *xe_hw_fence_irq(struct xe_hw_fence *fence) +{ + return container_of(fence->dma.lock, struct xe_hw_fence_irq, lock); +} + +static const char *xe_hw_fence_get_driver_name(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + + return dev_name(gt_to_xe(fence->ctx->gt)->drm.dev); +} + +static const char *xe_hw_fence_get_timeline_name(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + + return fence->ctx->name; +} + +static bool xe_hw_fence_signaled(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + struct xe_device *xe = gt_to_xe(fence->ctx->gt); + u32 seqno = xe_map_rd(xe, &fence->seqno_map, 0, u32); + + return dma_fence->error || + (s32)fence->dma.seqno <= (s32)seqno; +} + +static bool xe_hw_fence_enable_signaling(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + struct xe_hw_fence_irq *irq = xe_hw_fence_irq(fence); + + dma_fence_get(dma_fence); + list_add_tail(&fence->irq_link, &irq->pending); + + /* SW completed (no HW IRQ) so kick handler to signal fence */ + if (xe_hw_fence_signaled(dma_fence)) + xe_hw_fence_irq_run(irq); + + return true; +} + +static void xe_hw_fence_release(struct dma_fence *dma_fence) +{ + struct xe_hw_fence *fence = to_xe_hw_fence(dma_fence); + + trace_xe_hw_fence_free(fence); + XE_BUG_ON(!list_empty(&fence->irq_link)); + call_rcu(&dma_fence->rcu, fence_free); +} + +static const struct dma_fence_ops xe_hw_fence_ops = { + .get_driver_name = xe_hw_fence_get_driver_name, + .get_timeline_name = xe_hw_fence_get_timeline_name, + .enable_signaling = xe_hw_fence_enable_signaling, + .signaled = xe_hw_fence_signaled, + .release = xe_hw_fence_release, +}; + +static struct xe_hw_fence *to_xe_hw_fence(struct dma_fence *fence) +{ + if (XE_WARN_ON(fence->ops != &xe_hw_fence_ops)) + return NULL; + + return container_of(fence, struct xe_hw_fence, dma); +} + +struct xe_hw_fence *xe_hw_fence_create(struct xe_hw_fence_ctx *ctx, + struct iosys_map seqno_map) +{ + struct xe_hw_fence *fence; + + fence = fence_alloc(); + if (!fence) + return ERR_PTR(-ENOMEM); + + dma_fence_init(&fence->dma, &xe_hw_fence_ops, &ctx->irq->lock, + ctx->dma_fence_ctx, ctx->next_seqno++); + + fence->ctx = ctx; + fence->seqno_map = seqno_map; + INIT_LIST_HEAD(&fence->irq_link); + + trace_xe_hw_fence_create(fence); + + return fence; +} diff --git a/drivers/gpu/drm/xe/xe_hw_fence.h b/drivers/gpu/drm/xe/xe_hw_fence.h new file mode 100644 index 000000000000..07f202db6526 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_fence.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_HW_FENCE_H_ +#define _XE_HW_FENCE_H_ + +#include "xe_hw_fence_types.h" + +int xe_hw_fence_module_init(void); +void xe_hw_fence_module_exit(void); + +void xe_hw_fence_irq_init(struct xe_hw_fence_irq *irq); +void xe_hw_fence_irq_finish(struct xe_hw_fence_irq *irq); +void xe_hw_fence_irq_run(struct xe_hw_fence_irq *irq); +void xe_hw_fence_irq_stop(struct xe_hw_fence_irq *irq); +void xe_hw_fence_irq_start(struct xe_hw_fence_irq *irq); + +void xe_hw_fence_ctx_init(struct xe_hw_fence_ctx *ctx, struct xe_gt *gt, + struct xe_hw_fence_irq *irq, const char *name); +void xe_hw_fence_ctx_finish(struct xe_hw_fence_ctx *ctx); + +struct xe_hw_fence *xe_hw_fence_create(struct xe_hw_fence_ctx *ctx, + struct iosys_map seqno_map); + +#endif diff --git a/drivers/gpu/drm/xe/xe_hw_fence_types.h b/drivers/gpu/drm/xe/xe_hw_fence_types.h new file mode 100644 index 000000000000..a78e50eb3cb8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_hw_fence_types.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_HW_FENCE_TYPES_H_ +#define _XE_HW_FENCE_TYPES_H_ + +#include +#include +#include +#include +#include + +struct xe_gt; + +/** + * struct xe_hw_fence_irq - hardware fence IRQ handler + * + * One per engine class, signals completed xe_hw_fences, triggered via hw engine + * interrupt. On each trigger, search list of pending fences and signal. + */ +struct xe_hw_fence_irq { + /** @lock: protects all xe_hw_fences + pending list */ + spinlock_t lock; + /** @work: IRQ worker run to signal the fences */ + struct irq_work work; + /** @pending: list of pending xe_hw_fences */ + struct list_head pending; + /** @enabled: fence signaling enabled */ + bool enabled; +}; + +#define MAX_FENCE_NAME_LEN 16 + +/** + * struct xe_hw_fence_ctx - hardware fence context + * + * The context for a hardware fence. 1 to 1 relationship with xe_engine. Points + * to a xe_hw_fence_irq, maintains serial seqno. + */ +struct xe_hw_fence_ctx { + /** @gt: graphics tile of hardware fence context */ + struct xe_gt *gt; + /** @irq: fence irq handler */ + struct xe_hw_fence_irq *irq; + /** @dma_fence_ctx: dma fence context for hardware fence */ + u64 dma_fence_ctx; + /** @next_seqno: next seqno for hardware fence */ + u32 next_seqno; + /** @name: name of hardware fence context */ + char name[MAX_FENCE_NAME_LEN]; +}; + +/** + * struct xe_hw_fence - hardware fence + * + * Used to indicate a xe_sched_job is complete via a seqno written to memory. + * Signals on error or seqno past. + */ +struct xe_hw_fence { + /** @dma: base dma fence for hardware fence context */ + struct dma_fence dma; + /** @ctx: hardware fence context */ + struct xe_hw_fence_ctx *ctx; + /** @seqno_map: I/O map for seqno */ + struct iosys_map seqno_map; + /** @irq_link: Link in struct xe_hw_fence_irq.pending */ + struct list_head irq_link; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c new file mode 100644 index 000000000000..df2e3573201d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_irq.c @@ -0,0 +1,565 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include + +#include + +#include "xe_device.h" +#include "xe_drv.h" +#include "xe_guc.h" +#include "xe_gt.h" +#include "xe_hw_engine.h" +#include "xe_mmio.h" + +#include "i915_reg.h" +#include "gt/intel_gt_regs.h" + +static void gen3_assert_iir_is_zero(struct xe_gt *gt, i915_reg_t reg) +{ + u32 val = xe_mmio_read32(gt, reg.reg); + + if (val == 0) + return; + + drm_WARN(>_to_xe(gt)->drm, 1, + "Interrupt register 0x%x is not zero: 0x%08x\n", + reg.reg, val); + xe_mmio_write32(gt, reg.reg, 0xffffffff); + xe_mmio_read32(gt, reg.reg); + xe_mmio_write32(gt, reg.reg, 0xffffffff); + xe_mmio_read32(gt, reg.reg); +} + +static void gen3_irq_init(struct xe_gt *gt, + i915_reg_t imr, u32 imr_val, + i915_reg_t ier, u32 ier_val, + i915_reg_t iir) +{ + gen3_assert_iir_is_zero(gt, iir); + + xe_mmio_write32(gt, ier.reg, ier_val); + xe_mmio_write32(gt, imr.reg, imr_val); + xe_mmio_read32(gt, imr.reg); +} +#define GEN3_IRQ_INIT(gt, type, imr_val, ier_val) \ + gen3_irq_init((gt), \ + type##IMR, imr_val, \ + type##IER, ier_val, \ + type##IIR) + +static void gen3_irq_reset(struct xe_gt *gt, i915_reg_t imr, i915_reg_t iir, + i915_reg_t ier) +{ + xe_mmio_write32(gt, imr.reg, 0xffffffff); + xe_mmio_read32(gt, imr.reg); + + xe_mmio_write32(gt, ier.reg, 0); + + /* IIR can theoretically queue up two events. Be paranoid. */ + xe_mmio_write32(gt, iir.reg, 0xffffffff); + xe_mmio_read32(gt, iir.reg); + xe_mmio_write32(gt, iir.reg, 0xffffffff); + xe_mmio_read32(gt, iir.reg); +} +#define GEN3_IRQ_RESET(gt, type) \ + gen3_irq_reset((gt), type##IMR, type##IIR, type##IER) + +static u32 gen11_intr_disable(struct xe_gt *gt) +{ + xe_mmio_write32(gt, GEN11_GFX_MSTR_IRQ.reg, 0); + + /* + * Now with master disabled, get a sample of level indications + * for this interrupt. Indications will be cleared on related acks. + * New indications can and will light up during processing, + * and will generate new interrupt after enabling master. + */ + return xe_mmio_read32(gt, GEN11_GFX_MSTR_IRQ.reg); +} + +static u32 +gen11_gu_misc_irq_ack(struct xe_gt *gt, const u32 master_ctl) +{ + u32 iir; + + if (!(master_ctl & GEN11_GU_MISC_IRQ)) + return 0; + + iir = xe_mmio_read32(gt, GEN11_GU_MISC_IIR.reg); + if (likely(iir)) + xe_mmio_write32(gt, GEN11_GU_MISC_IIR.reg, iir); + + return iir; +} + +static inline void gen11_intr_enable(struct xe_gt *gt, bool stall) +{ + xe_mmio_write32(gt, GEN11_GFX_MSTR_IRQ.reg, GEN11_MASTER_IRQ); + if (stall) + xe_mmio_read32(gt, GEN11_GFX_MSTR_IRQ.reg); +} + +static void gen11_gt_irq_postinstall(struct xe_device *xe, struct xe_gt *gt) +{ + u32 irqs, dmask, smask; + u32 ccs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COMPUTE); + u32 bcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COPY); + + if (xe_device_guc_submission_enabled(xe)) { + irqs = GT_RENDER_USER_INTERRUPT | + GT_RENDER_PIPECTL_NOTIFY_INTERRUPT; + } else { + irqs = GT_RENDER_USER_INTERRUPT | + GT_CS_MASTER_ERROR_INTERRUPT | + GT_CONTEXT_SWITCH_INTERRUPT | + GT_WAIT_SEMAPHORE_INTERRUPT; + } + + dmask = irqs << 16 | irqs; + smask = irqs << 16; + + /* Enable RCS, BCS, VCS and VECS class interrupts. */ + xe_mmio_write32(gt, GEN11_RENDER_COPY_INTR_ENABLE.reg, dmask); + xe_mmio_write32(gt, GEN11_VCS_VECS_INTR_ENABLE.reg, dmask); + if (ccs_mask) + xe_mmio_write32(gt, GEN12_CCS_RSVD_INTR_ENABLE.reg, smask); + + /* Unmask irqs on RCS, BCS, VCS and VECS engines. */ + xe_mmio_write32(gt, GEN11_RCS0_RSVD_INTR_MASK.reg, ~smask); + xe_mmio_write32(gt, GEN11_BCS_RSVD_INTR_MASK.reg, ~smask); + if (bcs_mask & (BIT(1)|BIT(2))) + xe_mmio_write32(gt, XEHPC_BCS1_BCS2_INTR_MASK.reg, ~dmask); + if (bcs_mask & (BIT(3)|BIT(4))) + xe_mmio_write32(gt, XEHPC_BCS3_BCS4_INTR_MASK.reg, ~dmask); + if (bcs_mask & (BIT(5)|BIT(6))) + xe_mmio_write32(gt, XEHPC_BCS5_BCS6_INTR_MASK.reg, ~dmask); + if (bcs_mask & (BIT(7)|BIT(8))) + xe_mmio_write32(gt, XEHPC_BCS7_BCS8_INTR_MASK.reg, ~dmask); + xe_mmio_write32(gt, GEN11_VCS0_VCS1_INTR_MASK.reg, ~dmask); + xe_mmio_write32(gt, GEN11_VCS2_VCS3_INTR_MASK.reg, ~dmask); + //if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5)) + // intel_uncore_write(uncore, GEN12_VCS4_VCS5_INTR_MASK, ~dmask); + //if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7)) + // intel_uncore_write(uncore, GEN12_VCS6_VCS7_INTR_MASK, ~dmask); + xe_mmio_write32(gt, GEN11_VECS0_VECS1_INTR_MASK.reg, ~dmask); + //if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) + // intel_uncore_write(uncore, GEN12_VECS2_VECS3_INTR_MASK, ~dmask); + if (ccs_mask & (BIT(0)|BIT(1))) + xe_mmio_write32(gt, GEN12_CCS0_CCS1_INTR_MASK.reg, ~dmask); + if (ccs_mask & (BIT(2)|BIT(3))) + xe_mmio_write32(gt, GEN12_CCS2_CCS3_INTR_MASK.reg, ~dmask); + + /* + * RPS interrupts will get enabled/disabled on demand when RPS itself + * is enabled/disabled. + */ + /* TODO: gt->pm_ier, gt->pm_imr */ + xe_mmio_write32(gt, GEN11_GPM_WGBOXPERF_INTR_ENABLE.reg, 0); + xe_mmio_write32(gt, GEN11_GPM_WGBOXPERF_INTR_MASK.reg, ~0); + + /* Same thing for GuC interrupts */ + xe_mmio_write32(gt, GEN11_GUC_SG_INTR_ENABLE.reg, 0); + xe_mmio_write32(gt, GEN11_GUC_SG_INTR_MASK.reg, ~0); +} + +static void gen11_irq_postinstall(struct xe_device *xe, struct xe_gt *gt) +{ + /* TODO: PCH */ + + gen11_gt_irq_postinstall(xe, gt); + + GEN3_IRQ_INIT(gt, GEN11_GU_MISC_, ~GEN11_GU_MISC_GSE, + GEN11_GU_MISC_GSE); + + gen11_intr_enable(gt, true); +} + +static u32 +gen11_gt_engine_identity(struct xe_device *xe, + struct xe_gt *gt, + const unsigned int bank, + const unsigned int bit) +{ + u32 timeout_ts; + u32 ident; + + lockdep_assert_held(&xe->irq.lock); + + xe_mmio_write32(gt, GEN11_IIR_REG_SELECTOR(bank).reg, BIT(bit)); + + /* + * NB: Specs do not specify how long to spin wait, + * so we do ~100us as an educated guess. + */ + timeout_ts = (local_clock() >> 10) + 100; + do { + ident = xe_mmio_read32(gt, GEN11_INTR_IDENTITY_REG(bank).reg); + } while (!(ident & GEN11_INTR_DATA_VALID) && + !time_after32(local_clock() >> 10, timeout_ts)); + + if (unlikely(!(ident & GEN11_INTR_DATA_VALID))) { + drm_err(&xe->drm, "INTR_IDENTITY_REG%u:%u 0x%08x not valid!\n", + bank, bit, ident); + return 0; + } + + xe_mmio_write32(gt, GEN11_INTR_IDENTITY_REG(bank).reg, + GEN11_INTR_DATA_VALID); + + return ident; +} + +#define OTHER_MEDIA_GUC_INSTANCE 16 + +static void +gen11_gt_other_irq_handler(struct xe_gt *gt, const u8 instance, const u16 iir) +{ + if (instance == OTHER_GUC_INSTANCE && !xe_gt_is_media_type(gt)) + return xe_guc_irq_handler(>->uc.guc, iir); + if (instance == OTHER_MEDIA_GUC_INSTANCE && xe_gt_is_media_type(gt)) + return xe_guc_irq_handler(>->uc.guc, iir); + + if (instance != OTHER_GUC_INSTANCE && + instance != OTHER_MEDIA_GUC_INSTANCE) { + WARN_ONCE(1, "unhandled other interrupt instance=0x%x, iir=0x%x\n", + instance, iir); + } +} + +static void gen11_gt_irq_handler(struct xe_device *xe, struct xe_gt *gt, + u32 master_ctl, long unsigned int *intr_dw, + u32 *identity) +{ + unsigned int bank, bit; + u16 instance, intr_vec; + enum xe_engine_class class; + struct xe_hw_engine *hwe; + + spin_lock(&xe->irq.lock); + + for (bank = 0; bank < 2; bank++) { + if (!(master_ctl & GEN11_GT_DW_IRQ(bank))) + continue; + + if (!xe_gt_is_media_type(gt)) { + intr_dw[bank] = + xe_mmio_read32(gt, GEN11_GT_INTR_DW(bank).reg); + for_each_set_bit(bit, intr_dw + bank, 32) + identity[bit] = gen11_gt_engine_identity(xe, gt, + bank, + bit); + xe_mmio_write32(gt, GEN11_GT_INTR_DW(bank).reg, + intr_dw[bank]); + } + + for_each_set_bit(bit, intr_dw + bank, 32) { + class = GEN11_INTR_ENGINE_CLASS(identity[bit]); + instance = GEN11_INTR_ENGINE_INSTANCE(identity[bit]); + intr_vec = GEN11_INTR_ENGINE_INTR(identity[bit]); + + if (class == XE_ENGINE_CLASS_OTHER) { + gen11_gt_other_irq_handler(gt, instance, + intr_vec); + continue; + } + + hwe = xe_gt_hw_engine(gt, class, instance, false); + if (!hwe) + continue; + + xe_hw_engine_handle_irq(hwe, intr_vec); + } + } + + spin_unlock(&xe->irq.lock); +} + +static irqreturn_t gen11_irq_handler(int irq, void *arg) +{ + struct xe_device *xe = arg; + struct xe_gt *gt = xe_device_get_gt(xe, 0); /* Only 1 GT here */ + u32 master_ctl, gu_misc_iir; + long unsigned int intr_dw[2]; + u32 identity[32]; + + master_ctl = gen11_intr_disable(gt); + if (!master_ctl) { + gen11_intr_enable(gt, false); + return IRQ_NONE; + } + + gen11_gt_irq_handler(xe, gt, master_ctl, intr_dw, identity); + + gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl); + + gen11_intr_enable(gt, false); + + return IRQ_HANDLED; +} + +static u32 dg1_intr_disable(struct xe_device *xe) +{ + struct xe_gt *gt = xe_device_get_gt(xe, 0); + u32 val; + + /* First disable interrupts */ + xe_mmio_write32(gt, DG1_MSTR_TILE_INTR.reg, 0); + + /* Get the indication levels and ack the master unit */ + val = xe_mmio_read32(gt, DG1_MSTR_TILE_INTR.reg); + if (unlikely(!val)) + return 0; + + xe_mmio_write32(gt, DG1_MSTR_TILE_INTR.reg, val); + + return val; +} + +static void dg1_intr_enable(struct xe_device *xe, bool stall) +{ + struct xe_gt *gt = xe_device_get_gt(xe, 0); + + xe_mmio_write32(gt, DG1_MSTR_TILE_INTR.reg, DG1_MSTR_IRQ); + if (stall) + xe_mmio_read32(gt, DG1_MSTR_TILE_INTR.reg); +} + +static void dg1_irq_postinstall(struct xe_device *xe, struct xe_gt *gt) +{ + gen11_gt_irq_postinstall(xe, gt); + + GEN3_IRQ_INIT(gt, GEN11_GU_MISC_, ~GEN11_GU_MISC_GSE, + GEN11_GU_MISC_GSE); + + if (gt->info.id + 1 == xe->info.tile_count) + dg1_intr_enable(xe, true); +} + +static irqreturn_t dg1_irq_handler(int irq, void *arg) +{ + struct xe_device *xe = arg; + struct xe_gt *gt; + u32 master_tile_ctl, master_ctl = 0, gu_misc_iir; + long unsigned int intr_dw[2]; + u32 identity[32]; + u8 id; + + /* TODO: This really shouldn't be copied+pasted */ + + master_tile_ctl = dg1_intr_disable(xe); + if (!master_tile_ctl) { + dg1_intr_enable(xe, false); + return IRQ_NONE; + } + + for_each_gt(gt, xe, id) { + if ((master_tile_ctl & DG1_MSTR_TILE(gt->info.vram_id)) == 0) + continue; + + if (!xe_gt_is_media_type(gt)) + master_ctl = xe_mmio_read32(gt, GEN11_GFX_MSTR_IRQ.reg); + + /* + * We might be in irq handler just when PCIe DPC is initiated + * and all MMIO reads will be returned with all 1's. Ignore this + * irq as device is inaccessible. + */ + if (master_ctl == REG_GENMASK(31, 0)) { + dev_dbg(gt_to_xe(gt)->drm.dev, + "Ignore this IRQ as device might be in DPC containment.\n"); + return IRQ_HANDLED; + } + + if (!xe_gt_is_media_type(gt)) + xe_mmio_write32(gt, GEN11_GFX_MSTR_IRQ.reg, master_ctl); + gen11_gt_irq_handler(xe, gt, master_ctl, intr_dw, identity); + } + + gu_misc_iir = gen11_gu_misc_irq_ack(gt, master_ctl); + + dg1_intr_enable(xe, false); + + return IRQ_HANDLED; +} + +static void gen11_gt_irq_reset(struct xe_gt *gt) +{ + u32 ccs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COMPUTE); + u32 bcs_mask = xe_hw_engine_mask_per_class(gt, XE_ENGINE_CLASS_COPY); + + /* Disable RCS, BCS, VCS and VECS class engines. */ + xe_mmio_write32(gt, GEN11_RENDER_COPY_INTR_ENABLE.reg, 0); + xe_mmio_write32(gt, GEN11_VCS_VECS_INTR_ENABLE.reg, 0); + if (ccs_mask) + xe_mmio_write32(gt, GEN12_CCS_RSVD_INTR_ENABLE.reg, 0); + + /* Restore masks irqs on RCS, BCS, VCS and VECS engines. */ + xe_mmio_write32(gt, GEN11_RCS0_RSVD_INTR_MASK.reg, ~0); + xe_mmio_write32(gt, GEN11_BCS_RSVD_INTR_MASK.reg, ~0); + if (bcs_mask & (BIT(1)|BIT(2))) + xe_mmio_write32(gt, XEHPC_BCS1_BCS2_INTR_MASK.reg, ~0); + if (bcs_mask & (BIT(3)|BIT(4))) + xe_mmio_write32(gt, XEHPC_BCS3_BCS4_INTR_MASK.reg, ~0); + if (bcs_mask & (BIT(5)|BIT(6))) + xe_mmio_write32(gt, XEHPC_BCS5_BCS6_INTR_MASK.reg, ~0); + if (bcs_mask & (BIT(7)|BIT(8))) + xe_mmio_write32(gt, XEHPC_BCS7_BCS8_INTR_MASK.reg, ~0); + xe_mmio_write32(gt, GEN11_VCS0_VCS1_INTR_MASK.reg, ~0); + xe_mmio_write32(gt, GEN11_VCS2_VCS3_INTR_MASK.reg, ~0); +// if (HAS_ENGINE(gt, VCS4) || HAS_ENGINE(gt, VCS5)) +// xe_mmio_write32(xe, GEN12_VCS4_VCS5_INTR_MASK.reg, ~0); +// if (HAS_ENGINE(gt, VCS6) || HAS_ENGINE(gt, VCS7)) +// xe_mmio_write32(xe, GEN12_VCS6_VCS7_INTR_MASK.reg, ~0); + xe_mmio_write32(gt, GEN11_VECS0_VECS1_INTR_MASK.reg, ~0); +// if (HAS_ENGINE(gt, VECS2) || HAS_ENGINE(gt, VECS3)) +// xe_mmio_write32(xe, GEN12_VECS2_VECS3_INTR_MASK.reg, ~0); + if (ccs_mask & (BIT(0)|BIT(1))) + xe_mmio_write32(gt, GEN12_CCS0_CCS1_INTR_MASK.reg, ~0); + if (ccs_mask & (BIT(2)|BIT(3))) + xe_mmio_write32(gt, GEN12_CCS2_CCS3_INTR_MASK.reg, ~0); + + xe_mmio_write32(gt, GEN11_GPM_WGBOXPERF_INTR_ENABLE.reg, 0); + xe_mmio_write32(gt, GEN11_GPM_WGBOXPERF_INTR_MASK.reg, ~0); + xe_mmio_write32(gt, GEN11_GUC_SG_INTR_ENABLE.reg, 0); + xe_mmio_write32(gt, GEN11_GUC_SG_INTR_MASK.reg, ~0); +} + +static void gen11_irq_reset(struct xe_gt *gt) +{ + gen11_intr_disable(gt); + + gen11_gt_irq_reset(gt); + + GEN3_IRQ_RESET(gt, GEN11_GU_MISC_); + GEN3_IRQ_RESET(gt, GEN8_PCU_); +} + +static void dg1_irq_reset(struct xe_gt *gt) +{ + if (gt->info.id == 0) + dg1_intr_disable(gt_to_xe(gt)); + + gen11_gt_irq_reset(gt); + + GEN3_IRQ_RESET(gt, GEN11_GU_MISC_); + GEN3_IRQ_RESET(gt, GEN8_PCU_); +} + +void xe_irq_reset(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + + for_each_gt(gt, xe, id) { + if (GRAPHICS_VERx100(xe) >= 1210) { + dg1_irq_reset(gt); + } else if (GRAPHICS_VER(xe) >= 11) { + gen11_irq_reset(gt); + } else { + drm_err(&xe->drm, "No interrupt reset hook"); + } + } +} + +void xe_gt_irq_postinstall(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + + if (GRAPHICS_VERx100(xe) >= 1210) + dg1_irq_postinstall(xe, gt); + else if (GRAPHICS_VER(xe) >= 11) + gen11_irq_postinstall(xe, gt); + else + drm_err(&xe->drm, "No interrupt postinstall hook"); +} + +static void xe_irq_postinstall(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + + for_each_gt(gt, xe, id) + xe_gt_irq_postinstall(gt); +} + +static irq_handler_t xe_irq_handler(struct xe_device *xe) +{ + if (GRAPHICS_VERx100(xe) >= 1210) { + return dg1_irq_handler; + } else if (GRAPHICS_VER(xe) >= 11) { + return gen11_irq_handler; + } else { + return NULL; + } +} + +static void irq_uninstall(struct drm_device *drm, void *arg) +{ + struct xe_device *xe = arg; + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int irq = pdev->irq; + + if (!xe->irq.enabled) + return; + + xe->irq.enabled = false; + xe_irq_reset(xe); + free_irq(irq, xe); + if (pdev->msi_enabled) + pci_disable_msi(pdev); +} + +int xe_irq_install(struct xe_device *xe) +{ + int irq = to_pci_dev(xe->drm.dev)->irq; + static irq_handler_t irq_handler; + int err; + + irq_handler = xe_irq_handler(xe); + if (!irq_handler) { + drm_err(&xe->drm, "No supported interrupt handler"); + return -EINVAL; + } + + xe->irq.enabled = true; + + xe_irq_reset(xe); + + err = request_irq(irq, irq_handler, + IRQF_SHARED, DRIVER_NAME, xe); + if (err < 0) { + xe->irq.enabled = false; + return err; + } + + err = drmm_add_action_or_reset(&xe->drm, irq_uninstall, xe); + if (err) + return err; + + return err; +} + +void xe_irq_shutdown(struct xe_device *xe) +{ + irq_uninstall(&xe->drm, xe); +} + +void xe_irq_suspend(struct xe_device *xe) +{ + spin_lock_irq(&xe->irq.lock); + xe->irq.enabled = false; + xe_irq_reset(xe); + spin_unlock_irq(&xe->irq.lock); +} + +void xe_irq_resume(struct xe_device *xe) +{ + spin_lock_irq(&xe->irq.lock); + xe->irq.enabled = true; + xe_irq_reset(xe); + xe_irq_postinstall(xe); + spin_unlock_irq(&xe->irq.lock); +} diff --git a/drivers/gpu/drm/xe/xe_irq.h b/drivers/gpu/drm/xe/xe_irq.h new file mode 100644 index 000000000000..34ecf22b32d3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_irq.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_IRQ_H_ +#define _XE_IRQ_H_ + +struct xe_device; +struct xe_gt; + +int xe_irq_install(struct xe_device *xe); +void xe_gt_irq_postinstall(struct xe_gt *gt); +void xe_irq_shutdown(struct xe_device *xe); +void xe_irq_suspend(struct xe_device *xe); +void xe_irq_resume(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c new file mode 100644 index 000000000000..056c2c5a0b81 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -0,0 +1,841 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_lrc.h" + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_engine_types.h" +#include "xe_gt.h" +#include "xe_map.h" +#include "xe_hw_fence.h" +#include "xe_vm.h" + +#include "i915_reg.h" +#include "gt/intel_gpu_commands.h" +#include "gt/intel_gt_regs.h" +#include "gt/intel_lrc_reg.h" +#include "gt/intel_engine_regs.h" + +#define GEN8_CTX_VALID (1 << 0) +#define GEN8_CTX_L3LLC_COHERENT (1 << 5) +#define GEN8_CTX_PRIVILEGE (1 << 8) +#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3 +#define INTEL_LEGACY_64B_CONTEXT 3 + +#define GEN11_ENGINE_CLASS_SHIFT 61 +#define GEN11_ENGINE_INSTANCE_SHIFT 48 + +static struct xe_device * +lrc_to_xe(struct xe_lrc *lrc) +{ + return gt_to_xe(lrc->fence_ctx.gt); +} + +size_t xe_lrc_size(struct xe_device *xe, enum xe_engine_class class) +{ + switch (class) { + case XE_ENGINE_CLASS_RENDER: + case XE_ENGINE_CLASS_COMPUTE: + /* 14 pages since graphics_ver == 11 */ + return 14 * SZ_4K; + default: + WARN(1, "Unknown engine class: %d", class); + fallthrough; + case XE_ENGINE_CLASS_COPY: + case XE_ENGINE_CLASS_VIDEO_DECODE: + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + return 2 * SZ_4K; + } +} + +/* + * The per-platform tables are u8-encoded in @data. Decode @data and set the + * addresses' offset and commands in @regs. The following encoding is used + * for each byte. There are 2 steps: decoding commands and decoding addresses. + * + * Commands: + * [7]: create NOPs - number of NOPs are set in lower bits + * [6]: When creating MI_LOAD_REGISTER_IMM command, allow to set + * MI_LRI_FORCE_POSTED + * [5:0]: Number of NOPs or registers to set values to in case of + * MI_LOAD_REGISTER_IMM + * + * Addresses: these are decoded after a MI_LOAD_REGISTER_IMM command by "count" + * number of registers. They are set by using the REG/REG16 macros: the former + * is used for offsets smaller than 0x200 while the latter is for values bigger + * than that. Those macros already set all the bits documented below correctly: + * + * [7]: When a register offset needs more than 6 bits, use additional bytes, to + * follow, for the lower bits + * [6:0]: Register offset, without considering the engine base. + * + * This function only tweaks the commands and register offsets. Values are not + * filled out. + */ +static void set_offsets(u32 *regs, + const u8 *data, + const struct xe_hw_engine *hwe) +#define NOP(x) (BIT(7) | (x)) +#define LRI(count, flags) ((flags) << 6 | (count) | \ + BUILD_BUG_ON_ZERO(count >= BIT(6))) +#define POSTED BIT(0) +#define REG(x) (((x) >> 2) | BUILD_BUG_ON_ZERO(x >= 0x200)) +#define REG16(x) \ + (((x) >> 9) | BIT(7) | BUILD_BUG_ON_ZERO(x >= 0x10000)), \ + (((x) >> 2) & 0x7f) +#define END 0 +{ + const u32 base = hwe->mmio_base; + + while (*data) { + u8 count, flags; + + if (*data & BIT(7)) { /* skip */ + count = *data++ & ~BIT(7); + regs += count; + continue; + } + + count = *data & 0x3f; + flags = *data >> 6; + data++; + + *regs = MI_LOAD_REGISTER_IMM(count); + if (flags & POSTED) + *regs |= MI_LRI_FORCE_POSTED; + *regs |= MI_LRI_LRM_CS_MMIO; + regs++; + + XE_BUG_ON(!count); + do { + u32 offset = 0; + u8 v; + + do { + v = *data++; + offset <<= 7; + offset |= v & ~BIT(7); + } while (v & BIT(7)); + + regs[0] = base + (offset << 2); + regs += 2; + } while (--count); + } + + *regs = MI_BATCH_BUFFER_END | BIT(0); +} + +static const u8 gen12_xcs_offsets[] = { + NOP(1), + LRI(13, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + + NOP(5), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + END +}; + +static const u8 dg2_xcs_offsets[] = { + NOP(1), + LRI(15, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + REG(0x120), + REG(0x124), + + NOP(1), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + END +}; + +static const u8 gen12_rcs_offsets[] = { + NOP(1), + LRI(13, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + + NOP(5), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + LRI(3, POSTED), + REG(0x1b0), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + NOP(3 + 9 + 1), + + LRI(51, POSTED), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG16(0x588), + REG(0x028), + REG(0x09c), + REG(0x0c0), + REG(0x178), + REG(0x17c), + REG16(0x358), + REG(0x170), + REG(0x150), + REG(0x154), + REG(0x158), + REG16(0x41c), + REG16(0x600), + REG16(0x604), + REG16(0x608), + REG16(0x60c), + REG16(0x610), + REG16(0x614), + REG16(0x618), + REG16(0x61c), + REG16(0x620), + REG16(0x624), + REG16(0x628), + REG16(0x62c), + REG16(0x630), + REG16(0x634), + REG16(0x638), + REG16(0x63c), + REG16(0x640), + REG16(0x644), + REG16(0x648), + REG16(0x64c), + REG16(0x650), + REG16(0x654), + REG16(0x658), + REG16(0x65c), + REG16(0x660), + REG16(0x664), + REG16(0x668), + REG16(0x66c), + REG16(0x670), + REG16(0x674), + REG16(0x678), + REG16(0x67c), + REG(0x068), + REG(0x084), + NOP(1), + + END +}; + +static const u8 xehp_rcs_offsets[] = { + NOP(1), + LRI(13, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + + NOP(5), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + LRI(3, POSTED), + REG(0x1b0), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + + END +}; + +static const u8 dg2_rcs_offsets[] = { + NOP(1), + LRI(15, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + REG(0x120), + REG(0x124), + + NOP(1), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + LRI(3, POSTED), + REG(0x1b0), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + + END +}; + +static const u8 mtl_rcs_offsets[] = { + NOP(1), + LRI(15, POSTED), + REG16(0x244), + REG(0x034), + REG(0x030), + REG(0x038), + REG(0x03c), + REG(0x168), + REG(0x140), + REG(0x110), + REG(0x1c0), + REG(0x1c4), + REG(0x1c8), + REG(0x180), + REG16(0x2b4), + REG(0x120), + REG(0x124), + + NOP(1), + LRI(9, POSTED), + REG16(0x3a8), + REG16(0x28c), + REG16(0x288), + REG16(0x284), + REG16(0x280), + REG16(0x27c), + REG16(0x278), + REG16(0x274), + REG16(0x270), + + NOP(2), + LRI(2, POSTED), + REG16(0x5a8), + REG16(0x5ac), + + NOP(6), + LRI(1, 0), + REG(0x0c8), + + END +}; + +#undef END +#undef REG16 +#undef REG +#undef LRI +#undef NOP + +static const u8 *reg_offsets(struct xe_device *xe, enum xe_engine_class class) +{ + if (class == XE_ENGINE_CLASS_RENDER) { + if (GRAPHICS_VERx100(xe) >= 1270) + return mtl_rcs_offsets; + else if (GRAPHICS_VERx100(xe) >= 1255) + return dg2_rcs_offsets; + else if (GRAPHICS_VERx100(xe) >= 1250) + return xehp_rcs_offsets; + else + return gen12_rcs_offsets; + } else { + if (GRAPHICS_VERx100(xe) >= 1255) + return dg2_xcs_offsets; + else + return gen12_xcs_offsets; + } +} + +static void set_context_control(u32 * regs, struct xe_hw_engine *hwe) +{ + regs[CTX_CONTEXT_CONTROL] = _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH) | + _MASKED_BIT_DISABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT) | + CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT; + + /* TODO: Timestamp */ +} + +static int lrc_ring_mi_mode(struct xe_hw_engine *hwe) +{ + struct xe_device *xe = gt_to_xe(hwe->gt); + + if (GRAPHICS_VERx100(xe) >= 1250) + return 0x70; + else + return 0x60; +} + +static void reset_stop_ring(u32 *regs, struct xe_hw_engine *hwe) +{ + int x; + + x = lrc_ring_mi_mode(hwe); + regs[x + 1] &= ~STOP_RING; + regs[x + 1] |= STOP_RING << 16; +} + +static inline u32 __xe_lrc_ring_offset(struct xe_lrc *lrc) +{ + return 0; +} + +u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc) +{ + return lrc->ring.size; +} + +/* Make the magic macros work */ +#define __xe_lrc_pphwsp_offset xe_lrc_pphwsp_offset + +#define LRC_SEQNO_PPHWSP_OFFSET 512 +#define LRC_START_SEQNO_PPHWSP_OFFSET LRC_SEQNO_PPHWSP_OFFSET + 8 +#define LRC_PARALLEL_PPHWSP_OFFSET 2048 +#define LRC_PPHWSP_SIZE SZ_4K + +static size_t lrc_reg_size(struct xe_device *xe) +{ + if (GRAPHICS_VERx100(xe) >= 1250) + return 96 * sizeof(u32); + else + return 80 * sizeof(u32); +} + +size_t xe_lrc_skip_size(struct xe_device *xe) +{ + return LRC_PPHWSP_SIZE + lrc_reg_size(xe); +} + +static inline u32 __xe_lrc_seqno_offset(struct xe_lrc *lrc) +{ + /* The seqno is stored in the driver-defined portion of PPHWSP */ + return xe_lrc_pphwsp_offset(lrc) + LRC_SEQNO_PPHWSP_OFFSET; +} + +static inline u32 __xe_lrc_start_seqno_offset(struct xe_lrc *lrc) +{ + /* The start seqno is stored in the driver-defined portion of PPHWSP */ + return xe_lrc_pphwsp_offset(lrc) + LRC_START_SEQNO_PPHWSP_OFFSET; +} + +static inline u32 __xe_lrc_parallel_offset(struct xe_lrc *lrc) +{ + /* The parallel is stored in the driver-defined portion of PPHWSP */ + return xe_lrc_pphwsp_offset(lrc) + LRC_PARALLEL_PPHWSP_OFFSET; +} + +static inline u32 __xe_lrc_regs_offset(struct xe_lrc *lrc) +{ + return xe_lrc_pphwsp_offset(lrc) + LRC_PPHWSP_SIZE; +} + +#define DECL_MAP_ADDR_HELPERS(elem) \ +static inline struct iosys_map __xe_lrc_##elem##_map(struct xe_lrc *lrc) \ +{ \ + struct iosys_map map = lrc->bo->vmap; \ +\ + XE_BUG_ON(iosys_map_is_null(&map)); \ + iosys_map_incr(&map, __xe_lrc_##elem##_offset(lrc)); \ + return map; \ +} \ +static inline u32 __xe_lrc_##elem##_ggtt_addr(struct xe_lrc *lrc) \ +{ \ + return xe_bo_ggtt_addr(lrc->bo) + __xe_lrc_##elem##_offset(lrc); \ +} \ + +DECL_MAP_ADDR_HELPERS(ring) +DECL_MAP_ADDR_HELPERS(pphwsp) +DECL_MAP_ADDR_HELPERS(seqno) +DECL_MAP_ADDR_HELPERS(regs) +DECL_MAP_ADDR_HELPERS(start_seqno) +DECL_MAP_ADDR_HELPERS(parallel) + +#undef DECL_MAP_ADDR_HELPERS + +u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_pphwsp_ggtt_addr(lrc); +} + +u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_regs_map(lrc); + iosys_map_incr(&map, reg_nr * sizeof(u32)); + return xe_map_read32(xe, &map); +} + +void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val) +{ + struct xe_device *xe = lrc_to_xe(lrc); + struct iosys_map map; + + map = __xe_lrc_regs_map(lrc); + iosys_map_incr(&map, reg_nr * sizeof(u32)); + xe_map_write32(xe, &map, val); +} + +static void *empty_lrc_data(struct xe_hw_engine *hwe) +{ + struct xe_device *xe = gt_to_xe(hwe->gt); + void *data; + u32 *regs; + + data = kzalloc(xe_lrc_size(xe, hwe->class), GFP_KERNEL); + if (!data) + return NULL; + + /* 1st page: Per-Process of HW status Page */ + regs = data + LRC_PPHWSP_SIZE; + set_offsets(regs, reg_offsets(xe, hwe->class), hwe); + set_context_control(regs, hwe); + reset_stop_ring(regs, hwe); + + return data; +} + +static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm) +{ + u64 desc = xe_vm_pdp4_descriptor(vm, lrc->full_gt); + + xe_lrc_write_ctx_reg(lrc, CTX_PDP0_UDW, upper_32_bits(desc)); + xe_lrc_write_ctx_reg(lrc, CTX_PDP0_LDW, lower_32_bits(desc)); +} + +#define PVC_CTX_ASID (0x2e + 1) +#define PVC_CTX_ACC_CTR_THOLD (0x2a + 1) +#define ACC_GRANULARITY_S 20 +#define ACC_NOTIFY_S 16 + +int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + struct xe_engine *e, struct xe_vm *vm, u32 ring_size) +{ + struct xe_gt *gt = hwe->gt; + struct xe_device *xe = gt_to_xe(gt); + struct iosys_map map; + void *init_data = NULL; + u32 arb_enable; + int err; + + lrc->flags = 0; + + lrc->bo = xe_bo_create_locked(xe, hwe->gt, vm, + ring_size + xe_lrc_size(xe, hwe->class), + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(hwe->gt) | + XE_BO_CREATE_GGTT_BIT); + if (IS_ERR(lrc->bo)) + return PTR_ERR(lrc->bo); + + if (xe_gt_is_media_type(hwe->gt)) + lrc->full_gt = xe_find_full_gt(hwe->gt); + else + lrc->full_gt = hwe->gt; + + /* + * FIXME: Perma-pinning LRC as we don't yet support moving GGTT address + * via VM bind calls. + */ + err = xe_bo_pin(lrc->bo); + if (err) + goto err_unlock_put_bo; + lrc->flags |= XE_LRC_PINNED; + + err = xe_bo_vmap(lrc->bo); + if (err) + goto err_unpin_bo; + + xe_bo_unlock_vm_held(lrc->bo); + + lrc->ring.size = ring_size; + lrc->ring.tail = 0; + + xe_hw_fence_ctx_init(&lrc->fence_ctx, hwe->gt, + hwe->fence_irq, hwe->name); + + if (!gt->default_lrc[hwe->class]) { + init_data = empty_lrc_data(hwe); + if (!init_data) { + xe_lrc_finish(lrc); + return -ENOMEM; + } + } + + /* + * Init Per-Process of HW status Page, LRC / context state to known + * values + */ + map = __xe_lrc_pphwsp_map(lrc); + if (!init_data) { + xe_map_memset(xe, &map, 0, 0, LRC_PPHWSP_SIZE); /* PPHWSP */ + xe_map_memcpy_to(xe, &map, LRC_PPHWSP_SIZE, + gt->default_lrc[hwe->class] + LRC_PPHWSP_SIZE, + xe_lrc_size(xe, hwe->class) - LRC_PPHWSP_SIZE); + } else { + xe_map_memcpy_to(xe, &map, 0, init_data, + xe_lrc_size(xe, hwe->class)); + kfree(init_data); + } + + if (vm) + xe_lrc_set_ppgtt(lrc, vm); + + xe_lrc_write_ctx_reg(lrc, CTX_RING_START, __xe_lrc_ring_ggtt_addr(lrc)); + xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, 0); + xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); + xe_lrc_write_ctx_reg(lrc, CTX_RING_CTL, + RING_CTL_SIZE(lrc->ring.size) | RING_VALID); + if (xe->info.supports_usm && vm) { + xe_lrc_write_ctx_reg(lrc, PVC_CTX_ASID, + (e->usm.acc_granularity << + ACC_GRANULARITY_S) | vm->usm.asid); + xe_lrc_write_ctx_reg(lrc, PVC_CTX_ACC_CTR_THOLD, + (e->usm.acc_notify << ACC_NOTIFY_S) | + e->usm.acc_trigger); + } + + lrc->desc = GEN8_CTX_VALID; + lrc->desc |= INTEL_LEGACY_64B_CONTEXT << GEN8_CTX_ADDRESSING_MODE_SHIFT; + /* TODO: Priority */ + + /* While this appears to have something about privileged batches or + * some such, it really just means PPGTT mode. + */ + if (vm) + lrc->desc |= GEN8_CTX_PRIVILEGE; + + if (GRAPHICS_VERx100(xe) < 1250) { + lrc->desc |= (u64)hwe->instance << GEN11_ENGINE_INSTANCE_SHIFT; + lrc->desc |= (u64)hwe->class << GEN11_ENGINE_CLASS_SHIFT; + } + + arb_enable = MI_ARB_ON_OFF | MI_ARB_ENABLE; + xe_lrc_write_ring(lrc, &arb_enable, sizeof(arb_enable)); + + return 0; + +err_unpin_bo: + if (lrc->flags & XE_LRC_PINNED) + xe_bo_unpin(lrc->bo); +err_unlock_put_bo: + xe_bo_unlock_vm_held(lrc->bo); + xe_bo_put(lrc->bo); + return err; +} + +void xe_lrc_finish(struct xe_lrc *lrc) +{ + struct ww_acquire_ctx ww; + + xe_hw_fence_ctx_finish(&lrc->fence_ctx); + if (lrc->flags & XE_LRC_PINNED) { + if (lrc->bo->vm) + xe_vm_lock(lrc->bo->vm, &ww, 0, false); + else + xe_bo_lock_no_vm(lrc->bo, NULL); + xe_bo_unpin(lrc->bo); + if (lrc->bo->vm) + xe_vm_unlock(lrc->bo->vm, &ww); + else + xe_bo_unlock_no_vm(lrc->bo); + } + xe_bo_put(lrc->bo); +} + +void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head) +{ + xe_lrc_write_ctx_reg(lrc, CTX_RING_HEAD, head); +} + +u32 xe_lrc_ring_head(struct xe_lrc *lrc) +{ + return xe_lrc_read_ctx_reg(lrc, CTX_RING_HEAD) & HEAD_ADDR; +} + +u32 xe_lrc_ring_space(struct xe_lrc *lrc) +{ + const u32 head = xe_lrc_ring_head(lrc); + const u32 tail = lrc->ring.tail; + const u32 size = lrc->ring.size; + + return ((head - tail - 1) & (size - 1)) + 1; +} + +static void __xe_lrc_write_ring(struct xe_lrc *lrc, struct iosys_map ring, + const void *data, size_t size) +{ + struct xe_device *xe = lrc_to_xe(lrc); + + iosys_map_incr(&ring, lrc->ring.tail); + xe_map_memcpy_to(xe, &ring, 0, data, size); + lrc->ring.tail = (lrc->ring.tail + size) & (lrc->ring.size - 1); +} + +void xe_lrc_write_ring(struct xe_lrc *lrc, const void *data, size_t size) +{ + struct iosys_map ring; + u32 rhs; + size_t aligned_size; + + XE_BUG_ON(!IS_ALIGNED(size, 4)); + aligned_size = ALIGN(size, 8); + + ring = __xe_lrc_ring_map(lrc); + + XE_BUG_ON(lrc->ring.tail >= lrc->ring.size); + rhs = lrc->ring.size - lrc->ring.tail; + if (size > rhs) { + __xe_lrc_write_ring(lrc, ring, data, rhs); + __xe_lrc_write_ring(lrc, ring, data + rhs, size - rhs); + } else { + __xe_lrc_write_ring(lrc, ring, data, size); + } + + if (aligned_size > size) { + u32 noop = MI_NOOP; + + __xe_lrc_write_ring(lrc, ring, &noop, sizeof(noop)); + } +} + +u64 xe_lrc_descriptor(struct xe_lrc *lrc) +{ + return lrc->desc | xe_lrc_ggtt_addr(lrc); +} + +u32 xe_lrc_seqno_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_seqno_ggtt_addr(lrc); +} + +struct dma_fence *xe_lrc_create_seqno_fence(struct xe_lrc *lrc) +{ + return &xe_hw_fence_create(&lrc->fence_ctx, + __xe_lrc_seqno_map(lrc))->dma; +} + +s32 xe_lrc_seqno(struct xe_lrc *lrc) +{ + struct iosys_map map = __xe_lrc_seqno_map(lrc); + + return xe_map_read32(lrc_to_xe(lrc), &map); +} + +s32 xe_lrc_start_seqno(struct xe_lrc *lrc) +{ + struct iosys_map map = __xe_lrc_start_seqno_map(lrc); + + return xe_map_read32(lrc_to_xe(lrc), &map); +} + +u32 xe_lrc_start_seqno_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_start_seqno_ggtt_addr(lrc); +} + +u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc) +{ + return __xe_lrc_parallel_ggtt_addr(lrc); +} + +struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc) +{ + return __xe_lrc_parallel_map(lrc); +} diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h new file mode 100644 index 000000000000..e37f89e75ef8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -0,0 +1,50 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ +#ifndef _XE_LRC_H_ +#define _XE_LRC_H_ + +#include "xe_lrc_types.h" + +struct xe_device; +struct xe_engine; +enum xe_engine_class; +struct xe_hw_engine; +struct xe_vm; + +#define LRC_PPHWSP_SCRATCH_ADDR (0x34 * 4) + +int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, + struct xe_engine *e, struct xe_vm *vm, u32 ring_size); +void xe_lrc_finish(struct xe_lrc *lrc); + +size_t xe_lrc_size(struct xe_device *xe, enum xe_engine_class class); +u32 xe_lrc_pphwsp_offset(struct xe_lrc *lrc); + +void xe_lrc_set_ring_head(struct xe_lrc *lrc, u32 head); +u32 xe_lrc_ring_head(struct xe_lrc *lrc); +u32 xe_lrc_ring_space(struct xe_lrc *lrc); +void xe_lrc_write_ring(struct xe_lrc *lrc, const void *data, size_t size); + +u32 xe_lrc_ggtt_addr(struct xe_lrc *lrc); +u32 *xe_lrc_regs(struct xe_lrc *lrc); + +u32 xe_lrc_read_ctx_reg(struct xe_lrc *lrc, int reg_nr); +void xe_lrc_write_ctx_reg(struct xe_lrc *lrc, int reg_nr, u32 val); + +u64 xe_lrc_descriptor(struct xe_lrc *lrc); + +u32 xe_lrc_seqno_ggtt_addr(struct xe_lrc *lrc); +struct dma_fence *xe_lrc_create_seqno_fence(struct xe_lrc *lrc); +s32 xe_lrc_seqno(struct xe_lrc *lrc); + +u32 xe_lrc_start_seqno_ggtt_addr(struct xe_lrc *lrc); +s32 xe_lrc_start_seqno(struct xe_lrc *lrc); + +u32 xe_lrc_parallel_ggtt_addr(struct xe_lrc *lrc); +struct iosys_map xe_lrc_parallel_map(struct xe_lrc *lrc); + +size_t xe_lrc_skip_size(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h new file mode 100644 index 000000000000..2827efa2091d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_lrc_types.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_LRC_TYPES_H_ +#define _XE_LRC_TYPES_H_ + +#include "xe_hw_fence_types.h" + +struct xe_bo; + +/** + * struct xe_lrc - Logical ring context (LRC) and submission ring object + */ +struct xe_lrc { + /** + * @bo: buffer object (memory) for logical ring context, per process HW + * status page, and submission ring. + */ + struct xe_bo *bo; + + /** @full_gt: full GT which this LRC belongs to */ + struct xe_gt *full_gt; + + /** @flags: LRC flags */ + u32 flags; +#define XE_LRC_PINNED BIT(1) + + /** @ring: submission ring state */ + struct { + /** @size: size of submission ring */ + u32 size; + /** @tail: tail of submission ring */ + u32 tail; + /** @old_tail: shadow of tail */ + u32 old_tail; + } ring; + + /** @desc: LRC descriptor */ + u64 desc; + + /** @fence_ctx: context for hw fence */ + struct xe_hw_fence_ctx fence_ctx; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_macros.h b/drivers/gpu/drm/xe/xe_macros.h new file mode 100644 index 000000000000..0d24c124d202 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_macros.h @@ -0,0 +1,20 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_MACROS_H_ +#define _XE_MACROS_H_ + +#include + +#define XE_EXTRA_DEBUG 1 +#define XE_WARN_ON WARN_ON +#define XE_BUG_ON BUG_ON + +#define XE_IOCTL_ERR(xe, cond) \ + ((cond) && (drm_info(&(xe)->drm, \ + "Ioctl argument check failed at %s:%d: %s", \ + __FILE__, __LINE__, #cond), 1)) + +#endif diff --git a/drivers/gpu/drm/xe/xe_map.h b/drivers/gpu/drm/xe/xe_map.h new file mode 100644 index 000000000000..0bac1f73a80d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_map.h @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __XE_MAP_H__ +#define __XE_MAP_H__ + +#include + +#include + +/** + * DOC: Map layer + * + * All access to any memory shared with a device (both sysmem and vram) in the + * XE driver should go through this layer (xe_map). This layer is built on top + * of :ref:`driver-api/device-io:Generalizing Access to System and I/O Memory` + * and with extra hooks into the XE driver that allows adding asserts to memory + * accesses (e.g. for blocking runtime_pm D3Cold on Discrete Graphics). + */ + +static inline void xe_map_memcpy_to(struct xe_device *xe, struct iosys_map *dst, + size_t dst_offset, const void *src, + size_t len) +{ + xe_device_assert_mem_access(xe); + iosys_map_memcpy_to(dst, dst_offset, src, len); +} + +static inline void xe_map_memcpy_from(struct xe_device *xe, void *dst, + const struct iosys_map *src, + size_t src_offset, size_t len) +{ + xe_device_assert_mem_access(xe); + iosys_map_memcpy_from(dst, src, src_offset, len); +} + +static inline void xe_map_memset(struct xe_device *xe, + struct iosys_map *dst, size_t offset, + int value, size_t len) +{ + xe_device_assert_mem_access(xe); + iosys_map_memset(dst, offset, value, len); +} + +/* FIXME: We likely should kill these two functions sooner or later */ +static inline u32 xe_map_read32(struct xe_device *xe, struct iosys_map *map) +{ + xe_device_assert_mem_access(xe); + + if (map->is_iomem) + return readl(map->vaddr_iomem); + else + return READ_ONCE(*(u32 *)map->vaddr); +} + +static inline void xe_map_write32(struct xe_device *xe, struct iosys_map *map, + u32 val) +{ + xe_device_assert_mem_access(xe); + + if (map->is_iomem) + writel(val, map->vaddr_iomem); + else + *(u32 *)map->vaddr = val; +} + +#define xe_map_rd(xe__, map__, offset__, type__) ({ \ + struct xe_device *__xe = xe__; \ + xe_device_assert_mem_access(__xe); \ + iosys_map_rd(map__, offset__, type__); \ +}) + +#define xe_map_wr(xe__, map__, offset__, type__, val__) ({ \ + struct xe_device *__xe = xe__; \ + xe_device_assert_mem_access(__xe); \ + iosys_map_wr(map__, offset__, type__, val__); \ +}) + +#define xe_map_rd_field(xe__, map__, struct_offset__, struct_type__, field__) ({ \ + struct xe_device *__xe = xe__; \ + xe_device_assert_mem_access(__xe); \ + iosys_map_rd_field(map__, struct_offset__, struct_type__, field__); \ +}) + +#define xe_map_wr_field(xe__, map__, struct_offset__, struct_type__, field__, val__) ({ \ + struct xe_device *__xe = xe__; \ + xe_device_assert_mem_access(__xe); \ + iosys_map_wr_field(map__, struct_offset__, struct_type__, field__, val__); \ +}) + +#endif diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c new file mode 100644 index 000000000000..7fc40e8009c3 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -0,0 +1,1168 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2020 Intel Corporation + */ +#include "xe_migrate.h" + +#include "xe_bb.h" +#include "xe_bo.h" +#include "xe_engine.h" +#include "xe_ggtt.h" +#include "xe_gt.h" +#include "xe_hw_engine.h" +#include "xe_lrc.h" +#include "xe_map.h" +#include "xe_mocs.h" +#include "xe_pt.h" +#include "xe_res_cursor.h" +#include "xe_sched_job.h" +#include "xe_sync.h" +#include "xe_trace.h" +#include "xe_vm.h" + +#include +#include +#include +#include + +#include "gt/intel_gpu_commands.h" + +struct xe_migrate { + struct xe_engine *eng; + struct xe_gt *gt; + struct mutex job_mutex; + struct xe_bo *pt_bo; + struct xe_bo *cleared_bo; + u64 batch_base_ofs; + u64 usm_batch_base_ofs; + u64 cleared_vram_ofs; + struct dma_fence *fence; + struct drm_suballoc_manager vm_update_sa; +}; + +#define MAX_PREEMPTDISABLE_TRANSFER SZ_8M /* Around 1ms. */ +#define NUM_KERNEL_PDE 17 +#define NUM_PT_SLOTS 32 +#define NUM_PT_PER_BLIT (MAX_PREEMPTDISABLE_TRANSFER / SZ_2M) + +struct xe_engine *xe_gt_migrate_engine(struct xe_gt *gt) +{ + return gt->migrate->eng; +} + +static void xe_migrate_fini(struct drm_device *dev, void *arg) +{ + struct xe_migrate *m = arg; + struct ww_acquire_ctx ww; + + xe_vm_lock(m->eng->vm, &ww, 0, false); + xe_bo_unpin(m->pt_bo); + if (m->cleared_bo) + xe_bo_unpin(m->cleared_bo); + xe_vm_unlock(m->eng->vm, &ww); + + dma_fence_put(m->fence); + if (m->cleared_bo) + xe_bo_put(m->cleared_bo); + xe_bo_put(m->pt_bo); + drm_suballoc_manager_fini(&m->vm_update_sa); + mutex_destroy(&m->job_mutex); + xe_vm_close_and_put(m->eng->vm); + xe_engine_put(m->eng); +} + +static u64 xe_migrate_vm_addr(u64 slot, u32 level) +{ + XE_BUG_ON(slot >= NUM_PT_SLOTS); + + /* First slot is reserved for mapping of PT bo and bb, start from 1 */ + return (slot + 1ULL) << xe_pt_shift(level + 1); +} + +static u64 xe_migrate_vram_ofs(u64 addr) +{ + return addr + (256ULL << xe_pt_shift(2)); +} + +/* + * For flat CCS clearing we need a cleared chunk of memory to copy from, + * since the CCS clearing mode of XY_FAST_COLOR_BLT appears to be buggy + * (it clears on only 14 bytes in each chunk of 16). + * If clearing the main surface one can use the part of the main surface + * already cleared, but for clearing as part of copying non-compressed + * data out of system memory, we don't readily have a cleared part of + * VRAM to copy from, so create one to use for that case. + */ +static int xe_migrate_create_cleared_bo(struct xe_migrate *m, struct xe_vm *vm) +{ + struct xe_gt *gt = m->gt; + struct xe_device *xe = vm->xe; + size_t cleared_size; + u64 vram_addr; + bool is_vram; + + if (!xe_device_has_flat_ccs(xe)) + return 0; + + cleared_size = xe_device_ccs_bytes(xe, MAX_PREEMPTDISABLE_TRANSFER); + cleared_size = PAGE_ALIGN(cleared_size); + m->cleared_bo = xe_bo_create_pin_map(xe, gt, vm, cleared_size, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_PINNED_BIT); + if (IS_ERR(m->cleared_bo)) + return PTR_ERR(m->cleared_bo); + + xe_map_memset(xe, &m->cleared_bo->vmap, 0, 0x00, cleared_size); + vram_addr = xe_bo_addr(m->cleared_bo, 0, GEN8_PAGE_SIZE, &is_vram); + XE_BUG_ON(!is_vram); + m->cleared_vram_ofs = xe_migrate_vram_ofs(vram_addr); + + return 0; +} + +static int xe_migrate_prepare_vm(struct xe_gt *gt, struct xe_migrate *m, + struct xe_vm *vm) +{ + u8 id = gt->info.id; + u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; + u32 map_ofs, level, i; + struct xe_device *xe = gt_to_xe(m->gt); + struct xe_bo *bo, *batch = gt->kernel_bb_pool.bo; + u64 entry; + int ret; + + /* Can't bump NUM_PT_SLOTS too high */ + BUILD_BUG_ON(NUM_PT_SLOTS > SZ_2M/GEN8_PAGE_SIZE); + /* Must be a multiple of 64K to support all platforms */ + BUILD_BUG_ON(NUM_PT_SLOTS * GEN8_PAGE_SIZE % SZ_64K); + /* And one slot reserved for the 4KiB page table updates */ + BUILD_BUG_ON(!(NUM_KERNEL_PDE & 1)); + + /* Need to be sure everything fits in the first PT, or create more */ + XE_BUG_ON(m->batch_base_ofs + batch->size >= SZ_2M); + + bo = xe_bo_create_pin_map(vm->xe, m->gt, vm, + num_entries * GEN8_PAGE_SIZE, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(m->gt) | + XE_BO_CREATE_PINNED_BIT); + if (IS_ERR(bo)) + return PTR_ERR(bo); + + ret = xe_migrate_create_cleared_bo(m, vm); + if (ret) { + xe_bo_put(bo); + return ret; + } + + entry = gen8_pde_encode(bo, bo->size - GEN8_PAGE_SIZE, XE_CACHE_WB); + xe_pt_write(xe, &vm->pt_root[id]->bo->vmap, 0, entry); + + map_ofs = (num_entries - num_level) * GEN8_PAGE_SIZE; + + /* Map the entire BO in our level 0 pt */ + for (i = 0, level = 0; i < num_entries; level++) { + entry = gen8_pte_encode(NULL, bo, i * GEN8_PAGE_SIZE, + XE_CACHE_WB, 0, 0); + + xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, entry); + + if (vm->flags & XE_VM_FLAGS_64K) + i += 16; + else + i += 1; + } + + if (!IS_DGFX(xe)) { + XE_BUG_ON(xe->info.supports_usm); + + /* Write out batch too */ + m->batch_base_ofs = NUM_PT_SLOTS * GEN8_PAGE_SIZE; + for (i = 0; i < batch->size; + i += vm->flags & XE_VM_FLAGS_64K ? GEN8_64K_PAGE_SIZE : + GEN8_PAGE_SIZE) { + entry = gen8_pte_encode(NULL, batch, i, + XE_CACHE_WB, 0, 0); + + xe_map_wr(xe, &bo->vmap, map_ofs + level * 8, u64, + entry); + level++; + } + } else { + bool is_lmem; + u64 batch_addr = xe_bo_addr(batch, 0, GEN8_PAGE_SIZE, &is_lmem); + + m->batch_base_ofs = xe_migrate_vram_ofs(batch_addr); + + if (xe->info.supports_usm) { + batch = gt->usm.bb_pool.bo; + batch_addr = xe_bo_addr(batch, 0, GEN8_PAGE_SIZE, + &is_lmem); + m->usm_batch_base_ofs = xe_migrate_vram_ofs(batch_addr); + } + } + + for (level = 1; level < num_level; level++) { + u32 flags = 0; + + if (vm->flags & XE_VM_FLAGS_64K && level == 1) + flags = GEN12_PDE_64K; + + entry = gen8_pde_encode(bo, map_ofs + (level - 1) * + GEN8_PAGE_SIZE, XE_CACHE_WB); + xe_map_wr(xe, &bo->vmap, map_ofs + GEN8_PAGE_SIZE * level, u64, + entry | flags); + } + + /* Write PDE's that point to our BO. */ + for (i = 0; i < num_entries - num_level; i++) { + entry = gen8_pde_encode(bo, i * GEN8_PAGE_SIZE, + XE_CACHE_WB); + + xe_map_wr(xe, &bo->vmap, map_ofs + GEN8_PAGE_SIZE + + (i + 1) * 8, u64, entry); + } + + /* Identity map the entire vram at 256GiB offset */ + if (IS_DGFX(xe)) { + u64 pos, ofs, flags; + + level = 2; + ofs = map_ofs + GEN8_PAGE_SIZE * level + 256 * 8; + flags = GEN8_PAGE_RW | GEN8_PAGE_PRESENT | PPAT_CACHED | + GEN12_PPGTT_PTE_LM | GEN8_PDPE_PS_1G; + + /* + * Use 1GB pages, it shouldn't matter the physical amount of + * vram is less, when we don't access it. + */ + for (pos = 0; pos < xe->mem.vram.size; pos += SZ_1G, ofs += 8) + xe_map_wr(xe, &bo->vmap, ofs, u64, pos | flags); + } + + /* + * Example layout created above, with root level = 3: + * [PT0...PT7]: kernel PT's for copy/clear; 64 or 4KiB PTE's + * [PT8]: Kernel PT for VM_BIND, 4 KiB PTE's + * [PT9...PT28]: Userspace PT's for VM_BIND, 4 KiB PTE's + * [PT29 = PDE 0] [PT30 = PDE 1] [PT31 = PDE 2] + * + * This makes the lowest part of the VM point to the pagetables. + * Hence the lowest 2M in the vm should point to itself, with a few writes + * and flushes, other parts of the VM can be used either for copying and + * clearing. + * + * For performance, the kernel reserves PDE's, so about 20 are left + * for async VM updates. + * + * To make it easier to work, each scratch PT is put in slot (1 + PT #) + * everywhere, this allows lockless updates to scratch pages by using + * the different addresses in VM. + */ +#define NUM_VMUSA_UNIT_PER_PAGE 32 +#define VM_SA_UPDATE_UNIT_SIZE (GEN8_PAGE_SIZE / NUM_VMUSA_UNIT_PER_PAGE) +#define NUM_VMUSA_WRITES_PER_UNIT (VM_SA_UPDATE_UNIT_SIZE / sizeof(u64)) + drm_suballoc_manager_init(&m->vm_update_sa, + (map_ofs / GEN8_PAGE_SIZE - NUM_KERNEL_PDE) * + NUM_VMUSA_UNIT_PER_PAGE, 0); + + m->pt_bo = bo; + return 0; +} + +struct xe_migrate *xe_migrate_init(struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_migrate *m; + struct xe_vm *vm; + struct ww_acquire_ctx ww; + int err; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + + m = drmm_kzalloc(&xe->drm, sizeof(*m), GFP_KERNEL); + if (!m) + return ERR_PTR(-ENOMEM); + + m->gt = gt; + + /* Special layout, prepared below.. */ + vm = xe_vm_create(xe, XE_VM_FLAG_MIGRATION | + XE_VM_FLAG_SET_GT_ID(gt)); + if (IS_ERR(vm)) + return ERR_CAST(vm); + + xe_vm_lock(vm, &ww, 0, false); + err = xe_migrate_prepare_vm(gt, m, vm); + xe_vm_unlock(vm, &ww); + if (err) { + xe_vm_close_and_put(vm); + return ERR_PTR(err); + } + + if (xe->info.supports_usm) { + struct xe_hw_engine *hwe = xe_gt_hw_engine(gt, + XE_ENGINE_CLASS_COPY, + gt->usm.reserved_bcs_instance, + false); + if (!hwe) + return ERR_PTR(-EINVAL); + + m->eng = xe_engine_create(xe, vm, + BIT(hwe->logical_instance), 1, + hwe, ENGINE_FLAG_KERNEL); + } else { + m->eng = xe_engine_create_class(xe, gt, vm, + XE_ENGINE_CLASS_COPY, + ENGINE_FLAG_KERNEL); + } + if (IS_ERR(m->eng)) { + xe_vm_close_and_put(vm); + return ERR_CAST(m->eng); + } + + mutex_init(&m->job_mutex); + + err = drmm_add_action_or_reset(&xe->drm, xe_migrate_fini, m); + if (err) + return ERR_PTR(err); + + return m; +} + +static void emit_arb_clear(struct xe_bb *bb) +{ + /* 1 dword */ + bb->cs[bb->len++] = MI_ARB_ON_OFF | MI_ARB_DISABLE; +} + +static u64 xe_migrate_res_sizes(struct xe_res_cursor *cur) +{ + /* + * For VRAM we use identity mapped pages so we are limited to current + * cursor size. For system we program the pages ourselves so we have no + * such limitation. + */ + return min_t(u64, MAX_PREEMPTDISABLE_TRANSFER, + mem_type_is_vram(cur->mem_type) ? cur->size : + cur->remaining); +} + +static u32 pte_update_size(struct xe_migrate *m, + bool is_vram, + struct xe_res_cursor *cur, + u64 *L0, u64 *L0_ofs, u32 *L0_pt, + u32 cmd_size, u32 pt_ofs, u32 avail_pts) +{ + u32 cmds = 0; + + *L0_pt = pt_ofs; + if (!is_vram) { + /* Clip L0 to available size */ + u64 size = min(*L0, (u64)avail_pts * SZ_2M); + u64 num_4k_pages = DIV_ROUND_UP(size, GEN8_PAGE_SIZE); + + *L0 = size; + *L0_ofs = xe_migrate_vm_addr(pt_ofs, 0); + + /* MI_STORE_DATA_IMM */ + cmds += 3 * DIV_ROUND_UP(num_4k_pages, 0x1ff); + + /* PDE qwords */ + cmds += num_4k_pages * 2; + + /* Each chunk has a single blit command */ + cmds += cmd_size; + } else { + /* Offset into identity map. */ + *L0_ofs = xe_migrate_vram_ofs(cur->start); + cmds += cmd_size; + } + + return cmds; +} + +static void emit_pte(struct xe_migrate *m, + struct xe_bb *bb, u32 at_pt, + bool is_vram, + struct xe_res_cursor *cur, + u32 size, struct xe_bo *bo) +{ + u32 ptes; + u64 ofs = at_pt * GEN8_PAGE_SIZE; + u64 cur_ofs; + + /* + * FIXME: Emitting VRAM PTEs to L0 PTs is forbidden. Currently + * we're only emitting VRAM PTEs during sanity tests, so when + * that's moved to a Kunit test, we should condition VRAM PTEs + * on running tests. + */ + + ptes = DIV_ROUND_UP(size, GEN8_PAGE_SIZE); + + while (ptes) { + u32 chunk = min(0x1ffU, ptes); + + bb->cs[bb->len++] = MI_STORE_DATA_IMM | BIT(21) | + (chunk * 2 + 1); + bb->cs[bb->len++] = ofs; + bb->cs[bb->len++] = 0; + + cur_ofs = ofs; + ofs += chunk * 8; + ptes -= chunk; + + while (chunk--) { + u64 addr; + + XE_BUG_ON(cur->start & (PAGE_SIZE - 1)); + + if (is_vram) { + addr = cur->start; + + /* Is this a 64K PTE entry? */ + if ((m->eng->vm->flags & XE_VM_FLAGS_64K) && + !(cur_ofs & (16 * 8 - 1))) { + XE_WARN_ON(!IS_ALIGNED(addr, SZ_64K)); + addr |= GEN12_PTE_PS64; + } + + addr |= GEN12_PPGTT_PTE_LM; + } else { + addr = xe_res_dma(cur); + } + addr |= PPAT_CACHED | GEN8_PAGE_PRESENT | GEN8_PAGE_RW; + bb->cs[bb->len++] = lower_32_bits(addr); + bb->cs[bb->len++] = upper_32_bits(addr); + + xe_res_next(cur, PAGE_SIZE); + cur_ofs += 8; + } + } +} + +#define EMIT_COPY_CCS_DW 5 +static void emit_copy_ccs(struct xe_gt *gt, struct xe_bb *bb, + u64 dst_ofs, bool dst_is_indirect, + u64 src_ofs, bool src_is_indirect, + u32 size) +{ + u32 *cs = bb->cs + bb->len; + u32 num_ccs_blks; + u32 mocs = xe_mocs_index_to_value(gt->mocs.uc_index); + + num_ccs_blks = DIV_ROUND_UP(xe_device_ccs_bytes(gt_to_xe(gt), size), + NUM_CCS_BYTES_PER_BLOCK); + XE_BUG_ON(num_ccs_blks > NUM_CCS_BLKS_PER_XFER); + *cs++ = XY_CTRL_SURF_COPY_BLT | + (src_is_indirect ? 0x0 : 0x1) << SRC_ACCESS_TYPE_SHIFT | + (dst_is_indirect ? 0x0 : 0x1) << DST_ACCESS_TYPE_SHIFT | + ((num_ccs_blks - 1) & CCS_SIZE_MASK) << CCS_SIZE_SHIFT; + *cs++ = lower_32_bits(src_ofs); + *cs++ = upper_32_bits(src_ofs) | + FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs); + *cs++ = lower_32_bits(dst_ofs); + *cs++ = upper_32_bits(dst_ofs) | + FIELD_PREP(XY_CTRL_SURF_MOCS_MASK, mocs); + + bb->len = cs - bb->cs; +} + +#define EMIT_COPY_DW 10 +static void emit_copy(struct xe_gt *gt, struct xe_bb *bb, + u64 src_ofs, u64 dst_ofs, unsigned int size, + unsigned pitch) +{ + XE_BUG_ON(size / pitch > S16_MAX); + XE_BUG_ON(pitch / 4 > S16_MAX); + XE_BUG_ON(pitch > U16_MAX); + + bb->cs[bb->len++] = GEN9_XY_FAST_COPY_BLT_CMD | (10 - 2); + bb->cs[bb->len++] = BLT_DEPTH_32 | pitch; + bb->cs[bb->len++] = 0; + bb->cs[bb->len++] = (size / pitch) << 16 | pitch / 4; + bb->cs[bb->len++] = lower_32_bits(dst_ofs); + bb->cs[bb->len++] = upper_32_bits(dst_ofs); + bb->cs[bb->len++] = 0; + bb->cs[bb->len++] = pitch; + bb->cs[bb->len++] = lower_32_bits(src_ofs); + bb->cs[bb->len++] = upper_32_bits(src_ofs); +} + +static int job_add_deps(struct xe_sched_job *job, struct dma_resv *resv, + enum dma_resv_usage usage) +{ + return drm_sched_job_add_resv_dependencies(&job->drm, resv, usage); +} + +static u64 xe_migrate_batch_base(struct xe_migrate *m, bool usm) +{ + return usm ? m->usm_batch_base_ofs : m->batch_base_ofs; +} + +static u32 xe_migrate_ccs_copy(struct xe_migrate *m, + struct xe_bb *bb, + u64 src_ofs, bool src_is_vram, + u64 dst_ofs, bool dst_is_vram, u32 dst_size, + u64 ccs_ofs, bool copy_ccs) +{ + struct xe_gt *gt = m->gt; + u32 flush_flags = 0; + + if (xe_device_has_flat_ccs(gt_to_xe(gt)) && !copy_ccs && dst_is_vram) { + /* + * If the bo doesn't have any CCS metadata attached, we still + * need to clear it for security reasons. + */ + emit_copy_ccs(gt, bb, dst_ofs, true, m->cleared_vram_ofs, false, + dst_size); + flush_flags = MI_FLUSH_DW_CCS; + } else if (copy_ccs) { + if (!src_is_vram) + src_ofs = ccs_ofs; + else if (!dst_is_vram) + dst_ofs = ccs_ofs; + + /* + * At the moment, we don't support copying CCS metadata from + * system to system. + */ + XE_BUG_ON(!src_is_vram && !dst_is_vram); + + emit_copy_ccs(gt, bb, dst_ofs, dst_is_vram, src_ofs, + src_is_vram, dst_size); + if (dst_is_vram) + flush_flags = MI_FLUSH_DW_CCS; + } + + return flush_flags; +} + +struct dma_fence *xe_migrate_copy(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *src, + struct ttm_resource *dst) +{ + struct xe_gt *gt = m->gt; + struct xe_device *xe = gt_to_xe(gt); + struct dma_fence *fence = NULL; + u64 size = bo->size; + struct xe_res_cursor src_it, dst_it, ccs_it; + u64 src_L0_ofs, dst_L0_ofs; + u32 src_L0_pt, dst_L0_pt; + u64 src_L0, dst_L0; + int pass = 0; + int err; + bool src_is_vram = mem_type_is_vram(src->mem_type); + bool dst_is_vram = mem_type_is_vram(dst->mem_type); + bool copy_ccs = xe_device_has_flat_ccs(xe) && xe_bo_needs_ccs_pages(bo); + bool copy_system_ccs = copy_ccs && (!src_is_vram || !dst_is_vram); + + if (!src_is_vram) + xe_res_first_sg(xe_bo_get_sg(bo), 0, bo->size, &src_it); + else + xe_res_first(src, 0, bo->size, &src_it); + if (!dst_is_vram) + xe_res_first_sg(xe_bo_get_sg(bo), 0, bo->size, &dst_it); + else + xe_res_first(dst, 0, bo->size, &dst_it); + + if (copy_system_ccs) + xe_res_first_sg(xe_bo_get_sg(bo), xe_bo_ccs_pages_start(bo), + PAGE_ALIGN(xe_device_ccs_bytes(xe, size)), + &ccs_it); + + while (size) { + u32 batch_size = 2; /* arb_clear() + MI_BATCH_BUFFER_END */ + struct xe_sched_job *job; + struct xe_bb *bb; + u32 flush_flags; + u32 update_idx; + u64 ccs_ofs, ccs_size; + u32 ccs_pt; + bool usm = xe->info.supports_usm; + + src_L0 = xe_migrate_res_sizes(&src_it); + dst_L0 = xe_migrate_res_sizes(&dst_it); + + drm_dbg(&xe->drm, "Pass %u, sizes: %llu & %llu\n", + pass++, src_L0, dst_L0); + + src_L0 = min(src_L0, dst_L0); + + batch_size += pte_update_size(m, src_is_vram, &src_it, &src_L0, + &src_L0_ofs, &src_L0_pt, 0, 0, + NUM_PT_PER_BLIT); + + batch_size += pte_update_size(m, dst_is_vram, &dst_it, &src_L0, + &dst_L0_ofs, &dst_L0_pt, 0, + NUM_PT_PER_BLIT, NUM_PT_PER_BLIT); + + if (copy_system_ccs) { + ccs_size = xe_device_ccs_bytes(xe, src_L0); + batch_size += pte_update_size(m, false, &ccs_it, &ccs_size, + &ccs_ofs, &ccs_pt, 0, + 2 * NUM_PT_PER_BLIT, + NUM_PT_PER_BLIT); + } + + /* Add copy commands size here */ + batch_size += EMIT_COPY_DW + + (xe_device_has_flat_ccs(xe) ? EMIT_COPY_CCS_DW : 0); + + bb = xe_bb_new(gt, batch_size, usm); + if (IS_ERR(bb)) { + err = PTR_ERR(bb); + goto err_sync; + } + + /* Preemption is enabled again by the ring ops. */ + if (!src_is_vram || !dst_is_vram) + emit_arb_clear(bb); + + if (!src_is_vram) + emit_pte(m, bb, src_L0_pt, src_is_vram, &src_it, src_L0, + bo); + else + xe_res_next(&src_it, src_L0); + + if (!dst_is_vram) + emit_pte(m, bb, dst_L0_pt, dst_is_vram, &dst_it, src_L0, + bo); + else + xe_res_next(&dst_it, src_L0); + + if (copy_system_ccs) + emit_pte(m, bb, ccs_pt, false, &ccs_it, ccs_size, bo); + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + emit_copy(gt, bb, src_L0_ofs, dst_L0_ofs, src_L0, GEN8_PAGE_SIZE); + flush_flags = xe_migrate_ccs_copy(m, bb, src_L0_ofs, src_is_vram, + dst_L0_ofs, dst_is_vram, + src_L0, ccs_ofs, copy_ccs); + + mutex_lock(&m->job_mutex); + job = xe_bb_create_migration_job(m->eng, bb, + xe_migrate_batch_base(m, usm), + update_idx); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err; + } + + xe_sched_job_add_migrate_flush(job, flush_flags); + if (!fence) { + err = job_add_deps(job, bo->ttm.base.resv, + DMA_RESV_USAGE_BOOKKEEP); + if (err) + goto err_job; + } + + xe_sched_job_arm(job); + dma_fence_put(fence); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + dma_fence_put(m->fence); + m->fence = dma_fence_get(fence); + + mutex_unlock(&m->job_mutex); + + xe_bb_free(bb, fence); + size -= src_L0; + continue; + +err_job: + xe_sched_job_put(job); +err: + mutex_unlock(&m->job_mutex); + xe_bb_free(bb, NULL); + +err_sync: + /* Sync partial copy if any. */ + if (fence) { + dma_fence_wait(fence, false); + dma_fence_put(fence); + } + + return ERR_PTR(err); + } + + return fence; +} + +static int emit_clear(struct xe_gt *gt, struct xe_bb *bb, u64 src_ofs, + u32 size, u32 pitch, u32 value, bool is_vram) +{ + u32 *cs = bb->cs + bb->len; + u32 len = XY_FAST_COLOR_BLT_DW; + u32 mocs = xe_mocs_index_to_value(gt->mocs.uc_index); + + if (GRAPHICS_VERx100(gt->xe) < 1250) + len = 11; + + *cs++ = XY_FAST_COLOR_BLT_CMD | XY_FAST_COLOR_BLT_DEPTH_32 | + (len - 2); + *cs++ = FIELD_PREP(XY_FAST_COLOR_BLT_MOCS_MASK, mocs) | + (pitch - 1); + *cs++ = 0; + *cs++ = (size / pitch) << 16 | pitch / 4; + *cs++ = lower_32_bits(src_ofs); + *cs++ = upper_32_bits(src_ofs); + *cs++ = (is_vram ? 0x0 : 0x1) << XY_FAST_COLOR_BLT_MEM_TYPE_SHIFT; + *cs++ = value; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + + if (len > 11) { + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + *cs++ = 0; + } + + XE_BUG_ON(cs - bb->cs != len + bb->len); + bb->len += len; + + return 0; +} + +struct dma_fence *xe_migrate_clear(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *dst, + u32 value) +{ + bool clear_vram = mem_type_is_vram(dst->mem_type); + struct xe_gt *gt = m->gt; + struct xe_device *xe = gt_to_xe(gt); + struct dma_fence *fence = NULL; + u64 size = bo->size; + struct xe_res_cursor src_it; + struct ttm_resource *src = dst; + int err; + int pass = 0; + + if (!clear_vram) + xe_res_first_sg(xe_bo_get_sg(bo), 0, bo->size, &src_it); + else + xe_res_first(src, 0, bo->size, &src_it); + + while (size) { + u64 clear_L0_ofs; + u32 clear_L0_pt; + u32 flush_flags = 0; + u64 clear_L0; + struct xe_sched_job *job; + struct xe_bb *bb; + u32 batch_size, update_idx; + bool usm = xe->info.supports_usm; + + clear_L0 = xe_migrate_res_sizes(&src_it); + drm_dbg(&xe->drm, "Pass %u, size: %llu\n", pass++, clear_L0); + + /* Calculate final sizes and batch size.. */ + batch_size = 2 + + pte_update_size(m, clear_vram, &src_it, + &clear_L0, &clear_L0_ofs, &clear_L0_pt, + XY_FAST_COLOR_BLT_DW, 0, NUM_PT_PER_BLIT); + if (xe_device_has_flat_ccs(xe) && clear_vram) + batch_size += EMIT_COPY_CCS_DW; + + /* Clear commands */ + + if (WARN_ON_ONCE(!clear_L0)) + break; + + bb = xe_bb_new(gt, batch_size, usm); + if (IS_ERR(bb)) { + err = PTR_ERR(bb); + goto err_sync; + } + + size -= clear_L0; + + /* TODO: Add dependencies here */ + + /* Preemption is enabled again by the ring ops. */ + if (!clear_vram) { + emit_arb_clear(bb); + emit_pte(m, bb, clear_L0_pt, clear_vram, &src_it, clear_L0, + bo); + } else { + xe_res_next(&src_it, clear_L0); + } + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + emit_clear(gt, bb, clear_L0_ofs, clear_L0, GEN8_PAGE_SIZE, + value, clear_vram); + if (xe_device_has_flat_ccs(xe) && clear_vram) { + emit_copy_ccs(gt, bb, clear_L0_ofs, true, + m->cleared_vram_ofs, false, clear_L0); + flush_flags = MI_FLUSH_DW_CCS; + } + + mutex_lock(&m->job_mutex); + job = xe_bb_create_migration_job(m->eng, bb, + xe_migrate_batch_base(m, usm), + update_idx); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err; + } + + xe_sched_job_add_migrate_flush(job, flush_flags); + + xe_sched_job_arm(job); + dma_fence_put(fence); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + dma_fence_put(m->fence); + m->fence = dma_fence_get(fence); + + mutex_unlock(&m->job_mutex); + + xe_bb_free(bb, fence); + continue; + +err: + mutex_unlock(&m->job_mutex); + xe_bb_free(bb, NULL); +err_sync: + /* Sync partial copies if any. */ + if (fence) { + dma_fence_wait(m->fence, false); + dma_fence_put(fence); + } + + return ERR_PTR(err); + } + + return fence; +} + +static void write_pgtable(struct xe_gt *gt, struct xe_bb *bb, u64 ppgtt_ofs, + const struct xe_vm_pgtable_update *update, + struct xe_migrate_pt_update *pt_update) +{ + const struct xe_migrate_pt_update_ops *ops = pt_update->ops; + u32 chunk; + u32 ofs = update->ofs, size = update->qwords; + + /* + * If we have 512 entries (max), we would populate it ourselves, + * and update the PDE above it to the new pointer. + * The only time this can only happen if we have to update the top + * PDE. This requires a BO that is almost vm->size big. + * + * This shouldn't be possible in practice.. might change when 16K + * pages are used. Hence the BUG_ON. + */ + XE_BUG_ON(update->qwords > 0x1ff); + if (!ppgtt_ofs) { + bool is_lmem; + + ppgtt_ofs = xe_migrate_vram_ofs(xe_bo_addr(update->pt_bo, 0, + GEN8_PAGE_SIZE, + &is_lmem)); + XE_BUG_ON(!is_lmem); + } + + do { + u64 addr = ppgtt_ofs + ofs * 8; + chunk = min(update->qwords, 0x1ffU); + + /* Ensure populatefn can do memset64 by aligning bb->cs */ + if (!(bb->len & 1)) + bb->cs[bb->len++] = MI_NOOP; + + bb->cs[bb->len++] = MI_STORE_DATA_IMM | BIT(21) | + (chunk * 2 + 1); + bb->cs[bb->len++] = lower_32_bits(addr); + bb->cs[bb->len++] = upper_32_bits(addr); + ops->populate(pt_update, gt, NULL, bb->cs + bb->len, ofs, chunk, + update); + + bb->len += chunk * 2; + ofs += chunk; + size -= chunk; + } while (size); +} + +struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) +{ + return xe_vm_get(m->eng->vm); +} + +static struct dma_fence * +xe_migrate_update_pgtables_cpu(struct xe_migrate *m, + struct xe_vm *vm, struct xe_bo *bo, + const struct xe_vm_pgtable_update *updates, + u32 num_updates, bool wait_vm, + struct xe_migrate_pt_update *pt_update) +{ + const struct xe_migrate_pt_update_ops *ops = pt_update->ops; + struct dma_fence *fence; + int err; + u32 i; + + /* Wait on BO moves for 10 ms, then fall back to GPU job */ + if (bo) { + long wait; + + wait = dma_resv_wait_timeout(bo->ttm.base.resv, + DMA_RESV_USAGE_KERNEL, + true, HZ / 100); + if (wait <= 0) + return ERR_PTR(-ETIME); + } + if (wait_vm) { + long wait; + + wait = dma_resv_wait_timeout(&vm->resv, + DMA_RESV_USAGE_BOOKKEEP, + true, HZ / 100); + if (wait <= 0) + return ERR_PTR(-ETIME); + } + + if (ops->pre_commit) { + err = ops->pre_commit(pt_update); + if (err) + return ERR_PTR(err); + } + for (i = 0; i < num_updates; i++) { + const struct xe_vm_pgtable_update *update = &updates[i]; + + ops->populate(pt_update, m->gt, &update->pt_bo->vmap, NULL, + update->ofs, update->qwords, update); + } + + trace_xe_vm_cpu_bind(vm); + xe_device_wmb(vm->xe); + + fence = dma_fence_get_stub(); + + return fence; +} + +static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs) +{ + int i; + + for (i = 0; i < num_syncs; i++) { + struct dma_fence *fence = syncs[i].fence; + + if (fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + &fence->flags)) + return false; + } + + return true; +} + +static bool engine_is_idle(struct xe_engine *e) +{ + return !e || e->lrc[0].fence_ctx.next_seqno == 1 || + xe_lrc_seqno(&e->lrc[0]) == e->lrc[0].fence_ctx.next_seqno; +} + +struct dma_fence * +xe_migrate_update_pgtables(struct xe_migrate *m, + struct xe_vm *vm, + struct xe_bo *bo, + struct xe_engine *eng, + const struct xe_vm_pgtable_update *updates, + u32 num_updates, + struct xe_sync_entry *syncs, u32 num_syncs, + struct xe_migrate_pt_update *pt_update) +{ + const struct xe_migrate_pt_update_ops *ops = pt_update->ops; + struct xe_gt *gt = m->gt; + struct xe_device *xe = gt_to_xe(gt); + struct xe_sched_job *job; + struct dma_fence *fence; + struct drm_suballoc *sa_bo = NULL; + struct xe_vma *vma = pt_update->vma; + struct xe_bb *bb; + u32 i, batch_size, ppgtt_ofs, update_idx, page_ofs = 0; + u64 addr; + int err = 0; + bool usm = !eng && xe->info.supports_usm; + bool first_munmap_rebind = vma && vma->first_munmap_rebind; + + /* Use the CPU if no in syncs and engine is idle */ + if (no_in_syncs(syncs, num_syncs) && engine_is_idle(eng)) { + fence = xe_migrate_update_pgtables_cpu(m, vm, bo, updates, + num_updates, + first_munmap_rebind, + pt_update); + if (!IS_ERR(fence) || fence == ERR_PTR(-EAGAIN)) + return fence; + } + + /* fixed + PTE entries */ + if (IS_DGFX(xe)) + batch_size = 2; + else + batch_size = 6 + num_updates * 2; + + for (i = 0; i < num_updates; i++) { + u32 num_cmds = DIV_ROUND_UP(updates[i].qwords, 0x1ff); + + /* align noop + MI_STORE_DATA_IMM cmd prefix */ + batch_size += 4 * num_cmds + updates[i].qwords * 2; + } + + /* + * XXX: Create temp bo to copy from, if batch_size becomes too big? + * + * Worst case: Sum(2 * (each lower level page size) + (top level page size)) + * Should be reasonably bound.. + */ + XE_BUG_ON(batch_size >= SZ_128K); + + bb = xe_bb_new(gt, batch_size, !eng && xe->info.supports_usm); + if (IS_ERR(bb)) + return ERR_CAST(bb); + + /* For sysmem PTE's, need to map them in our hole.. */ + if (!IS_DGFX(xe)) { + ppgtt_ofs = NUM_KERNEL_PDE - 1; + if (eng) { + XE_BUG_ON(num_updates > NUM_VMUSA_WRITES_PER_UNIT); + + sa_bo = drm_suballoc_new(&m->vm_update_sa, 1, + GFP_KERNEL, true, 0); + if (IS_ERR(sa_bo)) { + err = PTR_ERR(sa_bo); + goto err; + } + + ppgtt_ofs = NUM_KERNEL_PDE + + (drm_suballoc_soffset(sa_bo) / + NUM_VMUSA_UNIT_PER_PAGE); + page_ofs = (drm_suballoc_soffset(sa_bo) % + NUM_VMUSA_UNIT_PER_PAGE) * + VM_SA_UPDATE_UNIT_SIZE; + } + + /* Preemption is enabled again by the ring ops. */ + emit_arb_clear(bb); + + /* Map our PT's to gtt */ + bb->cs[bb->len++] = MI_STORE_DATA_IMM | BIT(21) | + (num_updates * 2 + 1); + bb->cs[bb->len++] = ppgtt_ofs * GEN8_PAGE_SIZE + page_ofs; + bb->cs[bb->len++] = 0; /* upper_32_bits */ + + for (i = 0; i < num_updates; i++) { + struct xe_bo *pt_bo = updates[i].pt_bo; + + BUG_ON(pt_bo->size != SZ_4K); + + addr = gen8_pte_encode(NULL, pt_bo, 0, XE_CACHE_WB, + 0, 0); + bb->cs[bb->len++] = lower_32_bits(addr); + bb->cs[bb->len++] = upper_32_bits(addr); + } + + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + + (page_ofs / sizeof(u64)) * GEN8_PAGE_SIZE; + for (i = 0; i < num_updates; i++) + write_pgtable(m->gt, bb, addr + i * GEN8_PAGE_SIZE, + &updates[i], pt_update); + } else { + /* phys pages, no preamble required */ + bb->cs[bb->len++] = MI_BATCH_BUFFER_END; + update_idx = bb->len; + + /* Preemption is enabled again by the ring ops. */ + emit_arb_clear(bb); + for (i = 0; i < num_updates; i++) + write_pgtable(m->gt, bb, 0, &updates[i], pt_update); + } + + if (!eng) + mutex_lock(&m->job_mutex); + + job = xe_bb_create_migration_job(eng ?: m->eng, bb, + xe_migrate_batch_base(m, usm), + update_idx); + if (IS_ERR(job)) { + err = PTR_ERR(job); + goto err_bb; + } + + /* Wait on BO move */ + if (bo) { + err = job_add_deps(job, bo->ttm.base.resv, + DMA_RESV_USAGE_KERNEL); + if (err) + goto err_job; + } + + /* + * Munmap style VM unbind, need to wait for all jobs to be complete / + * trigger preempts before moving forward + */ + if (first_munmap_rebind) { + err = job_add_deps(job, &vm->resv, + DMA_RESV_USAGE_BOOKKEEP); + if (err) + goto err_job; + } + + for (i = 0; !err && i < num_syncs; i++) + err = xe_sync_entry_add_deps(&syncs[i], job); + + if (err) + goto err_job; + + if (ops->pre_commit) { + err = ops->pre_commit(pt_update); + if (err) + goto err_job; + } + xe_sched_job_arm(job); + fence = dma_fence_get(&job->drm.s_fence->finished); + xe_sched_job_push(job); + + if (!eng) + mutex_unlock(&m->job_mutex); + + xe_bb_free(bb, fence); + drm_suballoc_free(sa_bo, fence); + + return fence; + +err_job: + xe_sched_job_put(job); +err_bb: + if (!eng) + mutex_unlock(&m->job_mutex); + xe_bb_free(bb, NULL); +err: + drm_suballoc_free(sa_bo, NULL); + return ERR_PTR(err); +} + +void xe_migrate_wait(struct xe_migrate *m) +{ + if (m->fence) + dma_fence_wait(m->fence, false); +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +#include "tests/xe_migrate.c" +#endif diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h new file mode 100644 index 000000000000..267057a3847f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2020 Intel Corporation + */ + +#ifndef __XE_MIGRATE__ +#define __XE_MIGRATE__ + +#include + +struct dma_fence; +struct iosys_map; +struct ttm_resource; + +struct xe_bo; +struct xe_gt; +struct xe_engine; +struct xe_migrate; +struct xe_migrate_pt_update; +struct xe_sync_entry; +struct xe_pt; +struct xe_vm; +struct xe_vm_pgtable_update; +struct xe_vma; + +struct xe_migrate_pt_update_ops { + /** + * populate() - Populate a command buffer or page-table with ptes. + * @pt_update: Embeddable callback argument. + * @gt: The gt for the current operation. + * @map: struct iosys_map into the memory to be populated. + * @pos: If @map is NULL, map into the memory to be populated. + * @ofs: qword offset into @map, unused if @map is NULL. + * @num_qwords: Number of qwords to write. + * @update: Information about the PTEs to be inserted. + * + * This interface is intended to be used as a callback into the + * page-table system to populate command buffers or shared + * page-tables with PTEs. + */ + void (*populate)(struct xe_migrate_pt_update *pt_update, + struct xe_gt *gt, struct iosys_map *map, + void *pos, u32 ofs, u32 num_qwords, + const struct xe_vm_pgtable_update *update); + + /** + * pre_commit(): Callback to be called just before arming the + * sched_job. + * @pt_update: Pointer to embeddable callback argument. + * + * Return: 0 on success, negative error code on error. + */ + int (*pre_commit)(struct xe_migrate_pt_update *pt_update); +}; + +struct xe_migrate_pt_update { + const struct xe_migrate_pt_update_ops *ops; + struct xe_vma *vma; +}; + +struct xe_migrate *xe_migrate_init(struct xe_gt *gt); + +struct dma_fence *xe_migrate_copy(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *src, + struct ttm_resource *dst); + +struct dma_fence *xe_migrate_clear(struct xe_migrate *m, + struct xe_bo *bo, + struct ttm_resource *dst, + u32 value); + +struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m); + +struct dma_fence * +xe_migrate_update_pgtables(struct xe_migrate *m, + struct xe_vm *vm, + struct xe_bo *bo, + struct xe_engine *eng, + const struct xe_vm_pgtable_update *updates, + u32 num_updates, + struct xe_sync_entry *syncs, u32 num_syncs, + struct xe_migrate_pt_update *pt_update); + +void xe_migrate_wait(struct xe_migrate *m); + +struct xe_engine *xe_gt_migrate_engine(struct xe_gt *gt); +#endif diff --git a/drivers/gpu/drm/xe/xe_migrate_doc.h b/drivers/gpu/drm/xe/xe_migrate_doc.h new file mode 100644 index 000000000000..6a68fdff08dc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_migrate_doc.h @@ -0,0 +1,88 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_MIGRATE_DOC_H_ +#define _XE_MIGRATE_DOC_H_ + +/** + * DOC: Migrate Layer + * + * The XE migrate layer is used generate jobs which can copy memory (eviction), + * clear memory, or program tables (binds). This layer exists in every GT, has + * a migrate engine, and uses a special VM for all generated jobs. + * + * Special VM details + * ================== + * + * The special VM is configured with a page structure where we can dynamically + * map BOs which need to be copied and cleared, dynamically map other VM's page + * table BOs for updates, and identity map the entire device's VRAM with 1 GB + * pages. + * + * Currently the page structure consists of 48 phyiscal pages with 16 being + * reserved for BO mapping during copies and clear, 1 reserved for kernel binds, + * several pages are needed to setup the identity mappings (exact number based + * on how many bits of address space the device has), and the rest are reserved + * user bind operations. + * + * TODO: Diagram of layout + * + * Bind jobs + * ========= + * + * A bind job consist of two batches and runs either on the migrate engine + * (kernel binds) or the bind engine passed in (user binds). In both cases the + * VM of the engine is the migrate VM. + * + * The first batch is used to update the migration VM page structure to point to + * the bind VM page table BOs which need to be updated. A physical page is + * required for this. If it is a user bind, the page is allocated from pool of + * pages reserved user bind operations with drm_suballoc managing this pool. If + * it is a kernel bind, the page reserved for kernel binds is used. + * + * The first batch is only required for devices without VRAM as when the device + * has VRAM the bind VM page table BOs are in VRAM and the identity mapping can + * be used. + * + * The second batch is used to program page table updated in the bind VM. Why + * not just one batch? Well the TLBs need to be invalidated between these two + * batches and that only can be done from the ring. + * + * When the bind job complete, the page allocated is returned the pool of pages + * reserved for user bind operations if a user bind. No need do this for kernel + * binds as the reserved kernel page is serially used by each job. + * + * Copy / clear jobs + * ================= + * + * A copy or clear job consist of two batches and runs on the migrate engine. + * + * Like binds, the first batch is used update the migration VM page structure. + * In copy jobs, we need to map the source and destination of the BO into page + * the structure. In clear jobs, we just need to add 1 mapping of BO into the + * page structure. We use the 16 reserved pages in migration VM for mappings, + * this gives us a maximum copy size of 16 MB and maximum clear size of 32 MB. + * + * The second batch is used do either do the copy or clear. Again similar to + * binds, two batches are required as the TLBs need to be invalidated from the + * ring between the batches. + * + * More than one job will be generated if the BO is larger than maximum copy / + * clear size. + * + * Future work + * =========== + * + * Update copy and clear code to use identity mapped VRAM. + * + * Can we rework the use of the pages async binds to use all the entries in each + * page? + * + * Using large pages for sysmem mappings. + * + * Is it possible to identity map the sysmem? We should explore this. + */ + +#endif diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c new file mode 100644 index 000000000000..42e2405f2f48 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -0,0 +1,466 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_mmio.h" + +#include +#include + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_macros.h" +#include "xe_module.h" + +#include "i915_reg.h" +#include "gt/intel_engine_regs.h" +#include "gt/intel_gt_regs.h" + +#define XEHP_MTCFG_ADDR _MMIO(0x101800) +#define TILE_COUNT REG_GENMASK(15, 8) +#define GEN12_LMEM_BAR 2 + +static int xe_set_dma_info(struct xe_device *xe) +{ + unsigned int mask_size = xe->info.dma_mask_size; + int err; + + /* + * We don't have a max segment size, so set it to the max so sg's + * debugging layer doesn't complain + */ + dma_set_max_seg_size(xe->drm.dev, UINT_MAX); + + err = dma_set_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); + if (err) + goto mask_err; + + err = dma_set_coherent_mask(xe->drm.dev, DMA_BIT_MASK(mask_size)); + if (err) + goto mask_err; + + return 0; + +mask_err: + drm_err(&xe->drm, "Can't set DMA mask/consistent mask (%d)\n", err); + return err; +} + +#ifdef CONFIG_64BIT +static int +_resize_bar(struct xe_device *xe, int resno, resource_size_t size) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + int bar_size = pci_rebar_bytes_to_size(size); + int ret; + + if (pci_resource_len(pdev, resno)) + pci_release_resource(pdev, resno); + + ret = pci_resize_resource(pdev, resno, bar_size); + if (ret) { + drm_info(&xe->drm, "Failed to resize BAR%d to %dM (%pe)\n", + resno, 1 << bar_size, ERR_PTR(ret)); + return -1; + } + + drm_info(&xe->drm, "BAR%d resized to %dM\n", resno, 1 << bar_size); + return 1; +} + +static int xe_resize_lmem_bar(struct xe_device *xe, resource_size_t lmem_size) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct pci_bus *root = pdev->bus; + struct resource *root_res; + resource_size_t rebar_size; + resource_size_t current_size; + u32 pci_cmd; + int i; + int ret; + u64 force_lmem_bar_size = xe_force_lmem_bar_size; + + current_size = roundup_pow_of_two(pci_resource_len(pdev, GEN12_LMEM_BAR)); + + if (force_lmem_bar_size) { + u32 bar_sizes; + + rebar_size = force_lmem_bar_size * (resource_size_t)SZ_1M; + bar_sizes = pci_rebar_get_possible_sizes(pdev, GEN12_LMEM_BAR); + + if (rebar_size == current_size) + return 0; + + if (!(bar_sizes & BIT(pci_rebar_bytes_to_size(rebar_size))) || + rebar_size >= roundup_pow_of_two(lmem_size)) { + rebar_size = lmem_size; + drm_info(&xe->drm, + "Given bar size is not within supported size, setting it to default: %llu\n", + (u64)lmem_size >> 20); + } + } else { + rebar_size = current_size; + + if (rebar_size != roundup_pow_of_two(lmem_size)) + rebar_size = lmem_size; + else + return 0; + } + + while (root->parent) + root = root->parent; + + pci_bus_for_each_resource(root, root_res, i) { + if (root_res && root_res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) && + root_res->start > 0x100000000ull) + break; + } + + if (!root_res) { + drm_info(&xe->drm, "Can't resize LMEM BAR - platform support is missing\n"); + return -1; + } + + pci_read_config_dword(pdev, PCI_COMMAND, &pci_cmd); + pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd & ~PCI_COMMAND_MEMORY); + + ret = _resize_bar(xe, GEN12_LMEM_BAR, rebar_size); + + pci_assign_unassigned_bus_resources(pdev->bus); + pci_write_config_dword(pdev, PCI_COMMAND, pci_cmd); + return ret; +} +#else +static int xe_resize_lmem_bar(struct xe_device *xe, resource_size_t lmem_size) { return 0; } +#endif + +static bool xe_pci_resource_valid(struct pci_dev *pdev, int bar) +{ + if (!pci_resource_flags(pdev, bar)) + return false; + + if (pci_resource_flags(pdev, bar) & IORESOURCE_UNSET) + return false; + + if (!pci_resource_len(pdev, bar)) + return false; + + return true; +} + +int xe_mmio_probe_vram(struct xe_device *xe) +{ + struct pci_dev *pdev = to_pci_dev(xe->drm.dev); + struct xe_gt *gt; + u8 id; + u64 lmem_size; + u64 original_size; + u64 current_size; + u64 flat_ccs_base; + int resize_result; + + if (!IS_DGFX(xe)) { + xe->mem.vram.mapping = 0; + xe->mem.vram.size = 0; + xe->mem.vram.io_start = 0; + + for_each_gt(gt, xe, id) { + gt->mem.vram.mapping = 0; + gt->mem.vram.size = 0; + gt->mem.vram.io_start = 0; + } + return 0; + } + + if (!xe_pci_resource_valid(pdev, GEN12_LMEM_BAR)) { + drm_err(&xe->drm, "pci resource is not valid\n"); + return -ENXIO; + } + + gt = xe_device_get_gt(xe, 0); + lmem_size = xe_mmio_read64(gt, GEN12_GSMBASE.reg); + + original_size = pci_resource_len(pdev, GEN12_LMEM_BAR); + + if (xe->info.has_flat_ccs) { + int err; + u32 reg; + + err = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT); + if (err) + return err; + reg = xe_gt_mcr_unicast_read_any(gt, XEHP_TILE0_ADDR_RANGE); + lmem_size = (u64)REG_FIELD_GET(GENMASK(14, 8), reg) * SZ_1G; + reg = xe_gt_mcr_unicast_read_any(gt, XEHP_FLAT_CCS_BASE_ADDR); + flat_ccs_base = (u64)REG_FIELD_GET(GENMASK(31, 8), reg) * SZ_64K; + + drm_info(&xe->drm, "lmem_size: 0x%llx flat_ccs_base: 0x%llx\n", + lmem_size, flat_ccs_base); + + err = xe_force_wake_put(gt_to_fw(gt), XE_FW_GT); + if (err) + return err; + } else { + flat_ccs_base = lmem_size; + } + + resize_result = xe_resize_lmem_bar(xe, lmem_size); + current_size = pci_resource_len(pdev, GEN12_LMEM_BAR); + xe->mem.vram.io_start = pci_resource_start(pdev, GEN12_LMEM_BAR); + + xe->mem.vram.size = min(current_size, lmem_size); + + if (!xe->mem.vram.size) + return -EIO; + + if (resize_result > 0) + drm_info(&xe->drm, "Successfully resize LMEM from %lluMiB to %lluMiB\n", + (u64)original_size >> 20, + (u64)current_size >> 20); + else if (xe->mem.vram.size < lmem_size && !xe_force_lmem_bar_size) + drm_info(&xe->drm, "Using a reduced BAR size of %lluMiB. Consider enabling 'Resizable BAR' support in your BIOS.\n", + (u64)xe->mem.vram.size >> 20); + if (xe->mem.vram.size < lmem_size) + drm_warn(&xe->drm, "Restricting VRAM size to PCI resource size (0x%llx->0x%llx)\n", + lmem_size, xe->mem.vram.size); + +#ifdef CONFIG_64BIT + xe->mem.vram.mapping = ioremap_wc(xe->mem.vram.io_start, xe->mem.vram.size); +#endif + + xe->mem.vram.size = min_t(u64, xe->mem.vram.size, flat_ccs_base); + + drm_info(&xe->drm, "TOTAL VRAM: %pa, %pa\n", &xe->mem.vram.io_start, &xe->mem.vram.size); + + /* FIXME: Assuming equally partitioned VRAM, incorrect */ + if (xe->info.tile_count > 1) { + u8 adj_tile_count = xe->info.tile_count; + resource_size_t size, io_start; + + for_each_gt(gt, xe, id) + if (xe_gt_is_media_type(gt)) + --adj_tile_count; + + XE_BUG_ON(!adj_tile_count); + + size = xe->mem.vram.size / adj_tile_count; + io_start = xe->mem.vram.io_start; + + for_each_gt(gt, xe, id) { + if (id && !xe_gt_is_media_type(gt)) + io_start += size; + + gt->mem.vram.size = size; + gt->mem.vram.io_start = io_start; + gt->mem.vram.mapping = xe->mem.vram.mapping + + (io_start - xe->mem.vram.io_start); + + drm_info(&xe->drm, "VRAM[%u, %u]: %pa, %pa\n", + id, gt->info.vram_id, >->mem.vram.io_start, + >->mem.vram.size); + } + } else { + gt->mem.vram.size = xe->mem.vram.size; + gt->mem.vram.io_start = xe->mem.vram.io_start; + gt->mem.vram.mapping = xe->mem.vram.mapping; + + drm_info(&xe->drm, "VRAM: %pa\n", >->mem.vram.size); + } + return 0; +} + +static void xe_mmio_probe_tiles(struct xe_device *xe) +{ + struct xe_gt *gt = xe_device_get_gt(xe, 0); + u32 mtcfg; + u8 adj_tile_count; + u8 id; + + if (xe->info.tile_count == 1) + return; + + mtcfg = xe_mmio_read64(gt, XEHP_MTCFG_ADDR.reg); + adj_tile_count = xe->info.tile_count = + REG_FIELD_GET(TILE_COUNT, mtcfg) + 1; + if (xe->info.media_ver >= 13) + xe->info.tile_count *= 2; + + drm_info(&xe->drm, "tile_count: %d, adj_tile_count %d\n", + xe->info.tile_count, adj_tile_count); + + if (xe->info.tile_count > 1) { + const int mmio_bar = 0; + size_t size; + void *regs; + + if (adj_tile_count > 1) { + pci_iounmap(to_pci_dev(xe->drm.dev), xe->mmio.regs); + xe->mmio.size = SZ_16M * adj_tile_count; + xe->mmio.regs = pci_iomap(to_pci_dev(xe->drm.dev), + mmio_bar, xe->mmio.size); + } + + size = xe->mmio.size / adj_tile_count; + regs = xe->mmio.regs; + + for_each_gt(gt, xe, id) { + if (id && !xe_gt_is_media_type(gt)) + regs += size; + gt->mmio.size = size; + gt->mmio.regs = regs; + } + } +} + +static void mmio_fini(struct drm_device *drm, void *arg) +{ + struct xe_device *xe = arg; + + pci_iounmap(to_pci_dev(xe->drm.dev), xe->mmio.regs); + if (xe->mem.vram.mapping) + iounmap(xe->mem.vram.mapping); +} + +int xe_mmio_init(struct xe_device *xe) +{ + struct xe_gt *gt = xe_device_get_gt(xe, 0); + const int mmio_bar = 0; + int err; + + /* + * Map the entire BAR, which includes registers (0-4MB), reserved space + * (4MB-8MB), and GGTT (8MB-16MB). Other parts of the driver (GTs, + * GGTTs) will derive the pointers they need from the mapping in the + * device structure. + */ + xe->mmio.size = SZ_16M; + xe->mmio.regs = pci_iomap(to_pci_dev(xe->drm.dev), mmio_bar, + xe->mmio.size); + if (xe->mmio.regs == NULL) { + drm_err(&xe->drm, "failed to map registers\n"); + return -EIO; + } + + err = drmm_add_action_or_reset(&xe->drm, mmio_fini, xe); + if (err) + return err; + + /* 1 GT for now, 1 to 1 mapping, may change on multi-GT devices */ + gt->mmio.size = xe->mmio.size; + gt->mmio.regs = xe->mmio.regs; + + /* + * The boot firmware initializes local memory and assesses its health. + * If memory training fails, the punit will have been instructed to + * keep the GT powered down; we won't be able to communicate with it + * and we should not continue with driver initialization. + */ + if (IS_DGFX(xe) && !(xe_mmio_read32(gt, GU_CNTL.reg) & LMEM_INIT)) { + drm_err(&xe->drm, "LMEM not initialized by firmware\n"); + return -ENODEV; + } + + err = xe_set_dma_info(xe); + if (err) + return err; + + xe_mmio_probe_tiles(xe); + + return 0; +} + +#define VALID_MMIO_FLAGS (\ + DRM_XE_MMIO_BITS_MASK |\ + DRM_XE_MMIO_READ |\ + DRM_XE_MMIO_WRITE) + +static const i915_reg_t mmio_read_whitelist[] = { + RING_TIMESTAMP(RENDER_RING_BASE), +}; + +int xe_mmio_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct drm_xe_mmio *args = data; + unsigned int bits_flag, bytes; + bool allowed; + int ret = 0; + + if (XE_IOCTL_ERR(xe, args->extensions)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->flags & ~VALID_MMIO_FLAGS)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, !(args->flags & DRM_XE_MMIO_WRITE) && args->value)) + return -EINVAL; + + allowed = capable(CAP_SYS_ADMIN); + if (!allowed && ((args->flags & ~DRM_XE_MMIO_BITS_MASK) == DRM_XE_MMIO_READ)) { + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(mmio_read_whitelist); i++) { + if (mmio_read_whitelist[i].reg == args->addr) { + allowed = true; + break; + } + } + } + + if (XE_IOCTL_ERR(xe, !allowed)) + return -EPERM; + + bits_flag = args->flags & DRM_XE_MMIO_BITS_MASK; + bytes = 1 << bits_flag; + if (XE_IOCTL_ERR(xe, args->addr + bytes > xe->mmio.size)) + return -EINVAL; + + xe_force_wake_get(gt_to_fw(&xe->gt[0]), XE_FORCEWAKE_ALL); + + if (args->flags & DRM_XE_MMIO_WRITE) { + switch (bits_flag) { + case DRM_XE_MMIO_8BIT: + return -EINVAL; /* TODO */ + case DRM_XE_MMIO_16BIT: + return -EINVAL; /* TODO */ + case DRM_XE_MMIO_32BIT: + if (XE_IOCTL_ERR(xe, args->value > U32_MAX)) + return -EINVAL; + xe_mmio_write32(to_gt(xe), args->addr, args->value); + break; + case DRM_XE_MMIO_64BIT: + xe_mmio_write64(to_gt(xe), args->addr, args->value); + break; + default: + drm_WARN(&xe->drm, 1, "Invalid MMIO bit size"); + ret = -EINVAL; + goto exit; + } + } + + if (args->flags & DRM_XE_MMIO_READ) { + switch (bits_flag) { + case DRM_XE_MMIO_8BIT: + return -EINVAL; /* TODO */ + case DRM_XE_MMIO_16BIT: + return -EINVAL; /* TODO */ + case DRM_XE_MMIO_32BIT: + args->value = xe_mmio_read32(to_gt(xe), args->addr); + break; + case DRM_XE_MMIO_64BIT: + args->value = xe_mmio_read64(to_gt(xe), args->addr); + break; + default: + drm_WARN(&xe->drm, 1, "Invalid MMIO bit size"); + ret = -EINVAL; + } + } + +exit: + xe_force_wake_put(gt_to_fw(&xe->gt[0]), XE_FORCEWAKE_ALL); + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_mmio.h b/drivers/gpu/drm/xe/xe_mmio.h new file mode 100644 index 000000000000..09d24467096f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mmio.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_MMIO_H_ +#define _XE_MMIO_H_ + +#include + +#include "xe_gt_types.h" + +/* + * FIXME: This header has been deemed evil and we need to kill it. Temporarily + * including so we can use 'wait_for' and unblock initial development. A follow + * should replace 'wait_for' with a sane version and drop including this header. + */ +#include "i915_utils.h" + +struct drm_device; +struct drm_file; +struct xe_device; + +int xe_mmio_init(struct xe_device *xe); + +static inline u8 xe_mmio_read8(struct xe_gt *gt, u32 reg) +{ + if (reg < gt->mmio.adj_limit) + reg += gt->mmio.adj_offset; + + return readb(gt->mmio.regs + reg); +} + +static inline void xe_mmio_write32(struct xe_gt *gt, + u32 reg, u32 val) +{ + if (reg < gt->mmio.adj_limit) + reg += gt->mmio.adj_offset; + + writel(val, gt->mmio.regs + reg); +} + +static inline u32 xe_mmio_read32(struct xe_gt *gt, u32 reg) +{ + if (reg < gt->mmio.adj_limit) + reg += gt->mmio.adj_offset; + + return readl(gt->mmio.regs + reg); +} + +static inline u32 xe_mmio_rmw32(struct xe_gt *gt, u32 reg, u32 mask, + u32 val) +{ + u32 old, reg_val; + + old = xe_mmio_read32(gt, reg); + reg_val = (old & mask) | val; + xe_mmio_write32(gt, reg, reg_val); + + return old; +} + +static inline void xe_mmio_write64(struct xe_gt *gt, + u32 reg, u64 val) +{ + if (reg < gt->mmio.adj_limit) + reg += gt->mmio.adj_offset; + + writeq(val, gt->mmio.regs + reg); +} + +static inline u64 xe_mmio_read64(struct xe_gt *gt, u32 reg) +{ + if (reg < gt->mmio.adj_limit) + reg += gt->mmio.adj_offset; + + return readq(gt->mmio.regs + reg); +} + +static inline int xe_mmio_write32_and_verify(struct xe_gt *gt, + u32 reg, u32 val, + u32 mask, u32 eval) +{ + u32 reg_val; + + xe_mmio_write32(gt, reg, val); + reg_val = xe_mmio_read32(gt, reg); + + return (reg_val & mask) != eval ? -EINVAL : 0; +} + +static inline int xe_mmio_wait32(struct xe_gt *gt, + u32 reg, u32 val, + u32 mask, u32 timeout_ms) +{ + return wait_for((xe_mmio_read32(gt, reg) & mask) == val, + timeout_ms); +} + +int xe_mmio_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +static inline bool xe_mmio_in_range(const struct xe_mmio_range *range, u32 reg) +{ + return range && reg >= range->start && reg <= range->end; +} + +int xe_mmio_probe_vram(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_mocs.c b/drivers/gpu/drm/xe/xe_mocs.c new file mode 100644 index 000000000000..86b966fffbe5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mocs.c @@ -0,0 +1,557 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_engine.h" +#include "xe_gt.h" +#include "xe_platform_types.h" +#include "xe_mmio.h" +#include "xe_mocs.h" +#include "xe_step_types.h" + +#include "gt/intel_gt_regs.h" + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG) +#define mocs_dbg drm_dbg +#else +__printf(2, 3) +static inline void mocs_dbg(const struct drm_device *dev, + const char *format, ...) +{ /* noop */ } +#endif + +/* + * MOCS indexes used for GPU surfaces, defining the cacheability of the + * surface data and the coherency for this data wrt. CPU vs. GPU accesses. + */ +enum xe_mocs_info_index { + /* + * Not cached anywhere, coherency between CPU and GPU accesses is + * guaranteed. + */ + XE_MOCS_UNCACHED, + /* + * Cacheability and coherency controlled by the kernel automatically + * based on the xxxx IOCTL setting and the current + * usage of the surface (used for display scanout or not). + */ + XE_MOCS_PTE, + /* + * Cached in all GPU caches available on the platform. + * Coherency between CPU and GPU accesses to the surface is not + * guaranteed without extra synchronization. + */ + XE_MOCS_CACHED, +}; + +enum { + HAS_GLOBAL_MOCS = BIT(0), + HAS_RENDER_L3CC = BIT(1), +}; + +struct xe_mocs_entry { + u32 control_value; + u16 l3cc_value; + u16 used; +}; + +struct xe_mocs_info { + unsigned int size; + unsigned int n_entries; + const struct xe_mocs_entry *table; + u8 uc_index; + u8 wb_index; + u8 unused_entries_index; +}; + +/* Defines for the tables (XXX_MOCS_0 - XXX_MOCS_63) */ +#define _LE_CACHEABILITY(value) ((value) << 0) +#define _LE_TGT_CACHE(value) ((value) << 2) +#define LE_LRUM(value) ((value) << 4) +#define LE_AOM(value) ((value) << 6) +#define LE_RSC(value) ((value) << 7) +#define LE_SCC(value) ((value) << 8) +#define LE_PFM(value) ((value) << 11) +#define LE_SCF(value) ((value) << 14) +#define LE_COS(value) ((value) << 15) +#define LE_SSE(value) ((value) << 17) + +/* Defines for the tables (LNCFMOCS0 - LNCFMOCS31) - two entries per word */ +#define L3_ESC(value) ((value) << 0) +#define L3_SCC(value) ((value) << 1) +#define _L3_CACHEABILITY(value) ((value) << 4) +#define L3_GLBGO(value) ((value) << 6) +#define L3_LKUP(value) ((value) << 7) + +/* Helper defines */ +#define GEN9_NUM_MOCS_ENTRIES 64 /* 63-64 are reserved, but configured. */ +#define PVC_NUM_MOCS_ENTRIES 3 +#define MTL_NUM_MOCS_ENTRIES 16 + +/* (e)LLC caching options */ +/* + * Note: LE_0_PAGETABLE works only up to Gen11; for newer gens it means + * the same as LE_UC + */ +#define LE_0_PAGETABLE _LE_CACHEABILITY(0) +#define LE_1_UC _LE_CACHEABILITY(1) +#define LE_2_WT _LE_CACHEABILITY(2) +#define LE_3_WB _LE_CACHEABILITY(3) + +/* Target cache */ +#define LE_TC_0_PAGETABLE _LE_TGT_CACHE(0) +#define LE_TC_1_LLC _LE_TGT_CACHE(1) +#define LE_TC_2_LLC_ELLC _LE_TGT_CACHE(2) +#define LE_TC_3_LLC_ELLC_ALT _LE_TGT_CACHE(3) + +/* L3 caching options */ +#define L3_0_DIRECT _L3_CACHEABILITY(0) +#define L3_1_UC _L3_CACHEABILITY(1) +#define L3_2_RESERVED _L3_CACHEABILITY(2) +#define L3_3_WB _L3_CACHEABILITY(3) + +#define MOCS_ENTRY(__idx, __control_value, __l3cc_value) \ + [__idx] = { \ + .control_value = __control_value, \ + .l3cc_value = __l3cc_value, \ + .used = 1, \ + } + +/* + * MOCS tables + * + * These are the MOCS tables that are programmed across all the rings. + * The control value is programmed to all the rings that support the + * MOCS registers. While the l3cc_values are only programmed to the + * LNCFCMOCS0 - LNCFCMOCS32 registers. + * + * These tables are intended to be kept reasonably consistent across + * HW platforms, and for ICL+, be identical across OSes. To achieve + * that, for Icelake and above, list of entries is published as part + * of bspec. + * + * Entries not part of the following tables are undefined as far as + * userspace is concerned and shouldn't be relied upon. For Gen < 12 + * they will be initialized to PTE. Gen >= 12 don't have a setting for + * PTE and those platforms except TGL/RKL will be initialized L3 WB to + * catch accidental use of reserved and unused mocs indexes. + * + * The last few entries are reserved by the hardware. For ICL+ they + * should be initialized according to bspec and never used, for older + * platforms they should never be written to. + * + * NOTE1: These tables are part of bspec and defined as part of hardware + * interface for ICL+. For older platforms, they are part of kernel + * ABI. It is expected that, for specific hardware platform, existing + * entries will remain constant and the table will only be updated by + * adding new entries, filling unused positions. + * + * NOTE2: For GEN >= 12 except TGL and RKL, reserved and unspecified MOCS + * indices have been set to L3 WB. These reserved entries should never + * be used, they may be changed to low performant variants with better + * coherency in the future if more entries are needed. + * For TGL/RKL, all the unspecified MOCS indexes are mapped to L3 UC. + */ + +#define GEN11_MOCS_ENTRIES \ + /* Entries 0 and 1 are defined per-platform */ \ + /* Base - L3 + LLC */ \ + MOCS_ENTRY(2, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ + L3_3_WB), \ + /* Base - Uncached */ \ + MOCS_ENTRY(3, \ + LE_1_UC | LE_TC_1_LLC, \ + L3_1_UC), \ + /* Base - L3 */ \ + MOCS_ENTRY(4, \ + LE_1_UC | LE_TC_1_LLC, \ + L3_3_WB), \ + /* Base - LLC */ \ + MOCS_ENTRY(5, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ + L3_1_UC), \ + /* Age 0 - LLC */ \ + MOCS_ENTRY(6, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \ + L3_1_UC), \ + /* Age 0 - L3 + LLC */ \ + MOCS_ENTRY(7, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1), \ + L3_3_WB), \ + /* Age: Don't Chg. - LLC */ \ + MOCS_ENTRY(8, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \ + L3_1_UC), \ + /* Age: Don't Chg. - L3 + LLC */ \ + MOCS_ENTRY(9, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2), \ + L3_3_WB), \ + /* No AOM - LLC */ \ + MOCS_ENTRY(10, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \ + L3_1_UC), \ + /* No AOM - L3 + LLC */ \ + MOCS_ENTRY(11, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_AOM(1), \ + L3_3_WB), \ + /* No AOM; Age 0 - LLC */ \ + MOCS_ENTRY(12, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \ + L3_1_UC), \ + /* No AOM; Age 0 - L3 + LLC */ \ + MOCS_ENTRY(13, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(1) | LE_AOM(1), \ + L3_3_WB), \ + /* No AOM; Age:DC - LLC */ \ + MOCS_ENTRY(14, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \ + L3_1_UC), \ + /* No AOM; Age:DC - L3 + LLC */ \ + MOCS_ENTRY(15, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(2) | LE_AOM(1), \ + L3_3_WB), \ + /* Self-Snoop - L3 + LLC */ \ + MOCS_ENTRY(18, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SSE(3), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(12.5%) */ \ + MOCS_ENTRY(19, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(7), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(25%) */ \ + MOCS_ENTRY(20, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(3), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(50%) */ \ + MOCS_ENTRY(21, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_SCC(1), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(75%) */ \ + MOCS_ENTRY(22, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(3), \ + L3_3_WB), \ + /* Skip Caching - L3 + LLC(87.5%) */ \ + MOCS_ENTRY(23, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3) | LE_RSC(1) | LE_SCC(7), \ + L3_3_WB), \ + /* HW Reserved - SW program but never use */ \ + MOCS_ENTRY(62, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ + L3_1_UC), \ + /* HW Reserved - SW program but never use */ \ + MOCS_ENTRY(63, \ + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), \ + L3_1_UC) + +static const struct xe_mocs_entry tgl_mocs_desc[] = { + /* + * NOTE: + * Reserved and unspecified MOCS indices have been set to (L3 + LCC). + * These reserved entries should never be used, they may be changed + * to low performant variants with better coherency in the future if + * more entries are needed. We are programming index XE_MOCS_PTE(1) + * only, __init_mocs_table() take care to program unused index with + * this entry. + */ + MOCS_ENTRY(XE_MOCS_PTE, + LE_0_PAGETABLE | LE_TC_0_PAGETABLE, + L3_1_UC), + GEN11_MOCS_ENTRIES, + + /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ + MOCS_ENTRY(48, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_3_WB), + /* Implicitly enable L1 - HDC:L1 + L3 */ + MOCS_ENTRY(49, + LE_1_UC | LE_TC_1_LLC, + L3_3_WB), + /* Implicitly enable L1 - HDC:L1 + LLC */ + MOCS_ENTRY(50, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* Implicitly enable L1 - HDC:L1 */ + MOCS_ENTRY(51, + LE_1_UC | LE_TC_1_LLC, + L3_1_UC), + /* HW Special Case (CCS) */ + MOCS_ENTRY(60, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* HW Special Case (Displayable) */ + MOCS_ENTRY(61, + LE_1_UC | LE_TC_1_LLC, + L3_3_WB), +}; + +static const struct xe_mocs_entry dg1_mocs_desc[] = { + /* UC */ + MOCS_ENTRY(1, 0, L3_1_UC), + /* WB - L3 */ + MOCS_ENTRY(5, 0, L3_3_WB), + /* WB - L3 50% */ + MOCS_ENTRY(6, 0, L3_ESC(1) | L3_SCC(1) | L3_3_WB), + /* WB - L3 25% */ + MOCS_ENTRY(7, 0, L3_ESC(1) | L3_SCC(3) | L3_3_WB), + /* WB - L3 12.5% */ + MOCS_ENTRY(8, 0, L3_ESC(1) | L3_SCC(7) | L3_3_WB), + + /* HDC:L1 + L3 */ + MOCS_ENTRY(48, 0, L3_3_WB), + /* HDC:L1 */ + MOCS_ENTRY(49, 0, L3_1_UC), + + /* HW Reserved */ + MOCS_ENTRY(60, 0, L3_1_UC), + MOCS_ENTRY(61, 0, L3_1_UC), + MOCS_ENTRY(62, 0, L3_1_UC), + MOCS_ENTRY(63, 0, L3_1_UC), +}; + +static const struct xe_mocs_entry gen12_mocs_desc[] = { + GEN11_MOCS_ENTRIES, + /* Implicitly enable L1 - HDC:L1 + L3 + LLC */ + MOCS_ENTRY(48, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_3_WB), + /* Implicitly enable L1 - HDC:L1 + L3 */ + MOCS_ENTRY(49, + LE_1_UC | LE_TC_1_LLC, + L3_3_WB), + /* Implicitly enable L1 - HDC:L1 + LLC */ + MOCS_ENTRY(50, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* Implicitly enable L1 - HDC:L1 */ + MOCS_ENTRY(51, + LE_1_UC | LE_TC_1_LLC, + L3_1_UC), + /* HW Special Case (CCS) */ + MOCS_ENTRY(60, + LE_3_WB | LE_TC_1_LLC | LE_LRUM(3), + L3_1_UC), + /* HW Special Case (Displayable) */ + MOCS_ENTRY(61, + LE_1_UC | LE_TC_1_LLC, + L3_3_WB), +}; + +static const struct xe_mocs_entry dg2_mocs_desc[] = { + /* UC - Coherent; GO:L3 */ + MOCS_ENTRY(0, 0, L3_1_UC | L3_LKUP(1)), + /* UC - Coherent; GO:Memory */ + MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), + /* UC - Non-Coherent; GO:Memory */ + MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)), + + /* WB - LC */ + MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)), +}; + +static const struct xe_mocs_entry dg2_mocs_desc_g10_ax[] = { + /* Wa_14011441408: Set Go to Memory for MOCS#0 */ + MOCS_ENTRY(0, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), + /* UC - Coherent; GO:Memory */ + MOCS_ENTRY(1, 0, L3_1_UC | L3_GLBGO(1) | L3_LKUP(1)), + /* UC - Non-Coherent; GO:Memory */ + MOCS_ENTRY(2, 0, L3_1_UC | L3_GLBGO(1)), + + /* WB - LC */ + MOCS_ENTRY(3, 0, L3_3_WB | L3_LKUP(1)), +}; + +static const struct xe_mocs_entry pvc_mocs_desc[] = { + /* Error */ + MOCS_ENTRY(0, 0, L3_3_WB), + + /* UC */ + MOCS_ENTRY(1, 0, L3_1_UC), + + /* WB */ + MOCS_ENTRY(2, 0, L3_3_WB), +}; + +static unsigned int get_mocs_settings(struct xe_device *xe, + struct xe_mocs_info *info) +{ + unsigned int flags; + + memset(info, 0, sizeof(struct xe_mocs_info)); + + info->unused_entries_index = XE_MOCS_PTE; + switch (xe->info.platform) { + case XE_PVC: + info->size = ARRAY_SIZE(pvc_mocs_desc); + info->table = pvc_mocs_desc; + info->n_entries = PVC_NUM_MOCS_ENTRIES; + info->uc_index = 1; + info->wb_index = 2; + info->unused_entries_index = 2; + break; + case XE_METEORLAKE: + info->size = ARRAY_SIZE(dg2_mocs_desc); + info->table = dg2_mocs_desc; + info->n_entries = MTL_NUM_MOCS_ENTRIES; + info->uc_index = 1; + info->unused_entries_index = 3; + break; + case XE_DG2: + if (xe->info.subplatform == XE_SUBPLATFORM_DG2_G10 && + xe->info.step.graphics >= STEP_A0 && + xe->info.step.graphics <= STEP_B0) { + info->size = ARRAY_SIZE(dg2_mocs_desc_g10_ax); + info->table = dg2_mocs_desc_g10_ax; + } else { + info->size = ARRAY_SIZE(dg2_mocs_desc); + info->table = dg2_mocs_desc; + } + info->uc_index = 1; + info->n_entries = GEN9_NUM_MOCS_ENTRIES; + info->unused_entries_index = 3; + break; + case XE_DG1: + info->size = ARRAY_SIZE(dg1_mocs_desc); + info->table = dg1_mocs_desc; + info->uc_index = 1; + info->n_entries = GEN9_NUM_MOCS_ENTRIES; + info->uc_index = 1; + info->unused_entries_index = 5; + break; + case XE_TIGERLAKE: + info->size = ARRAY_SIZE(tgl_mocs_desc); + info->table = tgl_mocs_desc; + info->n_entries = GEN9_NUM_MOCS_ENTRIES; + info->uc_index = 3; + break; + case XE_ALDERLAKE_S: + case XE_ALDERLAKE_P: + info->size = ARRAY_SIZE(gen12_mocs_desc); + info->table = gen12_mocs_desc; + info->n_entries = GEN9_NUM_MOCS_ENTRIES; + info->uc_index = 3; + info->unused_entries_index = 2; + break; + default: + drm_err(&xe->drm, "Platform that should have a MOCS table does not.\n"); + return 0; + } + + if (XE_WARN_ON(info->size > info->n_entries)) + return 0; + + flags = HAS_RENDER_L3CC; + if (!IS_DGFX(xe)) + flags |= HAS_GLOBAL_MOCS; + + return flags; +} + +/* + * Get control_value from MOCS entry taking into account when it's not used + * then if unused_entries_index is non-zero then its value will be returned + * otherwise XE_MOCS_PTE's value is returned in this case. + */ +static u32 get_entry_control(const struct xe_mocs_info *info, + unsigned int index) +{ + if (index < info->size && info->table[index].used) + return info->table[index].control_value; + return info->table[info->unused_entries_index].control_value; +} + +static void __init_mocs_table(struct xe_gt *gt, + const struct xe_mocs_info *info, + u32 addr) +{ + struct xe_device *xe = gt_to_xe(gt); + + unsigned int i; + u32 mocs; + + mocs_dbg(>->xe->drm, "entries:%d\n", info->n_entries); + drm_WARN_ONCE(&xe->drm, !info->unused_entries_index, + "Unused entries index should have been defined\n"); + for (i = 0; + i < info->n_entries ? (mocs = get_entry_control(info, i)), 1 : 0; + i++) { + mocs_dbg(>->xe->drm, "%d 0x%x 0x%x\n", i, _MMIO(addr + i * 4).reg, mocs); + xe_mmio_write32(gt, _MMIO(addr + i * 4).reg, mocs); + } +} + +/* + * Get l3cc_value from MOCS entry taking into account when it's not used + * then if unused_entries_index is not zero then its value will be returned + * otherwise I915_MOCS_PTE's value is returned in this case. + */ +static u16 get_entry_l3cc(const struct xe_mocs_info *info, + unsigned int index) +{ + if (index < info->size && info->table[index].used) + return info->table[index].l3cc_value; + return info->table[info->unused_entries_index].l3cc_value; +} + +static u32 l3cc_combine(u16 low, u16 high) +{ + return low | (u32)high << 16; +} + +static void init_l3cc_table(struct xe_gt *gt, + const struct xe_mocs_info *info) +{ + unsigned int i; + u32 l3cc; + + mocs_dbg(>->xe->drm, "entries:%d\n", info->n_entries); + for (i = 0; + i < (info->n_entries + 1) / 2 ? + (l3cc = l3cc_combine(get_entry_l3cc(info, 2 * i), + get_entry_l3cc(info, 2 * i + 1))), 1 : 0; + i++) { + mocs_dbg(>->xe->drm, "%d 0x%x 0x%x\n", i, GEN9_LNCFCMOCS(i).reg, l3cc); + xe_mmio_write32(gt, GEN9_LNCFCMOCS(i).reg, l3cc); + } +} + +void xe_mocs_init_engine(const struct xe_engine *engine) +{ + struct xe_mocs_info table; + unsigned int flags; + + flags = get_mocs_settings(engine->gt->xe, &table); + if (!flags) + return; + + if (flags & HAS_RENDER_L3CC && engine->class == XE_ENGINE_CLASS_RENDER) + init_l3cc_table(engine->gt, &table); +} + +void xe_mocs_init(struct xe_gt *gt) +{ + struct xe_mocs_info table; + unsigned int flags; + + /* + * LLC and eDRAM control values are not applicable to dgfx + */ + flags = get_mocs_settings(gt->xe, &table); + mocs_dbg(>->xe->drm, "flag:0x%x\n", flags); + gt->mocs.uc_index = table.uc_index; + gt->mocs.wb_index = table.wb_index; + + if (flags & HAS_GLOBAL_MOCS) + __init_mocs_table(gt, &table, GEN12_GLOBAL_MOCS(0).reg); + + /* + * Initialize the L3CC table as part of mocs initalization to make + * sure the LNCFCMOCSx registers are programmed for the subsequent + * memory transactions including guc transactions + */ + if (flags & HAS_RENDER_L3CC) + init_l3cc_table(gt, &table); +} diff --git a/drivers/gpu/drm/xe/xe_mocs.h b/drivers/gpu/drm/xe/xe_mocs.h new file mode 100644 index 000000000000..aba1abe216ab --- /dev/null +++ b/drivers/gpu/drm/xe/xe_mocs.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_MOCS_H_ +#define _XE_MOCS_H_ + +#include + +struct xe_engine; +struct xe_gt; + +void xe_mocs_init_engine(const struct xe_engine *engine); +void xe_mocs_init(struct xe_gt *gt); + +/** + * xe_mocs_index_to_value - Translate mocs index to the mocs value exected by + * most blitter commands. + * @mocs_index: index into the mocs tables + * + * Return: The corresponding mocs value to be programmed. + */ +static inline u32 xe_mocs_index_to_value(u32 mocs_index) +{ + return mocs_index << 1; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c new file mode 100644 index 000000000000..cc862553a252 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_module.c @@ -0,0 +1,76 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include +#include + +#include "xe_drv.h" +#include "xe_hw_fence.h" +#include "xe_module.h" +#include "xe_pci.h" +#include "xe_sched_job.h" + +bool enable_guc = true; +module_param_named_unsafe(enable_guc, enable_guc, bool, 0444); +MODULE_PARM_DESC(enable_guc, "Enable GuC submission"); + +u32 xe_force_lmem_bar_size; +module_param_named(lmem_bar_size, xe_force_lmem_bar_size, uint, 0600); +MODULE_PARM_DESC(lmem_bar_size, "Set the lmem bar size(in MiB)"); + +int xe_guc_log_level = 5; +module_param_named(guc_log_level, xe_guc_log_level, int, 0600); +MODULE_PARM_DESC(guc_log_level, "GuC firmware logging level (0=disable, 1..5=enable with verbosity min..max)"); + +char *xe_param_force_probe = CONFIG_DRM_XE_FORCE_PROBE; +module_param_named_unsafe(force_probe, xe_param_force_probe, charp, 0400); +MODULE_PARM_DESC(force_probe, + "Force probe options for specified devices. See CONFIG_DRM_XE_FORCE_PROBE for details."); + +struct init_funcs { + int (*init)(void); + void (*exit)(void); +}; +#define MAKE_INIT_EXIT_FUNCS(name) \ + { .init = xe_##name##_module_init, \ + .exit = xe_##name##_module_exit, } +static const struct init_funcs init_funcs[] = { + MAKE_INIT_EXIT_FUNCS(hw_fence), + MAKE_INIT_EXIT_FUNCS(sched_job), +}; + +static int __init xe_init(void) +{ + int err, i; + + for (i = 0; i < ARRAY_SIZE(init_funcs); i++) { + err = init_funcs[i].init(); + if (err) { + while (i--) + init_funcs[i].exit(); + return err; + } + } + + return xe_register_pci_driver(); +} + +static void __exit xe_exit(void) +{ + int i; + + xe_unregister_pci_driver(); + + for (i = ARRAY_SIZE(init_funcs) - 1; i >= 0; i--) + init_funcs[i].exit(); +} + +module_init(xe_init); +module_exit(xe_exit); + +MODULE_AUTHOR("Intel Corporation"); + +MODULE_DESCRIPTION(DRIVER_DESC); +MODULE_LICENSE("GPL and additional rights"); diff --git a/drivers/gpu/drm/xe/xe_module.h b/drivers/gpu/drm/xe/xe_module.h new file mode 100644 index 000000000000..2c6ee46f5595 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_module.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#include + +/* Module modprobe variables */ +extern bool enable_guc; +extern bool enable_display; +extern u32 xe_force_lmem_bar_size; +extern int xe_guc_log_level; +extern char *xe_param_force_probe; diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c new file mode 100644 index 000000000000..55d8a597a068 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -0,0 +1,651 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_pci.h" + +#include +#include +#include +#include + +#include +#include +#include + +#include "xe_drv.h" +#include "xe_device.h" +#include "xe_macros.h" +#include "xe_module.h" +#include "xe_pm.h" +#include "xe_step.h" + +#include "i915_reg.h" + +#define DEV_INFO_FOR_EACH_FLAG(func) \ + func(require_force_probe); \ + func(is_dgfx); \ + /* Keep has_* in alphabetical order */ \ + +struct xe_subplatform_desc { + enum xe_subplatform subplatform; + const char *name; + const u16 *pciidlist; +}; + +struct xe_gt_desc { + enum xe_gt_type type; + u8 vram_id; + u64 engine_mask; + u32 mmio_adj_limit; + u32 mmio_adj_offset; +}; + +struct xe_device_desc { + u8 graphics_ver; + u8 graphics_rel; + u8 media_ver; + u8 media_rel; + + u64 platform_engine_mask; /* Engines supported by the HW */ + + enum xe_platform platform; + const char *platform_name; + const struct xe_subplatform_desc *subplatforms; + const struct xe_gt_desc *extra_gts; + + u8 dma_mask_size; /* available DMA address bits */ + + u8 gt; /* GT number, 0 if undefined */ + +#define DEFINE_FLAG(name) u8 name:1 + DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG); +#undef DEFINE_FLAG + + u8 vram_flags; + u8 max_tiles; + u8 vm_max_level; + + bool supports_usm; + bool has_flat_ccs; + bool has_4tile; +}; + +#define PLATFORM(x) \ + .platform = (x), \ + .platform_name = #x + +#define NOP(x) x + +/* Keep in gen based order, and chronological order within a gen */ +#define GEN12_FEATURES \ + .require_force_probe = true, \ + .graphics_ver = 12, \ + .media_ver = 12, \ + .dma_mask_size = 39, \ + .max_tiles = 1, \ + .vm_max_level = 3, \ + .vram_flags = 0 + +static const struct xe_device_desc tgl_desc = { + GEN12_FEATURES, + PLATFORM(XE_TIGERLAKE), + .platform_engine_mask = + BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | + BIT(XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_VCS0) | + BIT(XE_HW_ENGINE_VCS2), +}; + +static const struct xe_device_desc adl_s_desc = { + GEN12_FEATURES, + PLATFORM(XE_ALDERLAKE_S), + .platform_engine_mask = + BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | + BIT(XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_VCS0) | + BIT(XE_HW_ENGINE_VCS2), +}; + +static const u16 adlp_rplu_ids[] = { XE_RPLU_IDS(NOP), 0 }; + +static const struct xe_device_desc adl_p_desc = { + GEN12_FEATURES, + PLATFORM(XE_ALDERLAKE_P), + .platform_engine_mask = + BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | + BIT(XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_VCS0) | + BIT(XE_HW_ENGINE_VCS2), + .subplatforms = (const struct xe_subplatform_desc[]) { + { XE_SUBPLATFORM_ADLP_RPLU, "RPLU", adlp_rplu_ids }, + {}, + }, +}; + +#define DGFX_FEATURES \ + .is_dgfx = 1 + +static const struct xe_device_desc dg1_desc = { + GEN12_FEATURES, + DGFX_FEATURES, + .graphics_rel = 10, + PLATFORM(XE_DG1), + .platform_engine_mask = + BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | + BIT(XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_VCS0) | + BIT(XE_HW_ENGINE_VCS2), +}; + +#define XE_HP_FEATURES \ + .require_force_probe = true, \ + .graphics_ver = 12, \ + .graphics_rel = 50, \ + .has_flat_ccs = true, \ + .dma_mask_size = 46, \ + .max_tiles = 1, \ + .vm_max_level = 3 + +#define XE_HPM_FEATURES \ + .media_ver = 12, \ + .media_rel = 50 + +static const u16 dg2_g10_ids[] = { XE_DG2_G10_IDS(NOP), XE_ATS_M150_IDS(NOP), 0 }; +static const u16 dg2_g11_ids[] = { XE_DG2_G11_IDS(NOP), XE_ATS_M75_IDS(NOP), 0 }; +static const u16 dg2_g12_ids[] = { XE_DG2_G12_IDS(NOP), 0 }; + +#define DG2_FEATURES \ + DGFX_FEATURES, \ + .graphics_rel = 55, \ + .media_rel = 55, \ + PLATFORM(XE_DG2), \ + .subplatforms = (const struct xe_subplatform_desc[]) { \ + { XE_SUBPLATFORM_DG2_G10, "G10", dg2_g10_ids }, \ + { XE_SUBPLATFORM_DG2_G11, "G11", dg2_g11_ids }, \ + { XE_SUBPLATFORM_DG2_G12, "G12", dg2_g12_ids }, \ + { } \ + }, \ + .platform_engine_mask = \ + BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | \ + BIT(XE_HW_ENGINE_VECS0) | BIT(XE_HW_ENGINE_VECS1) | \ + BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS2) | \ + BIT(XE_HW_ENGINE_CCS0) | BIT(XE_HW_ENGINE_CCS1) | \ + BIT(XE_HW_ENGINE_CCS2) | BIT(XE_HW_ENGINE_CCS3), \ + .require_force_probe = true, \ + .vram_flags = XE_VRAM_FLAGS_NEED64K, \ + .has_4tile = 1 + +static const struct xe_device_desc ats_m_desc = { + XE_HP_FEATURES, + XE_HPM_FEATURES, + + DG2_FEATURES, +}; + +static const struct xe_device_desc dg2_desc = { + XE_HP_FEATURES, + XE_HPM_FEATURES, + + DG2_FEATURES, +}; + +#define PVC_ENGINES \ + BIT(XE_HW_ENGINE_BCS0) | BIT(XE_HW_ENGINE_BCS1) | \ + BIT(XE_HW_ENGINE_BCS2) | BIT(XE_HW_ENGINE_BCS3) | \ + BIT(XE_HW_ENGINE_BCS4) | BIT(XE_HW_ENGINE_BCS5) | \ + BIT(XE_HW_ENGINE_BCS6) | BIT(XE_HW_ENGINE_BCS7) | \ + BIT(XE_HW_ENGINE_BCS8) | \ + BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS1) | \ + BIT(XE_HW_ENGINE_VCS2) | \ + BIT(XE_HW_ENGINE_CCS0) | BIT(XE_HW_ENGINE_CCS1) | \ + BIT(XE_HW_ENGINE_CCS2) | BIT(XE_HW_ENGINE_CCS3) + +static const struct xe_gt_desc pvc_gts[] = { + { + .type = XE_GT_TYPE_REMOTE, + .vram_id = 1, + .engine_mask = PVC_ENGINES, + .mmio_adj_limit = 0, + .mmio_adj_offset = 0, + }, +}; + +static const __maybe_unused struct xe_device_desc pvc_desc = { + XE_HP_FEATURES, + XE_HPM_FEATURES, + DGFX_FEATURES, + PLATFORM(XE_PVC), + .extra_gts = pvc_gts, + .graphics_rel = 60, + .has_flat_ccs = 0, + .media_rel = 60, + .platform_engine_mask = PVC_ENGINES, + .vram_flags = XE_VRAM_FLAGS_NEED64K, + .dma_mask_size = 52, + .max_tiles = 2, + .vm_max_level = 4, + .supports_usm = true, +}; + +#define MTL_MEDIA_ENGINES \ + BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VCS2) | \ + BIT(XE_HW_ENGINE_VECS0) /* TODO: GSC0 */ + +static const struct xe_gt_desc xelpmp_gts[] = { + { + .type = XE_GT_TYPE_MEDIA, + .vram_id = 0, + .engine_mask = MTL_MEDIA_ENGINES, + .mmio_adj_limit = 0x40000, + .mmio_adj_offset = 0x380000, + }, +}; + +#define MTL_MAIN_ENGINES \ + BIT(XE_HW_ENGINE_RCS0) | BIT(XE_HW_ENGINE_BCS0) | \ + BIT(XE_HW_ENGINE_CCS0) + +static const struct xe_device_desc mtl_desc = { + /* + * Real graphics IP version will be obtained from hardware GMD_ID + * register. Value provided here is just for sanity checking. + */ + .require_force_probe = true, + .graphics_ver = 12, + .graphics_rel = 70, + .dma_mask_size = 46, + .max_tiles = 2, + .vm_max_level = 3, + .media_ver = 13, + PLATFORM(XE_METEORLAKE), + .extra_gts = xelpmp_gts, + .platform_engine_mask = MTL_MAIN_ENGINES, +}; + +#undef PLATFORM + +#define INTEL_VGA_DEVICE(id, info) { \ + PCI_DEVICE(PCI_VENDOR_ID_INTEL, id), \ + PCI_BASE_CLASS_DISPLAY << 16, 0xff << 16, \ + (unsigned long) info } + +/* + * Make sure any device matches here are from most specific to most + * general. For example, since the Quanta match is based on the subsystem + * and subvendor IDs, we need it to come before the more general IVB + * PCI ID matches, otherwise we'll use the wrong info struct above. + */ +static const struct pci_device_id pciidlist[] = { + XE_TGL_GT2_IDS(INTEL_VGA_DEVICE, &tgl_desc), + XE_DG1_IDS(INTEL_VGA_DEVICE, &dg1_desc), + XE_ATS_M_IDS(INTEL_VGA_DEVICE, &ats_m_desc), + XE_DG2_IDS(INTEL_VGA_DEVICE, &dg2_desc), + XE_ADLS_IDS(INTEL_VGA_DEVICE, &adl_s_desc), + XE_ADLP_IDS(INTEL_VGA_DEVICE, &adl_p_desc), + XE_MTL_IDS(INTEL_VGA_DEVICE, &mtl_desc), + { } +}; +MODULE_DEVICE_TABLE(pci, pciidlist); + +#undef INTEL_VGA_DEVICE + +/* is device_id present in comma separated list of ids */ +static bool device_id_in_list(u16 device_id, const char *devices, bool negative) +{ + char *s, *p, *tok; + bool ret; + + if (!devices || !*devices) + return false; + + /* match everything */ + if (negative && strcmp(devices, "!*") == 0) + return true; + if (!negative && strcmp(devices, "*") == 0) + return true; + + s = kstrdup(devices, GFP_KERNEL); + if (!s) + return false; + + for (p = s, ret = false; (tok = strsep(&p, ",")) != NULL; ) { + u16 val; + + if (negative && tok[0] == '!') + tok++; + else if ((negative && tok[0] != '!') || + (!negative && tok[0] == '!')) + continue; + + if (kstrtou16(tok, 16, &val) == 0 && val == device_id) { + ret = true; + break; + } + } + + kfree(s); + + return ret; +} + +static bool id_forced(u16 device_id) +{ + return device_id_in_list(device_id, xe_param_force_probe, false); +} + +static bool id_blocked(u16 device_id) +{ + return device_id_in_list(device_id, xe_param_force_probe, true); +} + +static const struct xe_subplatform_desc * +subplatform_get(const struct xe_device *xe, const struct xe_device_desc *desc) +{ + const struct xe_subplatform_desc *sp; + const u16 *id; + + for (sp = desc->subplatforms; sp && sp->subplatform; sp++) + for (id = sp->pciidlist; *id; id++) + if (*id == xe->info.devid) + return sp; + + return NULL; +} + +static void xe_pci_remove(struct pci_dev *pdev) +{ + struct xe_device *xe; + + xe = pci_get_drvdata(pdev); + if (!xe) /* driver load aborted, nothing to cleanup */ + return; + + xe_device_remove(xe); + pci_set_drvdata(pdev, NULL); +} + +static int xe_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + const struct xe_device_desc *desc = (void *)ent->driver_data; + const struct xe_subplatform_desc *spd; + struct xe_device *xe; + struct xe_gt *gt; + u8 id; + int err; + + if (desc->require_force_probe && !id_forced(pdev->device)) { + dev_info(&pdev->dev, + "Your graphics device %04x is not officially supported\n" + "by xe driver in this kernel version. To force Xe probe,\n" + "use xe.force_probe='%04x' and i915.force_probe='!%04x'\n" + "module parameters or CONFIG_DRM_XE_FORCE_PROBE='%04x' and\n" + "CONFIG_DRM_I915_FORCE_PROBE='!%04x' configuration options.\n", + pdev->device, pdev->device, pdev->device, + pdev->device, pdev->device); + return -ENODEV; + } + + if (id_blocked(pdev->device)) { + dev_info(&pdev->dev, "Probe blocked for device [%04x:%04x].\n", + pdev->vendor, pdev->device); + return -ENODEV; + } + + xe = xe_device_create(pdev, ent); + if (IS_ERR(xe)) + return PTR_ERR(xe); + + xe->info.graphics_verx100 = desc->graphics_ver * 100 + + desc->graphics_rel; + xe->info.media_verx100 = desc->media_ver * 100 + + desc->media_rel; + xe->info.is_dgfx = desc->is_dgfx; + xe->info.platform = desc->platform; + xe->info.dma_mask_size = desc->dma_mask_size; + xe->info.vram_flags = desc->vram_flags; + xe->info.tile_count = desc->max_tiles; + xe->info.vm_max_level = desc->vm_max_level; + xe->info.media_ver = desc->media_ver; + xe->info.supports_usm = desc->supports_usm; + xe->info.has_flat_ccs = desc->has_flat_ccs; + xe->info.has_4tile = desc->has_4tile; + + spd = subplatform_get(xe, desc); + xe->info.subplatform = spd ? spd->subplatform : XE_SUBPLATFORM_NONE; + xe->info.step = xe_step_get(xe); + + for (id = 0; id < xe->info.tile_count; ++id) { + gt = xe->gt + id; + gt->info.id = id; + gt->xe = xe; + + if (id == 0) { + gt->info.type = XE_GT_TYPE_MAIN; + gt->info.vram_id = id; + gt->info.engine_mask = desc->platform_engine_mask; + gt->mmio.adj_limit = 0; + gt->mmio.adj_offset = 0; + } else { + gt->info.type = desc->extra_gts[id - 1].type; + gt->info.vram_id = desc->extra_gts[id - 1].vram_id; + gt->info.engine_mask = + desc->extra_gts[id - 1].engine_mask; + gt->mmio.adj_limit = + desc->extra_gts[id - 1].mmio_adj_limit; + gt->mmio.adj_offset = + desc->extra_gts[id - 1].mmio_adj_offset; + } + } + + drm_dbg(&xe->drm, "%s %s %04x:%04x dgfx:%d gfx100:%d media100:%d dma_m_s:%d tc:%d", + desc->platform_name, spd ? spd->name : "", + xe->info.devid, xe->info.revid, + xe->info.is_dgfx, xe->info.graphics_verx100, + xe->info.media_verx100, + xe->info.dma_mask_size, xe->info.tile_count); + + drm_dbg(&xe->drm, "Stepping = (G:%s, M:%s, D:%s, B:%s)\n", + xe_step_name(xe->info.step.graphics), + xe_step_name(xe->info.step.media), + xe_step_name(xe->info.step.display), + xe_step_name(xe->info.step.basedie)); + + pci_set_drvdata(pdev, xe); + err = pci_enable_device(pdev); + if (err) { + drm_dev_put(&xe->drm); + return err; + } + + pci_set_master(pdev); + + if (pci_enable_msi(pdev) < 0) + drm_dbg(&xe->drm, "can't enable MSI"); + + err = xe_device_probe(xe); + if (err) { + pci_disable_device(pdev); + return err; + } + + xe_pm_runtime_init(xe); + + return 0; +} + +static void xe_pci_shutdown(struct pci_dev *pdev) +{ + xe_device_shutdown(pdev_to_xe_device(pdev)); +} + +#ifdef CONFIG_PM_SLEEP +static int xe_pci_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int err; + + err = xe_pm_suspend(pdev_to_xe_device(pdev)); + if (err) + return err; + + pci_save_state(pdev); + pci_disable_device(pdev); + + err = pci_set_power_state(pdev, PCI_D3hot); + if (err) + return err; + + return 0; +} + +static int xe_pci_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int err; + + err = pci_set_power_state(pdev, PCI_D0); + if (err) + return err; + + pci_restore_state(pdev); + + err = pci_enable_device(pdev); + if (err) + return err; + + pci_set_master(pdev); + + err = xe_pm_resume(pdev_to_xe_device(pdev)); + if (err) + return err; + + return 0; +} +#endif + +static int xe_pci_runtime_suspend(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + int err; + + err = xe_pm_runtime_suspend(xe); + if (err) + return err; + + pci_save_state(pdev); + + if (xe->d3cold_allowed) { + pci_disable_device(pdev); + pci_ignore_hotplug(pdev); + pci_set_power_state(pdev, PCI_D3cold); + } else { + pci_set_power_state(pdev, PCI_D3hot); + } + + return 0; +} + +static int xe_pci_runtime_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + int err; + + err = pci_set_power_state(pdev, PCI_D0); + if (err) + return err; + + pci_restore_state(pdev); + + if (xe->d3cold_allowed) { + err = pci_enable_device(pdev); + if (err) + return err; + + pci_set_master(pdev); + } + + return xe_pm_runtime_resume(xe); +} + +static int xe_pci_runtime_idle(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct xe_device *xe = pdev_to_xe_device(pdev); + + /* + * FIXME: d3cold should be allowed (true) if + * (IS_DGFX(xe) && !xe_device_mem_access_ongoing(xe)) + * however the change to the buddy allocator broke the + * xe_bo_restore_kernel when the pci device is disabled + */ + xe->d3cold_allowed = false; + + return 0; +} + +static const struct dev_pm_ops xe_pm_ops = { + .suspend = xe_pci_suspend, + .resume = xe_pci_resume, + .freeze = xe_pci_suspend, + .thaw = xe_pci_resume, + .poweroff = xe_pci_suspend, + .restore = xe_pci_resume, + .runtime_suspend = xe_pci_runtime_suspend, + .runtime_resume = xe_pci_runtime_resume, + .runtime_idle = xe_pci_runtime_idle, +}; + +static struct pci_driver xe_pci_driver = { + .name = DRIVER_NAME, + .id_table = pciidlist, + .probe = xe_pci_probe, + .remove = xe_pci_remove, + .shutdown = xe_pci_shutdown, + .driver.pm = &xe_pm_ops, +}; + +int xe_register_pci_driver(void) +{ + return pci_register_driver(&xe_pci_driver); +} + +void xe_unregister_pci_driver(void) +{ + pci_unregister_driver(&xe_pci_driver); +} + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +static int dev_to_xe_device_fn(struct device *dev, void *data) + +{ + struct drm_device *drm = dev_get_drvdata(dev); + int (*xe_fn)(struct xe_device *xe) = data; + int ret = 0; + int idx; + + if (drm_dev_enter(drm, &idx)) + ret = xe_fn(to_xe_device(dev_get_drvdata(dev))); + drm_dev_exit(idx); + + return ret; +} + +/** + * xe_call_for_each_device - Iterate over all devices this driver binds to + * @xe_fn: Function to call for each device. + * + * This function iterated over all devices this driver binds to, and calls + * @xe_fn: for each one of them. If the called function returns anything else + * than 0, iteration is stopped and the return value is returned by this + * function. Across each function call, drm_dev_enter() / drm_dev_exit() is + * called for the corresponding drm device. + * + * Return: Zero or the error code of a call to @xe_fn returning an error + * code. + */ +int xe_call_for_each_device(xe_device_fn xe_fn) +{ + return driver_for_each_device(&xe_pci_driver.driver, NULL, + xe_fn, dev_to_xe_device_fn); +} +#endif diff --git a/drivers/gpu/drm/xe/xe_pci.h b/drivers/gpu/drm/xe/xe_pci.h new file mode 100644 index 000000000000..9e3089549d5f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pci.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_PCI_H_ +#define _XE_PCI_H_ + +#include "tests/xe_test.h" + +int xe_register_pci_driver(void); +void xe_unregister_pci_driver(void); + +#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) +struct xe_device; + +typedef int (*xe_device_fn)(struct xe_device *); + +int xe_call_for_each_device(xe_device_fn xe_fn); +#endif +#endif diff --git a/drivers/gpu/drm/xe/xe_pcode.c b/drivers/gpu/drm/xe/xe_pcode.c new file mode 100644 index 000000000000..236159c8a6c0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pcode.c @@ -0,0 +1,296 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_pcode_api.h" +#include "xe_pcode.h" + +#include "xe_gt.h" +#include "xe_mmio.h" + +#include + +/** + * DOC: PCODE + * + * Xe PCODE is the component responsible for interfacing with the PCODE + * firmware. + * It shall provide a very simple ABI to other Xe components, but be the + * single and consolidated place that will communicate with PCODE. All read + * and write operations to PCODE will be internal and private to this component. + * + * What's next: + * - PCODE hw metrics + * - PCODE for display operations + */ + +static int pcode_mailbox_status(struct xe_gt *gt) +{ + u32 err; + static const struct pcode_err_decode err_decode[] = { + [PCODE_ILLEGAL_CMD] = {-ENXIO, "Illegal Command"}, + [PCODE_TIMEOUT] = {-ETIMEDOUT, "Timed out"}, + [PCODE_ILLEGAL_DATA] = {-EINVAL, "Illegal Data"}, + [PCODE_ILLEGAL_SUBCOMMAND] = {-ENXIO, "Illegal Subcommand"}, + [PCODE_LOCKED] = {-EBUSY, "PCODE Locked"}, + [PCODE_GT_RATIO_OUT_OF_RANGE] = {-EOVERFLOW, + "GT ratio out of range"}, + [PCODE_REJECTED] = {-EACCES, "PCODE Rejected"}, + [PCODE_ERROR_MASK] = {-EPROTO, "Unknown"}, + }; + + lockdep_assert_held(>->pcode.lock); + + err = xe_mmio_read32(gt, PCODE_MAILBOX.reg) & PCODE_ERROR_MASK; + if (err) { + drm_err(>_to_xe(gt)->drm, "PCODE Mailbox failed: %d %s", err, + err_decode[err].str ?: "Unknown"); + return err_decode[err].errno ?: -EPROTO; + } + + return 0; +} + +static bool pcode_mailbox_done(struct xe_gt *gt) +{ + lockdep_assert_held(>->pcode.lock); + return (xe_mmio_read32(gt, PCODE_MAILBOX.reg) & PCODE_READY) == 0; +} + +static int pcode_mailbox_rw(struct xe_gt *gt, u32 mbox, u32 *data0, u32 *data1, + unsigned int timeout, bool return_data, bool atomic) +{ + lockdep_assert_held(>->pcode.lock); + + if (!pcode_mailbox_done(gt)) + return -EAGAIN; + + xe_mmio_write32(gt, PCODE_DATA0.reg, *data0); + xe_mmio_write32(gt, PCODE_DATA1.reg, data1 ? *data1 : 0); + xe_mmio_write32(gt, PCODE_MAILBOX.reg, PCODE_READY | mbox); + + if (atomic) + _wait_for_atomic(pcode_mailbox_done(gt), timeout * 1000, 1); + else + wait_for(pcode_mailbox_done(gt), timeout); + + if (return_data) { + *data0 = xe_mmio_read32(gt, PCODE_DATA0.reg); + if (data1) + *data1 = xe_mmio_read32(gt, PCODE_DATA1.reg); + } + + return pcode_mailbox_status(gt); +} + +int xe_pcode_write_timeout(struct xe_gt *gt, u32 mbox, u32 data, int timeout) +{ + int err; + + mutex_lock(>->pcode.lock); + err = pcode_mailbox_rw(gt, mbox, &data, NULL, timeout, false, false); + mutex_unlock(>->pcode.lock); + + return err; +} + +int xe_pcode_read(struct xe_gt *gt, u32 mbox, u32 *val, u32 *val1) +{ + int err; + + mutex_lock(>->pcode.lock); + err = pcode_mailbox_rw(gt, mbox, val, val1, 1, true, false); + mutex_unlock(>->pcode.lock); + + return err; +} + +static bool xe_pcode_try_request(struct xe_gt *gt, u32 mbox, + u32 request, u32 reply_mask, u32 reply, + u32 *status, bool atomic) +{ + *status = pcode_mailbox_rw(gt, mbox, &request, NULL, 1, true, atomic); + + return (*status == 0) && ((request & reply_mask) == reply); +} + +/** + * xe_pcode_request - send PCODE request until acknowledgment + * @gt: gt + * @mbox: PCODE mailbox ID the request is targeted for + * @request: request ID + * @reply_mask: mask used to check for request acknowledgment + * @reply: value used to check for request acknowledgment + * @timeout_base_ms: timeout for polling with preemption enabled + * + * Keep resending the @request to @mbox until PCODE acknowledges it, PCODE + * reports an error or an overall timeout of @timeout_base_ms+50 ms expires. + * The request is acknowledged once the PCODE reply dword equals @reply after + * applying @reply_mask. Polling is first attempted with preemption enabled + * for @timeout_base_ms and if this times out for another 50 ms with + * preemption disabled. + * + * Returns 0 on success, %-ETIMEDOUT in case of a timeout, <0 in case of some + * other error as reported by PCODE. + */ +int xe_pcode_request(struct xe_gt *gt, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_base_ms) +{ + u32 status; + int ret; + bool atomic = false; + + mutex_lock(>->pcode.lock); + +#define COND \ + xe_pcode_try_request(gt, mbox, request, reply_mask, reply, &status, atomic) + + /* + * Prime the PCODE by doing a request first. Normally it guarantees + * that a subsequent request, at most @timeout_base_ms later, succeeds. + * _wait_for() doesn't guarantee when its passed condition is evaluated + * first, so send the first request explicitly. + */ + if (COND) { + ret = 0; + goto out; + } + ret = _wait_for(COND, timeout_base_ms * 1000, 10, 10); + if (!ret) + goto out; + + /* + * The above can time out if the number of requests was low (2 in the + * worst case) _and_ PCODE was busy for some reason even after a + * (queued) request and @timeout_base_ms delay. As a workaround retry + * the poll with preemption disabled to maximize the number of + * requests. Increase the timeout from @timeout_base_ms to 50ms to + * account for interrupts that could reduce the number of these + * requests, and for any quirks of the PCODE firmware that delays + * the request completion. + */ + drm_err(>_to_xe(gt)->drm, + "PCODE timeout, retrying with preemption disabled\n"); + drm_WARN_ON_ONCE(>_to_xe(gt)->drm, timeout_base_ms > 1); + preempt_disable(); + atomic = true; + ret = wait_for_atomic(COND, 50); + atomic = false; + preempt_enable(); + +out: + mutex_unlock(>->pcode.lock); + return status ? status : ret; +#undef COND +} +/** + * xe_pcode_init_min_freq_table - Initialize PCODE's QOS frequency table + * @gt: gt instance + * @min_gt_freq: Minimal (RPn) GT frequency in units of 50MHz. + * @max_gt_freq: Maximal (RP0) GT frequency in units of 50MHz. + * + * This function initialize PCODE's QOS frequency table for a proper minimal + * frequency/power steering decision, depending on the current requested GT + * frequency. For older platforms this was a more complete table including + * the IA freq. However for the latest platforms this table become a simple + * 1-1 Ring vs GT frequency. Even though, without setting it, PCODE might + * not take the right decisions for some memory frequencies and affect latency. + * + * It returns 0 on success, and -ERROR number on failure, -EINVAL if max + * frequency is higher then the minimal, and other errors directly translated + * from the PCODE Error returs: + * - -ENXIO: "Illegal Command" + * - -ETIMEDOUT: "Timed out" + * - -EINVAL: "Illegal Data" + * - -ENXIO, "Illegal Subcommand" + * - -EBUSY: "PCODE Locked" + * - -EOVERFLOW, "GT ratio out of range" + * - -EACCES, "PCODE Rejected" + * - -EPROTO, "Unknown" + */ +int xe_pcode_init_min_freq_table(struct xe_gt *gt, u32 min_gt_freq, + u32 max_gt_freq) +{ + int ret; + u32 freq; + + if (IS_DGFX(gt_to_xe(gt))) + return 0; + + if (max_gt_freq <= min_gt_freq) + return -EINVAL; + + mutex_lock(>->pcode.lock); + for (freq = min_gt_freq; freq <= max_gt_freq; freq++) { + u32 data = freq << PCODE_FREQ_RING_RATIO_SHIFT | freq; + + ret = pcode_mailbox_rw(gt, PCODE_WRITE_MIN_FREQ_TABLE, + &data, NULL, 1, false, false); + if (ret) + goto unlock; + } + +unlock: + mutex_unlock(>->pcode.lock); + return ret; +} + +static bool pcode_dgfx_status_complete(struct xe_gt *gt) +{ + u32 data = DGFX_GET_INIT_STATUS; + int status = pcode_mailbox_rw(gt, DGFX_PCODE_STATUS, + &data, NULL, 1, true, false); + + return status == 0 && + (data & DGFX_INIT_STATUS_COMPLETE) == DGFX_INIT_STATUS_COMPLETE; +} + +/** + * xe_pcode_init - Ensure PCODE is initialized + * @gt: gt instance + * + * This function ensures that PCODE is properly initialized. To be called during + * probe and resume paths. + * + * It returns 0 on success, and -error number on failure. + */ +int xe_pcode_init(struct xe_gt *gt) +{ + int timeout = 180000; /* 3 min */ + int ret; + + if (!IS_DGFX(gt_to_xe(gt))) + return 0; + + mutex_lock(>->pcode.lock); + ret = wait_for(pcode_dgfx_status_complete(gt), timeout); + mutex_unlock(>->pcode.lock); + + if (ret) + drm_err(>_to_xe(gt)->drm, + "PCODE initialization timedout after: %d min\n", + timeout / 60000); + + return ret; +} + +/** + * xe_pcode_probe - Prepare xe_pcode and also ensure PCODE is initialized. + * @gt: gt instance + * + * This function initializes the xe_pcode component, and when needed, it ensures + * that PCODE has properly performed its initialization and it is really ready + * to go. To be called once only during probe. + * + * It returns 0 on success, and -error number on failure. + */ +int xe_pcode_probe(struct xe_gt *gt) +{ + mutex_init(>->pcode.lock); + + if (!IS_DGFX(gt_to_xe(gt))) + return 0; + + return xe_pcode_init(gt); +} diff --git a/drivers/gpu/drm/xe/xe_pcode.h b/drivers/gpu/drm/xe/xe_pcode.h new file mode 100644 index 000000000000..3b4aa8c1a3ba --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pcode.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PCODE_H_ +#define _XE_PCODE_H_ + +#include +struct xe_gt; + +int xe_pcode_probe(struct xe_gt *gt); +int xe_pcode_init(struct xe_gt *gt); +int xe_pcode_init_min_freq_table(struct xe_gt *gt, u32 min_gt_freq, + u32 max_gt_freq); +int xe_pcode_read(struct xe_gt *gt, u32 mbox, u32 *val, u32 *val1); +int xe_pcode_write_timeout(struct xe_gt *gt, u32 mbox, u32 val, + int timeout_ms); +#define xe_pcode_write(gt, mbox, val) \ + xe_pcode_write_timeout(gt, mbox, val, 1) + +int xe_pcode_request(struct xe_gt *gt, u32 mbox, u32 request, + u32 reply_mask, u32 reply, int timeout_ms); + +#endif diff --git a/drivers/gpu/drm/xe/xe_pcode_api.h b/drivers/gpu/drm/xe/xe_pcode_api.h new file mode 100644 index 000000000000..0762c8a912c7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pcode_api.h @@ -0,0 +1,40 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +/* Internal to xe_pcode */ + +#define PCODE_MAILBOX _MMIO(0x138124) +#define PCODE_READY REG_BIT(31) +#define PCODE_MB_PARAM2 REG_GENMASK(23, 16) +#define PCODE_MB_PARAM1 REG_GENMASK(15, 8) +#define PCODE_MB_COMMAND REG_GENMASK(7, 0) +#define PCODE_ERROR_MASK 0xFF +#define PCODE_SUCCESS 0x0 +#define PCODE_ILLEGAL_CMD 0x1 +#define PCODE_TIMEOUT 0x2 +#define PCODE_ILLEGAL_DATA 0x3 +#define PCODE_ILLEGAL_SUBCOMMAND 0x4 +#define PCODE_LOCKED 0x6 +#define PCODE_GT_RATIO_OUT_OF_RANGE 0x10 +#define PCODE_REJECTED 0x11 + +#define PCODE_DATA0 _MMIO(0x138128) +#define PCODE_DATA1 _MMIO(0x13812C) + +/* Min Freq QOS Table */ +#define PCODE_WRITE_MIN_FREQ_TABLE 0x8 +#define PCODE_READ_MIN_FREQ_TABLE 0x9 +#define PCODE_FREQ_RING_RATIO_SHIFT 16 + +/* PCODE Init */ +#define DGFX_PCODE_STATUS 0x7E +#define DGFX_GET_INIT_STATUS 0x0 +#define DGFX_INIT_STATUS_COMPLETE 0x1 + +struct pcode_err_decode { + int errno; + const char *str; +}; + diff --git a/drivers/gpu/drm/xe/xe_platform_types.h b/drivers/gpu/drm/xe/xe_platform_types.h new file mode 100644 index 000000000000..72612c832e88 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_platform_types.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PLATFORM_INFO_TYPES_H_ +#define _XE_PLATFORM_INFO_TYPES_H_ + +/* Keep in gen based order, and chronological order within a gen */ +enum xe_platform { + XE_PLATFORM_UNINITIALIZED = 0, + /* gen12 */ + XE_TIGERLAKE, + XE_ROCKETLAKE, + XE_DG1, + XE_DG2, + XE_PVC, + XE_ALDERLAKE_S, + XE_ALDERLAKE_P, + XE_METEORLAKE, +}; + +enum xe_subplatform { + XE_SUBPLATFORM_UNINITIALIZED = 0, + XE_SUBPLATFORM_NONE, + XE_SUBPLATFORM_DG2_G10, + XE_SUBPLATFORM_DG2_G11, + XE_SUBPLATFORM_DG2_G12, + XE_SUBPLATFORM_ADLP_RPLU, +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c new file mode 100644 index 000000000000..fb0355530e7b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -0,0 +1,207 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include + +#include "xe_bo.h" +#include "xe_bo_evict.h" +#include "xe_device.h" +#include "xe_pm.h" +#include "xe_gt.h" +#include "xe_ggtt.h" +#include "xe_irq.h" +#include "xe_pcode.h" + +/** + * DOC: Xe Power Management + * + * Xe PM shall be guided by the simplicity. + * Use the simplest hook options whenever possible. + * Let's not reinvent the runtime_pm references and hooks. + * Shall have a clear separation of display and gt underneath this component. + * + * What's next: + * + * For now s2idle and s3 are only working in integrated devices. The next step + * is to iterate through all VRAM's BO backing them up into the system memory + * before allowing the system suspend. + * + * Also runtime_pm needs to be here from the beginning. + * + * RC6/RPS are also critical PM features. Let's start with GuCRC and GuC SLPC + * and no wait boost. Frequency optimizations should come on a next stage. + */ + +/** + * xe_pm_suspend - Helper for System suspend, i.e. S0->S3 / S0->S2idle + * @xe: xe device instance + * + * Return: 0 on success + */ +int xe_pm_suspend(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + int err; + + for_each_gt(gt, xe, id) + xe_gt_suspend_prepare(gt); + + /* FIXME: Super racey... */ + err = xe_bo_evict_all(xe); + if (err) + return err; + + for_each_gt(gt, xe, id) { + err = xe_gt_suspend(gt); + if (err) + return err; + } + + xe_irq_suspend(xe); + + return 0; +} + +/** + * xe_pm_resume - Helper for System resume S3->S0 / S2idle->S0 + * @xe: xe device instance + * + * Return: 0 on success + */ +int xe_pm_resume(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + int err; + + for_each_gt(gt, xe, id) { + err = xe_pcode_init(gt); + if (err) + return err; + } + + /* + * This only restores pinned memory which is the memory required for the + * GT(s) to resume. + */ + err = xe_bo_restore_kernel(xe); + if (err) + return err; + + xe_irq_resume(xe); + + for_each_gt(gt, xe, id) + xe_gt_resume(gt); + + err = xe_bo_restore_user(xe); + if (err) + return err; + + return 0; +} + +void xe_pm_runtime_init(struct xe_device *xe) +{ + struct device *dev = xe->drm.dev; + + pm_runtime_use_autosuspend(dev); + pm_runtime_set_autosuspend_delay(dev, 1000); + pm_runtime_set_active(dev); + pm_runtime_allow(dev); + pm_runtime_mark_last_busy(dev); + pm_runtime_put_autosuspend(dev); +} + +int xe_pm_runtime_suspend(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + int err; + + if (xe->d3cold_allowed) { + if (xe_device_mem_access_ongoing(xe)) + return -EBUSY; + + err = xe_bo_evict_all(xe); + if (err) + return err; + } + + for_each_gt(gt, xe, id) { + err = xe_gt_suspend(gt); + if (err) + return err; + } + + xe_irq_suspend(xe); + + return 0; +} + +int xe_pm_runtime_resume(struct xe_device *xe) +{ + struct xe_gt *gt; + u8 id; + int err; + + if (xe->d3cold_allowed) { + for_each_gt(gt, xe, id) { + err = xe_pcode_init(gt); + if (err) + return err; + } + + /* + * This only restores pinned memory which is the memory + * required for the GT(s) to resume. + */ + err = xe_bo_restore_kernel(xe); + if (err) + return err; + } + + xe_irq_resume(xe); + + for_each_gt(gt, xe, id) + xe_gt_resume(gt); + + if (xe->d3cold_allowed) { + err = xe_bo_restore_user(xe); + if (err) + return err; + } + + return 0; +} + +int xe_pm_runtime_get(struct xe_device *xe) +{ + return pm_runtime_get_sync(xe->drm.dev); +} + +int xe_pm_runtime_put(struct xe_device *xe) +{ + pm_runtime_mark_last_busy(xe->drm.dev); + return pm_runtime_put_autosuspend(xe->drm.dev); +} + +/* Return true if resume operation happened and usage count was increased */ +bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe) +{ + /* In case we are suspended we need to immediately wake up */ + if (pm_runtime_suspended(xe->drm.dev)) + return !pm_runtime_resume_and_get(xe->drm.dev); + + return false; +} + +int xe_pm_runtime_get_if_active(struct xe_device *xe) +{ + WARN_ON(pm_runtime_suspended(xe->drm.dev)); + return pm_runtime_get_if_active(xe->drm.dev, true); +} diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h new file mode 100644 index 000000000000..b8c5f9558e26 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pm.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PM_H_ +#define _XE_PM_H_ + +#include + +struct xe_device; + +int xe_pm_suspend(struct xe_device *xe); +int xe_pm_resume(struct xe_device *xe); + +void xe_pm_runtime_init(struct xe_device *xe); +int xe_pm_runtime_suspend(struct xe_device *xe); +int xe_pm_runtime_resume(struct xe_device *xe); +int xe_pm_runtime_get(struct xe_device *xe); +int xe_pm_runtime_put(struct xe_device *xe); +bool xe_pm_runtime_resume_if_suspended(struct xe_device *xe); +int xe_pm_runtime_get_if_active(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_preempt_fence.c b/drivers/gpu/drm/xe/xe_preempt_fence.c new file mode 100644 index 000000000000..6ab9ff442766 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_preempt_fence.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_engine.h" +#include "xe_preempt_fence.h" +#include "xe_vm.h" + +static void preempt_fence_work_func(struct work_struct *w) +{ + bool cookie = dma_fence_begin_signalling(); + struct xe_preempt_fence *pfence = + container_of(w, typeof(*pfence), preempt_work); + struct xe_engine *e = pfence->engine; + + if (pfence->error) + dma_fence_set_error(&pfence->base, pfence->error); + else + e->ops->suspend_wait(e); + + dma_fence_signal(&pfence->base); + dma_fence_end_signalling(cookie); + + queue_work(system_unbound_wq, &e->vm->preempt.rebind_work); + + xe_engine_put(e); +} + +static const char * +preempt_fence_get_driver_name(struct dma_fence *fence) +{ + return "xe"; +} + +static const char * +preempt_fence_get_timeline_name(struct dma_fence *fence) +{ + return "preempt"; +} + +static bool preempt_fence_enable_signaling(struct dma_fence *fence) +{ + struct xe_preempt_fence *pfence = + container_of(fence, typeof(*pfence), base); + struct xe_engine *e = pfence->engine; + + pfence->error = e->ops->suspend(e); + queue_work(system_unbound_wq, &pfence->preempt_work); + return true; +} + +static const struct dma_fence_ops preempt_fence_ops = { + .get_driver_name = preempt_fence_get_driver_name, + .get_timeline_name = preempt_fence_get_timeline_name, + .enable_signaling = preempt_fence_enable_signaling, +}; + +/** + * xe_preempt_fence_alloc() - Allocate a preempt fence with minimal + * initialization + * + * Allocate a preempt fence, and initialize its list head. + * If the preempt_fence allocated has been armed with + * xe_preempt_fence_arm(), it must be freed using dma_fence_put(). If not, + * it must be freed using xe_preempt_fence_free(). + * + * Return: A struct xe_preempt_fence pointer used for calling into + * xe_preempt_fence_arm() or xe_preempt_fence_free(). + * An error pointer on error. + */ +struct xe_preempt_fence *xe_preempt_fence_alloc(void) +{ + struct xe_preempt_fence *pfence; + + pfence = kmalloc(sizeof(*pfence), GFP_KERNEL); + if (!pfence) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&pfence->link); + INIT_WORK(&pfence->preempt_work, preempt_fence_work_func); + + return pfence; +} + +/** + * xe_preempt_fence_free() - Free a preempt fence allocated using + * xe_preempt_fence_alloc(). + * @pfence: pointer obtained from xe_preempt_fence_alloc(); + * + * Free a preempt fence that has not yet been armed. + */ +void xe_preempt_fence_free(struct xe_preempt_fence *pfence) +{ + list_del(&pfence->link); + kfree(pfence); +} + +/** + * xe_preempt_fence_arm() - Arm a preempt fence allocated using + * xe_preempt_fence_alloc(). + * @pfence: The struct xe_preempt_fence pointer returned from + * xe_preempt_fence_alloc(). + * @e: The struct xe_engine used for arming. + * @context: The dma-fence context used for arming. + * @seqno: The dma-fence seqno used for arming. + * + * Inserts the preempt fence into @context's timeline, takes @link off any + * list, and registers the struct xe_engine as the xe_engine to be preempted. + * + * Return: A pointer to a struct dma_fence embedded into the preempt fence. + * This function doesn't error. + */ +struct dma_fence * +xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_engine *e, + u64 context, u32 seqno) +{ + list_del_init(&pfence->link); + pfence->engine = xe_engine_get(e); + dma_fence_init(&pfence->base, &preempt_fence_ops, + &e->compute.lock, context, seqno); + + return &pfence->base; +} + +/** + * xe_preempt_fence_create() - Helper to create and arm a preempt fence. + * @e: The struct xe_engine used for arming. + * @context: The dma-fence context used for arming. + * @seqno: The dma-fence seqno used for arming. + * + * Allocates and inserts the preempt fence into @context's timeline, + * and registers @e as the struct xe_engine to be preempted. + * + * Return: A pointer to the resulting struct dma_fence on success. An error + * pointer on error. In particular if allocation fails it returns + * ERR_PTR(-ENOMEM); + */ +struct dma_fence * +xe_preempt_fence_create(struct xe_engine *e, + u64 context, u32 seqno) +{ + struct xe_preempt_fence *pfence; + + pfence = xe_preempt_fence_alloc(); + if (IS_ERR(pfence)) + return ERR_CAST(pfence); + + return xe_preempt_fence_arm(pfence, e, context, seqno); +} + +bool xe_fence_is_xe_preempt(const struct dma_fence *fence) +{ + return fence->ops == &preempt_fence_ops; +} diff --git a/drivers/gpu/drm/xe/xe_preempt_fence.h b/drivers/gpu/drm/xe/xe_preempt_fence.h new file mode 100644 index 000000000000..4f3966103203 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_preempt_fence.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PREEMPT_FENCE_H_ +#define _XE_PREEMPT_FENCE_H_ + +#include "xe_preempt_fence_types.h" + +struct list_head; + +struct dma_fence * +xe_preempt_fence_create(struct xe_engine *e, + u64 context, u32 seqno); + +struct xe_preempt_fence *xe_preempt_fence_alloc(void); + +void xe_preempt_fence_free(struct xe_preempt_fence *pfence); + +struct dma_fence * +xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_engine *e, + u64 context, u32 seqno); + +static inline struct xe_preempt_fence * +to_preempt_fence(struct dma_fence *fence) +{ + return container_of(fence, struct xe_preempt_fence, base); +} + +/** + * xe_preempt_fence_link() - Return a link used to keep unarmed preempt + * fences on a list. + * @pfence: Pointer to the preempt fence. + * + * The link is embedded in the struct xe_preempt_fence. Use + * link_to_preempt_fence() to convert back to the preempt fence. + * + * Return: A pointer to an embedded struct list_head. + */ +static inline struct list_head * +xe_preempt_fence_link(struct xe_preempt_fence *pfence) +{ + return &pfence->link; +} + +/** + * to_preempt_fence_from_link() - Convert back to a preempt fence pointer + * from a link obtained with xe_preempt_fence_link(). + * @link: The struct list_head obtained from xe_preempt_fence_link(). + * + * Return: A pointer to the embedding struct xe_preempt_fence. + */ +static inline struct xe_preempt_fence * +to_preempt_fence_from_link(struct list_head *link) +{ + return container_of(link, struct xe_preempt_fence, link); +} + +bool xe_fence_is_xe_preempt(const struct dma_fence *fence); +#endif diff --git a/drivers/gpu/drm/xe/xe_preempt_fence_types.h b/drivers/gpu/drm/xe/xe_preempt_fence_types.h new file mode 100644 index 000000000000..9d9efd8ff0ed --- /dev/null +++ b/drivers/gpu/drm/xe/xe_preempt_fence_types.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PREEMPT_FENCE_TYPES_H_ +#define _XE_PREEMPT_FENCE_TYPES_H_ + +#include +#include + +struct xe_engine; + +/** + * struct xe_preempt_fence - XE preempt fence + * + * A preemption fence which suspends the execution of an xe_engine on the + * hardware and triggers a callback once the xe_engine is complete. + */ +struct xe_preempt_fence { + /** @base: dma fence base */ + struct dma_fence base; + /** @link: link into list of pending preempt fences */ + struct list_head link; + /** @engine: xe engine for this preempt fence */ + struct xe_engine *engine; + /** @preempt_work: work struct which issues preemption */ + struct work_struct preempt_work; + /** @error: preempt fence is in error state */ + int error; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c new file mode 100644 index 000000000000..81193ddd0af7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -0,0 +1,1542 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_migrate.h" +#include "xe_pt.h" +#include "xe_pt_types.h" +#include "xe_pt_walk.h" +#include "xe_vm.h" +#include "xe_res_cursor.h" + +struct xe_pt_dir { + struct xe_pt pt; + /** @dir: Directory structure for the xe_pt_walk functionality */ + struct xe_ptw_dir dir; +}; + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) +#define xe_pt_set_addr(__xe_pt, __addr) ((__xe_pt)->addr = (__addr)) +#define xe_pt_addr(__xe_pt) ((__xe_pt)->addr) +#else +#define xe_pt_set_addr(__xe_pt, __addr) +#define xe_pt_addr(__xe_pt) 0ull +#endif + +static const u64 xe_normal_pt_shifts[] = {12, 21, 30, 39, 48}; +static const u64 xe_compact_pt_shifts[] = {16, 21, 30, 39, 48}; + +#define XE_PT_HIGHEST_LEVEL (ARRAY_SIZE(xe_normal_pt_shifts) - 1) + +static struct xe_pt_dir *as_xe_pt_dir(struct xe_pt *pt) +{ + return container_of(pt, struct xe_pt_dir, pt); +} + +static struct xe_pt *xe_pt_entry(struct xe_pt_dir *pt_dir, unsigned int index) +{ + return container_of(pt_dir->dir.entries[index], struct xe_pt, base); +} + +/** + * gen8_pde_encode() - Encode a page-table directory entry pointing to + * another page-table. + * @bo: The page-table bo of the page-table to point to. + * @bo_offset: Offset in the page-table bo to point to. + * @level: The cache level indicating the caching of @bo. + * + * TODO: Rename. + * + * Return: An encoded page directory entry. No errors. + */ +u64 gen8_pde_encode(struct xe_bo *bo, u64 bo_offset, + const enum xe_cache_level level) +{ + u64 pde; + bool is_lmem; + + pde = xe_bo_addr(bo, bo_offset, GEN8_PAGE_SIZE, &is_lmem); + pde |= GEN8_PAGE_PRESENT | GEN8_PAGE_RW; + + XE_WARN_ON(IS_DGFX(xe_bo_device(bo)) && !is_lmem); + + /* FIXME: I don't think the PPAT handling is correct for MTL */ + + if (level != XE_CACHE_NONE) + pde |= PPAT_CACHED_PDE; + else + pde |= PPAT_UNCACHED; + + return pde; +} + +static dma_addr_t vma_addr(struct xe_vma *vma, u64 offset, + size_t page_size, bool *is_lmem) +{ + if (xe_vma_is_userptr(vma)) { + struct xe_res_cursor cur; + u64 page; + + *is_lmem = false; + page = offset >> PAGE_SHIFT; + offset &= (PAGE_SIZE - 1); + + xe_res_first_sg(vma->userptr.sg, page << PAGE_SHIFT, page_size, + &cur); + return xe_res_dma(&cur) + offset; + } else { + return xe_bo_addr(vma->bo, offset, page_size, is_lmem); + } +} + +static u64 __gen8_pte_encode(u64 pte, enum xe_cache_level cache, u32 flags, + u32 pt_level) +{ + pte |= GEN8_PAGE_PRESENT | GEN8_PAGE_RW; + + if (unlikely(flags & PTE_READ_ONLY)) + pte &= ~GEN8_PAGE_RW; + + /* FIXME: I don't think the PPAT handling is correct for MTL */ + + switch (cache) { + case XE_CACHE_NONE: + pte |= PPAT_UNCACHED; + break; + case XE_CACHE_WT: + pte |= PPAT_DISPLAY_ELLC; + break; + default: + pte |= PPAT_CACHED; + break; + } + + if (pt_level == 1) + pte |= GEN8_PDE_PS_2M; + else if (pt_level == 2) + pte |= GEN8_PDPE_PS_1G; + + /* XXX: Does hw support 1 GiB pages? */ + XE_BUG_ON(pt_level > 2); + + return pte; +} + +/** + * gen8_pte_encode() - Encode a page-table entry pointing to memory. + * @vma: The vma representing the memory to point to. + * @bo: If @vma is NULL, representing the memory to point to. + * @offset: The offset into @vma or @bo. + * @cache: The cache level indicating + * @flags: Currently only supports PTE_READ_ONLY for read-only access. + * @pt_level: The page-table level of the page-table into which the entry + * is to be inserted. + * + * TODO: Rename. + * + * Return: An encoded page-table entry. No errors. + */ +u64 gen8_pte_encode(struct xe_vma *vma, struct xe_bo *bo, + u64 offset, enum xe_cache_level cache, + u32 flags, u32 pt_level) +{ + u64 pte; + bool is_vram; + + if (vma) + pte = vma_addr(vma, offset, GEN8_PAGE_SIZE, &is_vram); + else + pte = xe_bo_addr(bo, offset, GEN8_PAGE_SIZE, &is_vram); + + if (is_vram) { + pte |= GEN12_PPGTT_PTE_LM; + if (vma && vma->use_atomic_access_pte_bit) + pte |= GEN12_USM_PPGTT_PTE_AE; + } + + return __gen8_pte_encode(pte, cache, flags, pt_level); +} + +static u64 __xe_pt_empty_pte(struct xe_gt *gt, struct xe_vm *vm, + unsigned int level) +{ + u8 id = gt->info.id; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + + if (!vm->scratch_bo[id]) + return 0; + + if (level == 0) { + u64 empty = gen8_pte_encode(NULL, vm->scratch_bo[id], 0, + XE_CACHE_WB, 0, 0); + if (vm->flags & XE_VM_FLAGS_64K) + empty |= GEN12_PTE_PS64; + + return empty; + } else { + return gen8_pde_encode(vm->scratch_pt[id][level - 1]->bo, 0, + XE_CACHE_WB); + } +} + +/** + * xe_pt_create() - Create a page-table. + * @vm: The vm to create for. + * @gt: The gt to create for. + * @level: The page-table level. + * + * Allocate and initialize a single struct xe_pt metadata structure. Also + * create the corresponding page-table bo, but don't initialize it. If the + * level is grater than zero, then it's assumed to be a directory page- + * table and the directory structure is also allocated and initialized to + * NULL pointers. + * + * Return: A valid struct xe_pt pointer on success, Pointer error code on + * error. + */ +struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_gt *gt, + unsigned int level) +{ + struct xe_pt *pt; + struct xe_bo *bo; + size_t size; + int err; + + size = !level ? sizeof(struct xe_pt) : sizeof(struct xe_pt_dir) + + GEN8_PDES * sizeof(struct xe_ptw *); + pt = kzalloc(size, GFP_KERNEL); + if (!pt) + return ERR_PTR(-ENOMEM); + + bo = xe_bo_create_pin_map(vm->xe, gt, vm, SZ_4K, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT | + XE_BO_CREATE_PINNED_BIT); + if (IS_ERR(bo)) { + err = PTR_ERR(bo); + goto err_kfree; + } + pt->bo = bo; + pt->level = level; + pt->base.dir = level ? &as_xe_pt_dir(pt)->dir : NULL; + + XE_BUG_ON(level > XE_VM_MAX_LEVEL); + + return pt; + +err_kfree: + kfree(pt); + return ERR_PTR(err); +} + +/** + * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero + * entries. + * @gt: The gt the scratch pagetable of which to use. + * @vm: The vm we populate for. + * @pt: The pagetable the bo of which to initialize. + * + * Populate the page-table bo of @pt with entries pointing into the gt's + * scratch page-table tree if any. Otherwise populate with zeros. + */ +void xe_pt_populate_empty(struct xe_gt *gt, struct xe_vm *vm, + struct xe_pt *pt) +{ + struct iosys_map *map = &pt->bo->vmap; + u64 empty; + int i; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + + if (!vm->scratch_bo[gt->info.id]) { + /* + * FIXME: Some memory is allocated already allocated to zero? + * Find out which memory that is and avoid this memset... + */ + xe_map_memset(vm->xe, map, 0, 0, SZ_4K); + } else { + empty = __xe_pt_empty_pte(gt, vm, pt->level); + for (i = 0; i < GEN8_PDES; i++) + xe_pt_write(vm->xe, map, i, empty); + } +} + +/** + * xe_pt_shift() - Return the ilog2 value of the size of the address range of + * a page-table at a certain level. + * @level: The level. + * + * Return: The ilog2 value of the size of the address range of a page-table + * at level @level. + */ +unsigned int xe_pt_shift(unsigned int level) +{ + return GEN8_PTE_SHIFT + GEN8_PDE_SHIFT * level; +} + +/** + * xe_pt_destroy() - Destroy a page-table tree. + * @pt: The root of the page-table tree to destroy. + * @flags: vm flags. Currently unused. + * @deferred: List head of lockless list for deferred putting. NULL for + * immediate putting. + * + * Puts the page-table bo, recursively calls xe_pt_destroy on all children + * and finally frees @pt. TODO: Can we remove the @flags argument? + */ +void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) +{ + int i; + + if (!pt) + return; + + XE_BUG_ON(!list_empty(&pt->bo->vmas)); + xe_bo_unpin(pt->bo); + xe_bo_put_deferred(pt->bo, deferred); + + if (pt->level > 0 && pt->num_live) { + struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); + + for (i = 0; i < GEN8_PDES; i++) { + if (xe_pt_entry(pt_dir, i)) + xe_pt_destroy(xe_pt_entry(pt_dir, i), flags, + deferred); + } + } + kfree(pt); +} + +/** + * xe_pt_create_scratch() - Setup a scratch memory pagetable tree for the + * given gt and vm. + * @xe: xe device. + * @gt: gt to set up for. + * @vm: vm to set up for. + * + * Sets up a pagetable tree with one page-table per level and a single + * leaf bo. All pagetable entries point to the single page-table or, + * for L0, the single bo one level below. + * + * Return: 0 on success, negative error code on error. + */ +int xe_pt_create_scratch(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm) +{ + u8 id = gt->info.id; + int i; + + vm->scratch_bo[id] = xe_bo_create(xe, gt, vm, SZ_4K, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT | + XE_BO_CREATE_PINNED_BIT); + if (IS_ERR(vm->scratch_bo[id])) + return PTR_ERR(vm->scratch_bo[id]); + xe_bo_pin(vm->scratch_bo[id]); + + for (i = 0; i < vm->pt_root[id]->level; i++) { + vm->scratch_pt[id][i] = xe_pt_create(vm, gt, i); + if (IS_ERR(vm->scratch_pt[id][i])) + return PTR_ERR(vm->scratch_pt[id][i]); + + xe_pt_populate_empty(gt, vm, vm->scratch_pt[id][i]); + } + + return 0; +} + +/** + * DOC: Pagetable building + * + * Below we use the term "page-table" for both page-directories, containing + * pointers to lower level page-directories or page-tables, and level 0 + * page-tables that contain only page-table-entries pointing to memory pages. + * + * When inserting an address range in an already existing page-table tree + * there will typically be a set of page-tables that are shared with other + * address ranges, and a set that are private to this address range. + * The set of shared page-tables can be at most two per level, + * and those can't be updated immediately because the entries of those + * page-tables may still be in use by the gpu for other mappings. Therefore + * when inserting entries into those, we instead stage those insertions by + * adding insertion data into struct xe_vm_pgtable_update structures. This + * data, (subtrees for the cpu and page-table-entries for the gpu) is then + * added in a separate commit step. CPU-data is committed while still under the + * vm lock, the object lock and for userptr, the notifier lock in read mode. + * The GPU async data is committed either by the GPU or CPU after fulfilling + * relevant dependencies. + * For non-shared page-tables (and, in fact, for shared ones that aren't + * existing at the time of staging), we add the data in-place without the + * special update structures. This private part of the page-table tree will + * remain disconnected from the vm page-table tree until data is committed to + * the shared page tables of the vm tree in the commit phase. + */ + +struct xe_pt_update { + /** @update: The update structure we're building for this parent. */ + struct xe_vm_pgtable_update *update; + /** @parent: The parent. Used to detect a parent change. */ + struct xe_pt *parent; + /** @preexisting: Whether the parent was pre-existing or allocated */ + bool preexisting; +}; + +struct xe_pt_stage_bind_walk { + /** base: The base class. */ + struct xe_pt_walk base; + + /* Input parameters for the walk */ + /** @vm: The vm we're building for. */ + struct xe_vm *vm; + /** @gt: The gt we're building for. */ + struct xe_gt *gt; + /** @cache: Desired cache level for the ptes */ + enum xe_cache_level cache; + /** @default_pte: PTE flag only template. No address is associated */ + u64 default_pte; + /** @dma_offset: DMA offset to add to the PTE. */ + u64 dma_offset; + /** + * @needs_64k: This address range enforces 64K alignment and + * granularity. + */ + bool needs_64K; + /** + * @pte_flags: Flags determining PTE setup. These are not flags + * encoded directly in the PTE. See @default_pte for those. + */ + u32 pte_flags; + + /* Also input, but is updated during the walk*/ + /** @curs: The DMA address cursor. */ + struct xe_res_cursor *curs; + /** @va_curs_start: The Virtual address coresponding to @curs->start */ + u64 va_curs_start; + + /* Output */ + struct xe_walk_update { + /** @wupd.entries: Caller provided storage. */ + struct xe_vm_pgtable_update *entries; + /** @wupd.num_used_entries: Number of update @entries used. */ + unsigned int num_used_entries; + /** @wupd.updates: Tracks the update entry at a given level */ + struct xe_pt_update updates[XE_VM_MAX_LEVEL + 1]; + } wupd; + + /* Walk state */ + /** + * @l0_end_addr: The end address of the current l0 leaf. Used for + * 64K granularity detection. + */ + u64 l0_end_addr; + /** @addr_64K: The start address of the current 64K chunk. */ + u64 addr_64K; + /** @found_64: Whether @add_64K actually points to a 64K chunk. */ + bool found_64K; +}; + +static int +xe_pt_new_shared(struct xe_walk_update *wupd, struct xe_pt *parent, + pgoff_t offset, bool alloc_entries) +{ + struct xe_pt_update *upd = &wupd->updates[parent->level]; + struct xe_vm_pgtable_update *entry; + + /* + * For *each level*, we could only have one active + * struct xt_pt_update at any one time. Once we move on to a + * new parent and page-directory, the old one is complete, and + * updates are either already stored in the build tree or in + * @wupd->entries + */ + if (likely(upd->parent == parent)) + return 0; + + upd->parent = parent; + upd->preexisting = true; + + if (wupd->num_used_entries == XE_VM_MAX_LEVEL * 2 + 1) + return -EINVAL; + + entry = wupd->entries + wupd->num_used_entries++; + upd->update = entry; + entry->ofs = offset; + entry->pt_bo = parent->bo; + entry->pt = parent; + entry->flags = 0; + entry->qwords = 0; + + if (alloc_entries) { + entry->pt_entries = kmalloc_array(GEN8_PDES, + sizeof(*entry->pt_entries), + GFP_KERNEL); + if (!entry->pt_entries) + return -ENOMEM; + } + + return 0; +} + +/* + * NOTE: This is a very frequently called function so we allow ourselves + * to annotate (using branch prediction hints) the fastpath of updating a + * non-pre-existing pagetable with leaf ptes. + */ +static int +xe_pt_insert_entry(struct xe_pt_stage_bind_walk *xe_walk, struct xe_pt *parent, + pgoff_t offset, struct xe_pt *xe_child, u64 pte) +{ + struct xe_pt_update *upd = &xe_walk->wupd.updates[parent->level]; + struct xe_pt_update *child_upd = xe_child ? + &xe_walk->wupd.updates[xe_child->level] : NULL; + int ret; + + ret = xe_pt_new_shared(&xe_walk->wupd, parent, offset, true); + if (unlikely(ret)) + return ret; + + /* + * Register this new pagetable so that it won't be recognized as + * a shared pagetable by a subsequent insertion. + */ + if (unlikely(child_upd)) { + child_upd->update = NULL; + child_upd->parent = xe_child; + child_upd->preexisting = false; + } + + if (likely(!upd->preexisting)) { + /* Continue building a non-connected subtree. */ + struct iosys_map *map = &parent->bo->vmap; + + if (unlikely(xe_child)) + parent->base.dir->entries[offset] = &xe_child->base; + + xe_pt_write(xe_walk->vm->xe, map, offset, pte); + parent->num_live++; + } else { + /* Shared pt. Stage update. */ + unsigned int idx; + struct xe_vm_pgtable_update *entry = upd->update; + + idx = offset - entry->ofs; + entry->pt_entries[idx].pt = xe_child; + entry->pt_entries[idx].pte = pte; + entry->qwords++; + } + + return 0; +} + +static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, + struct xe_pt_stage_bind_walk *xe_walk) +{ + u64 size, dma; + + /* Does the virtual range requested cover a huge pte? */ + if (!xe_pt_covers(addr, next, level, &xe_walk->base)) + return false; + + /* Does the DMA segment cover the whole pte? */ + if (next - xe_walk->va_curs_start > xe_walk->curs->size) + return false; + + /* Is the DMA address huge PTE size aligned? */ + size = next - addr; + dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs); + + return IS_ALIGNED(dma, size); +} + +/* + * Scan the requested mapping to check whether it can be done entirely + * with 64K PTEs. + */ +static bool +xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) +{ + struct xe_res_cursor curs = *xe_walk->curs; + + if (!IS_ALIGNED(addr, SZ_64K)) + return false; + + if (next > xe_walk->l0_end_addr) + return false; + + xe_res_next(&curs, addr - xe_walk->va_curs_start); + for (; addr < next; addr += SZ_64K) { + if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K) + return false; + + xe_res_next(&curs, SZ_64K); + } + + return addr == next; +} + +/* + * For non-compact "normal" 4K level-0 pagetables, we want to try to group + * addresses together in 64K-contigous regions to add a 64K TLB hint for the + * device to the PTE. + * This function determines whether the address is part of such a + * segment. For VRAM in normal pagetables, this is strictly necessary on + * some devices. + */ +static bool +xe_pt_is_pte_ps64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) +{ + /* Address is within an already found 64k region */ + if (xe_walk->found_64K && addr - xe_walk->addr_64K < SZ_64K) + return true; + + xe_walk->found_64K = xe_pt_scan_64K(addr, addr + SZ_64K, xe_walk); + xe_walk->addr_64K = addr; + + return xe_walk->found_64K; +} + +static int +xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt_stage_bind_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base); + struct xe_pt *xe_child; + bool covers; + int ret = 0; + u64 pte; + + /* Is this a leaf entry ?*/ + if (level == 0 || xe_pt_hugepte_possible(addr, next, level, xe_walk)) { + struct xe_res_cursor *curs = xe_walk->curs; + + XE_WARN_ON(xe_walk->va_curs_start != addr); + + pte = __gen8_pte_encode(xe_res_dma(curs) + xe_walk->dma_offset, + xe_walk->cache, xe_walk->pte_flags, + level); + pte |= xe_walk->default_pte; + + /* + * Set the GEN12_PTE_PS64 hint if possible, otherwise if + * this device *requires* 64K PTE size for VRAM, fail. + */ + if (level == 0 && !xe_parent->is_compact) { + if (xe_pt_is_pte_ps64K(addr, next, xe_walk)) + pte |= GEN12_PTE_PS64; + else if (XE_WARN_ON(xe_walk->needs_64K)) + return -EINVAL; + } + + ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, NULL, pte); + if (unlikely(ret)) + return ret; + + xe_res_next(curs, next - addr); + xe_walk->va_curs_start = next; + *action = ACTION_CONTINUE; + + return ret; + } + + /* + * Descending to lower level. Determine if we need to allocate a + * new page table or -directory, which we do if there is no + * previous one or there is one we can completely replace. + */ + if (level == 1) { + walk->shifts = xe_normal_pt_shifts; + xe_walk->l0_end_addr = next; + } + + covers = xe_pt_covers(addr, next, level, &xe_walk->base); + if (covers || !*child) { + u64 flags = 0; + + xe_child = xe_pt_create(xe_walk->vm, xe_walk->gt, level - 1); + if (IS_ERR(xe_child)) + return PTR_ERR(xe_child); + + xe_pt_set_addr(xe_child, + round_down(addr, 1ull << walk->shifts[level])); + + if (!covers) + xe_pt_populate_empty(xe_walk->gt, xe_walk->vm, xe_child); + + *child = &xe_child->base; + + /* + * Prefer the compact pagetable layout for L0 if possible. + * TODO: Suballocate the pt bo to avoid wasting a lot of + * memory. + */ + if (GRAPHICS_VERx100(xe_walk->gt->xe) >= 1250 && level == 1 && + covers && xe_pt_scan_64K(addr, next, xe_walk)) { + walk->shifts = xe_compact_pt_shifts; + flags |= GEN12_PDE_64K; + xe_child->is_compact = true; + } + + pte = gen8_pde_encode(xe_child->bo, 0, xe_walk->cache) | flags; + ret = xe_pt_insert_entry(xe_walk, xe_parent, offset, xe_child, + pte); + } + + *action = ACTION_SUBTREE; + return ret; +} + +static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = { + .pt_entry = xe_pt_stage_bind_entry, +}; + +/** + * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address + * range. + * @gt: The gt we're building for. + * @vma: The vma indicating the address range. + * @entries: Storage for the update entries used for connecting the tree to + * the main tree at commit time. + * @num_entries: On output contains the number of @entries used. + * + * This function builds a disconnected page-table tree for a given address + * range. The tree is connected to the main vm tree for the gpu using + * xe_migrate_update_pgtables() and for the cpu using xe_pt_commit_bind(). + * The function builds xe_vm_pgtable_update structures for already existing + * shared page-tables, and non-existing shared and non-shared page-tables + * are built and populated directly. + * + * Return 0 on success, negative error code on error. + */ +static int +xe_pt_stage_bind(struct xe_gt *gt, struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, u32 *num_entries) +{ + struct xe_bo *bo = vma->bo; + bool is_vram = !xe_vma_is_userptr(vma) && bo && xe_bo_is_vram(bo); + struct xe_res_cursor curs; + struct xe_pt_stage_bind_walk xe_walk = { + .base = { + .ops = &xe_pt_stage_bind_ops, + .shifts = xe_normal_pt_shifts, + .max_level = XE_PT_HIGHEST_LEVEL, + }, + .vm = vma->vm, + .gt = gt, + .curs = &curs, + .va_curs_start = vma->start, + .pte_flags = vma->pte_flags, + .wupd.entries = entries, + .needs_64K = (vma->vm->flags & XE_VM_FLAGS_64K) && is_vram, + }; + struct xe_pt *pt = vma->vm->pt_root[gt->info.id]; + int ret; + + if (is_vram) { + xe_walk.default_pte = GEN12_PPGTT_PTE_LM; + if (vma && vma->use_atomic_access_pte_bit) + xe_walk.default_pte |= GEN12_USM_PPGTT_PTE_AE; + xe_walk.dma_offset = gt->mem.vram.io_start - + gt_to_xe(gt)->mem.vram.io_start; + xe_walk.cache = XE_CACHE_WB; + } else { + if (!xe_vma_is_userptr(vma) && bo->flags & XE_BO_SCANOUT_BIT) + xe_walk.cache = XE_CACHE_WT; + else + xe_walk.cache = XE_CACHE_WB; + } + + xe_bo_assert_held(bo); + if (xe_vma_is_userptr(vma)) + xe_res_first_sg(vma->userptr.sg, 0, vma->end - vma->start + 1, + &curs); + else if (xe_bo_is_vram(bo)) + xe_res_first(bo->ttm.resource, vma->bo_offset, + vma->end - vma->start + 1, &curs); + else + xe_res_first_sg(xe_bo_get_sg(bo), vma->bo_offset, + vma->end - vma->start + 1, &curs); + + ret = xe_pt_walk_range(&pt->base, pt->level, vma->start, vma->end + 1, + &xe_walk.base); + + *num_entries = xe_walk.wupd.num_used_entries; + return ret; +} + +/** + * xe_pt_nonshared_offsets() - Determine the non-shared entry offsets of a + * shared pagetable. + * @addr: The start address within the non-shared pagetable. + * @end: The end address within the non-shared pagetable. + * @level: The level of the non-shared pagetable. + * @walk: Walk info. The function adjusts the walk action. + * @action: next action to perform (see enum page_walk_action) + * @offset: Ignored on input, First non-shared entry on output. + * @end_offset: Ignored on input, Last non-shared entry + 1 on output. + * + * A non-shared page-table has some entries that belong to the address range + * and others that don't. This function determines the entries that belong + * fully to the address range. Depending on level, some entries may + * partially belong to the address range (that can't happen at level 0). + * The function detects that and adjust those offsets to not include those + * partial entries. Iff it does detect partial entries, we know that there must + * be shared page tables also at lower levels, so it adjusts the walk action + * accordingly. + * + * Return: true if there were non-shared entries, false otherwise. + */ +static bool xe_pt_nonshared_offsets(u64 addr, u64 end, unsigned int level, + struct xe_pt_walk *walk, + enum page_walk_action *action, + pgoff_t *offset, pgoff_t *end_offset) +{ + u64 size = 1ull << walk->shifts[level]; + + *offset = xe_pt_offset(addr, level, walk); + *end_offset = xe_pt_num_entries(addr, end, level, walk) + *offset; + + if (!level) + return true; + + /* + * If addr or next are not size aligned, there are shared pts at lower + * level, so in that case traverse down the subtree + */ + *action = ACTION_CONTINUE; + if (!IS_ALIGNED(addr, size)) { + *action = ACTION_SUBTREE; + (*offset)++; + } + + if (!IS_ALIGNED(end, size)) { + *action = ACTION_SUBTREE; + (*end_offset)--; + } + + return *end_offset > *offset; +} + +struct xe_pt_zap_ptes_walk { + /** @base: The walk base-class */ + struct xe_pt_walk base; + + /* Input parameters for the walk */ + /** @gt: The gt we're building for */ + struct xe_gt *gt; + + /* Output */ + /** @needs_invalidate: Whether we need to invalidate TLB*/ + bool needs_invalidate; +}; + +static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt_zap_ptes_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); + pgoff_t end_offset; + + XE_BUG_ON(!*child); + XE_BUG_ON(!level && xe_child->is_compact); + + /* + * Note that we're called from an entry callback, and we're dealing + * with the child of that entry rather than the parent, so need to + * adjust level down. + */ + if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset, + &end_offset)) { + xe_map_memset(gt_to_xe(xe_walk->gt), &xe_child->bo->vmap, + offset * sizeof(u64), 0, + (end_offset - offset) * sizeof(u64)); + xe_walk->needs_invalidate = true; + } + + return 0; +} + +static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = { + .pt_entry = xe_pt_zap_ptes_entry, +}; + +/** + * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range + * @gt: The gt we're zapping for. + * @vma: GPU VMA detailing address range. + * + * Eviction and Userptr invalidation needs to be able to zap the + * gpu ptes of a given address range in pagefaulting mode. + * In order to be able to do that, that function needs access to the shared + * page-table entrieaso it can either clear the leaf PTEs or + * clear the pointers to lower-level page-tables. The caller is required + * to hold the necessary locks to ensure neither the page-table connectivity + * nor the page-table entries of the range is updated from under us. + * + * Return: Whether ptes were actually updated and a TLB invalidation is + * required. + */ +bool xe_pt_zap_ptes(struct xe_gt *gt, struct xe_vma *vma) +{ + struct xe_pt_zap_ptes_walk xe_walk = { + .base = { + .ops = &xe_pt_zap_ptes_ops, + .shifts = xe_normal_pt_shifts, + .max_level = XE_PT_HIGHEST_LEVEL, + }, + .gt = gt, + }; + struct xe_pt *pt = vma->vm->pt_root[gt->info.id]; + + if (!(vma->gt_present & BIT(gt->info.id))) + return false; + + (void)xe_pt_walk_shared(&pt->base, pt->level, vma->start, vma->end + 1, + &xe_walk.base); + + return xe_walk.needs_invalidate; +} + +static void +xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_gt *gt, + struct iosys_map *map, void *data, + u32 qword_ofs, u32 num_qwords, + const struct xe_vm_pgtable_update *update) +{ + struct xe_pt_entry *ptes = update->pt_entries; + u64 *ptr = data; + u32 i; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + + for (i = 0; i < num_qwords; i++) { + if (map) + xe_map_wr(gt_to_xe(gt), map, (qword_ofs + i) * + sizeof(u64), u64, ptes[i].pte); + else + ptr[i] = ptes[i].pte; + } +} + +static void xe_pt_abort_bind(struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, + u32 num_entries) +{ + u32 i, j; + + for (i = 0; i < num_entries; i++) { + if (!entries[i].pt_entries) + continue; + + for (j = 0; j < entries[i].qwords; j++) + xe_pt_destroy(entries[i].pt_entries[j].pt, vma->vm->flags, NULL); + kfree(entries[i].pt_entries); + } +} + +static void xe_pt_commit_locks_assert(struct xe_vma *vma) +{ + struct xe_vm *vm = vma->vm; + + lockdep_assert_held(&vm->lock); + + if (xe_vma_is_userptr(vma)) + lockdep_assert_held_read(&vm->userptr.notifier_lock); + else + dma_resv_assert_held(vma->bo->ttm.base.resv); + + dma_resv_assert_held(&vm->resv); +} + +static void xe_pt_commit_bind(struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, + u32 num_entries, bool rebind, + struct llist_head *deferred) +{ + u32 i, j; + + xe_pt_commit_locks_assert(vma); + + for (i = 0; i < num_entries; i++) { + struct xe_pt *pt = entries[i].pt; + struct xe_pt_dir *pt_dir; + + if (!rebind) + pt->num_live += entries[i].qwords; + + if (!pt->level) { + kfree(entries[i].pt_entries); + continue; + } + + pt_dir = as_xe_pt_dir(pt); + for (j = 0; j < entries[i].qwords; j++) { + u32 j_ = j + entries[i].ofs; + struct xe_pt *newpte = entries[i].pt_entries[j].pt; + + if (xe_pt_entry(pt_dir, j_)) + xe_pt_destroy(xe_pt_entry(pt_dir, j_), + vma->vm->flags, deferred); + + pt_dir->dir.entries[j_] = &newpte->base; + } + kfree(entries[i].pt_entries); + } +} + +static int +xe_pt_prepare_bind(struct xe_gt *gt, struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, u32 *num_entries, + bool rebind) +{ + int err; + + *num_entries = 0; + err = xe_pt_stage_bind(gt, vma, entries, num_entries); + if (!err) + BUG_ON(!*num_entries); + else /* abort! */ + xe_pt_abort_bind(vma, entries, *num_entries); + + return err; +} + +static void xe_vm_dbg_print_entries(struct xe_device *xe, + const struct xe_vm_pgtable_update *entries, + unsigned int num_entries) +#if (IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM)) +{ + unsigned int i; + + vm_dbg(&xe->drm, "%u entries to update\n", num_entries); + for (i = 0; i < num_entries; i++) { + const struct xe_vm_pgtable_update *entry = &entries[i]; + struct xe_pt *xe_pt = entry->pt; + u64 page_size = 1ull << xe_pt_shift(xe_pt->level); + u64 end; + u64 start; + + XE_BUG_ON(entry->pt->is_compact); + start = entry->ofs * page_size; + end = start + page_size * entry->qwords; + vm_dbg(&xe->drm, + "\t%u: Update level %u at (%u + %u) [%llx...%llx) f:%x\n", + i, xe_pt->level, entry->ofs, entry->qwords, + xe_pt_addr(xe_pt) + start, xe_pt_addr(xe_pt) + end, 0); + } +} +#else +{} +#endif + +#ifdef CONFIG_DRM_XE_USERPTR_INVAL_INJECT + +static int xe_pt_userptr_inject_eagain(struct xe_vma *vma) +{ + u32 divisor = vma->userptr.divisor ? vma->userptr.divisor : 2; + static u32 count; + + if (count++ % divisor == divisor - 1) { + struct xe_vm *vm = vma->vm; + + vma->userptr.divisor = divisor << 1; + spin_lock(&vm->userptr.invalidated_lock); + list_move_tail(&vma->userptr.invalidate_link, + &vm->userptr.invalidated); + spin_unlock(&vm->userptr.invalidated_lock); + return true; + } + + return false; +} + +#else + +static bool xe_pt_userptr_inject_eagain(struct xe_vma *vma) +{ + return false; +} + +#endif + +/** + * struct xe_pt_migrate_pt_update - Callback argument for pre-commit callbacks + * @base: Base we derive from. + * @bind: Whether this is a bind or an unbind operation. A bind operation + * makes the pre-commit callback error with -EAGAIN if it detects a + * pending invalidation. + * @locked: Whether the pre-commit callback locked the userptr notifier lock + * and it needs unlocking. + */ +struct xe_pt_migrate_pt_update { + struct xe_migrate_pt_update base; + bool bind; + bool locked; +}; + +static int xe_pt_userptr_pre_commit(struct xe_migrate_pt_update *pt_update) +{ + struct xe_pt_migrate_pt_update *userptr_update = + container_of(pt_update, typeof(*userptr_update), base); + struct xe_vma *vma = pt_update->vma; + unsigned long notifier_seq = vma->userptr.notifier_seq; + struct xe_vm *vm = vma->vm; + + userptr_update->locked = false; + + /* + * Wait until nobody is running the invalidation notifier, and + * since we're exiting the loop holding the notifier lock, + * nobody can proceed invalidating either. + * + * Note that we don't update the vma->userptr.notifier_seq since + * we don't update the userptr pages. + */ + do { + down_read(&vm->userptr.notifier_lock); + if (!mmu_interval_read_retry(&vma->userptr.notifier, + notifier_seq)) + break; + + up_read(&vm->userptr.notifier_lock); + + if (userptr_update->bind) + return -EAGAIN; + + notifier_seq = mmu_interval_read_begin(&vma->userptr.notifier); + } while (true); + + /* Inject errors to test_whether they are handled correctly */ + if (userptr_update->bind && xe_pt_userptr_inject_eagain(vma)) { + up_read(&vm->userptr.notifier_lock); + return -EAGAIN; + } + + userptr_update->locked = true; + + return 0; +} + +static const struct xe_migrate_pt_update_ops bind_ops = { + .populate = xe_vm_populate_pgtable, +}; + +static const struct xe_migrate_pt_update_ops userptr_bind_ops = { + .populate = xe_vm_populate_pgtable, + .pre_commit = xe_pt_userptr_pre_commit, +}; + +/** + * __xe_pt_bind_vma() - Build and connect a page-table tree for the vma + * address range. + * @gt: The gt to bind for. + * @vma: The vma to bind. + * @e: The engine with which to do pipelined page-table updates. + * @syncs: Entries to sync on before binding the built tree to the live vm tree. + * @num_syncs: Number of @sync entries. + * @rebind: Whether we're rebinding this vma to the same address range without + * an unbind in-between. + * + * This function builds a page-table tree (see xe_pt_stage_bind() for more + * information on page-table building), and the xe_vm_pgtable_update entries + * abstracting the operations needed to attach it to the main vm tree. It + * then takes the relevant locks and updates the metadata side of the main + * vm tree and submits the operations for pipelined attachment of the + * gpu page-table to the vm main tree, (which can be done either by the + * cpu and the GPU). + * + * Return: A valid dma-fence representing the pipelined attachment operation + * on success, an error pointer on error. + */ +struct dma_fence * +__xe_pt_bind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, + struct xe_sync_entry *syncs, u32 num_syncs, + bool rebind) +{ + struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; + struct xe_pt_migrate_pt_update bind_pt_update = { + .base = { + .ops = xe_vma_is_userptr(vma) ? &userptr_bind_ops : &bind_ops, + .vma = vma, + }, + .bind = true, + }; + struct xe_vm *vm = vma->vm; + u32 num_entries; + struct dma_fence *fence; + int err; + + bind_pt_update.locked = false; + xe_bo_assert_held(vma->bo); + xe_vm_assert_held(vm); + XE_BUG_ON(xe_gt_is_media_type(gt)); + + vm_dbg(&vma->vm->xe->drm, + "Preparing bind, with range [%llx...%llx) engine %p.\n", + vma->start, vma->end, e); + + err = xe_pt_prepare_bind(gt, vma, entries, &num_entries, rebind); + if (err) + goto err; + XE_BUG_ON(num_entries > ARRAY_SIZE(entries)); + + xe_vm_dbg_print_entries(gt_to_xe(gt), entries, num_entries); + + fence = xe_migrate_update_pgtables(gt->migrate, + vm, vma->bo, + e ? e : vm->eng[gt->info.id], + entries, num_entries, + syncs, num_syncs, + &bind_pt_update.base); + if (!IS_ERR(fence)) { + LLIST_HEAD(deferred); + + /* add shared fence now for pagetable delayed destroy */ + dma_resv_add_fence(&vm->resv, fence, !rebind && + vma->last_munmap_rebind ? + DMA_RESV_USAGE_KERNEL : + DMA_RESV_USAGE_BOOKKEEP); + + if (!xe_vma_is_userptr(vma) && !vma->bo->vm) + dma_resv_add_fence(vma->bo->ttm.base.resv, fence, + DMA_RESV_USAGE_BOOKKEEP); + xe_pt_commit_bind(vma, entries, num_entries, rebind, + bind_pt_update.locked ? &deferred : NULL); + + /* This vma is live (again?) now */ + vma->gt_present |= BIT(gt->info.id); + + if (bind_pt_update.locked) { + vma->userptr.initial_bind = true; + up_read(&vm->userptr.notifier_lock); + xe_bo_put_commit(&deferred); + } + if (!rebind && vma->last_munmap_rebind && + xe_vm_in_compute_mode(vm)) + queue_work(vm->xe->ordered_wq, + &vm->preempt.rebind_work); + } else { + if (bind_pt_update.locked) + up_read(&vm->userptr.notifier_lock); + xe_pt_abort_bind(vma, entries, num_entries); + } + + return fence; + +err: + return ERR_PTR(err); +} + +struct xe_pt_stage_unbind_walk { + /** @base: The pagewalk base-class. */ + struct xe_pt_walk base; + + /* Input parameters for the walk */ + /** @gt: The gt we're unbinding from. */ + struct xe_gt *gt; + + /** + * @modified_start: Walk range start, modified to include any + * shared pagetables that we're the only user of and can thus + * treat as private. + */ + u64 modified_start; + /** @modified_end: Walk range start, modified like @modified_start. */ + u64 modified_end; + + /* Output */ + /* @wupd: Structure to track the page-table updates we're building */ + struct xe_walk_update wupd; +}; + +/* + * Check whether this range is the only one populating this pagetable, + * and in that case, update the walk range checks so that higher levels don't + * view us as a shared pagetable. + */ +static bool xe_pt_check_kill(u64 addr, u64 next, unsigned int level, + const struct xe_pt *child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt_stage_unbind_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + unsigned int shift = walk->shifts[level]; + u64 size = 1ull << shift; + + if (IS_ALIGNED(addr, size) && IS_ALIGNED(next, size) && + ((next - addr) >> shift) == child->num_live) { + u64 size = 1ull << walk->shifts[level + 1]; + + *action = ACTION_CONTINUE; + + if (xe_walk->modified_start >= addr) + xe_walk->modified_start = round_down(addr, size); + if (xe_walk->modified_end <= next) + xe_walk->modified_end = round_up(next, size); + + return true; + } + + return false; +} + +static int xe_pt_stage_unbind_entry(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); + + XE_BUG_ON(!*child); + XE_BUG_ON(!level && xe_child->is_compact); + + xe_pt_check_kill(addr, next, level - 1, xe_child, action, walk); + + return 0; +} + +static int +xe_pt_stage_unbind_post_descend(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk) +{ + struct xe_pt_stage_unbind_walk *xe_walk = + container_of(walk, typeof(*xe_walk), base); + struct xe_pt *xe_child = container_of(*child, typeof(*xe_child), base); + pgoff_t end_offset; + u64 size = 1ull << walk->shifts[--level]; + + if (!IS_ALIGNED(addr, size)) + addr = xe_walk->modified_start; + if (!IS_ALIGNED(next, size)) + next = xe_walk->modified_end; + + /* Parent == *child is the root pt. Don't kill it. */ + if (parent != *child && + xe_pt_check_kill(addr, next, level, xe_child, action, walk)) + return 0; + + if (!xe_pt_nonshared_offsets(addr, next, level, walk, action, &offset, + &end_offset)) + return 0; + + (void)xe_pt_new_shared(&xe_walk->wupd, xe_child, offset, false); + xe_walk->wupd.updates[level].update->qwords = end_offset - offset; + + return 0; +} + +static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = { + .pt_entry = xe_pt_stage_unbind_entry, + .pt_post_descend = xe_pt_stage_unbind_post_descend, +}; + +/** + * xe_pt_stage_unbind() - Build page-table update structures for an unbind + * operation + * @gt: The gt we're unbinding for. + * @vma: The vma we're unbinding. + * @entries: Caller-provided storage for the update structures. + * + * Builds page-table update structures for an unbind operation. The function + * will attempt to remove all page-tables that we're the only user + * of, and for that to work, the unbind operation must be committed in the + * same critical section that blocks racing binds to the same page-table tree. + * + * Return: The number of entries used. + */ +static unsigned int xe_pt_stage_unbind(struct xe_gt *gt, struct xe_vma *vma, + struct xe_vm_pgtable_update *entries) +{ + struct xe_pt_stage_unbind_walk xe_walk = { + .base = { + .ops = &xe_pt_stage_unbind_ops, + .shifts = xe_normal_pt_shifts, + .max_level = XE_PT_HIGHEST_LEVEL, + }, + .gt = gt, + .modified_start = vma->start, + .modified_end = vma->end + 1, + .wupd.entries = entries, + }; + struct xe_pt *pt = vma->vm->pt_root[gt->info.id]; + + (void)xe_pt_walk_shared(&pt->base, pt->level, vma->start, vma->end + 1, + &xe_walk.base); + + return xe_walk.wupd.num_used_entries; +} + +static void +xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update, + struct xe_gt *gt, struct iosys_map *map, + void *ptr, u32 qword_ofs, u32 num_qwords, + const struct xe_vm_pgtable_update *update) +{ + struct xe_vma *vma = pt_update->vma; + u64 empty = __xe_pt_empty_pte(gt, vma->vm, update->pt->level); + int i; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + + if (map && map->is_iomem) + for (i = 0; i < num_qwords; ++i) + xe_map_wr(gt_to_xe(gt), map, (qword_ofs + i) * + sizeof(u64), u64, empty); + else if (map) + memset64(map->vaddr + qword_ofs * sizeof(u64), empty, + num_qwords); + else + memset64(ptr, empty, num_qwords); +} + +static void +xe_pt_commit_unbind(struct xe_vma *vma, + struct xe_vm_pgtable_update *entries, u32 num_entries, + struct llist_head *deferred) +{ + u32 j; + + xe_pt_commit_locks_assert(vma); + + for (j = 0; j < num_entries; ++j) { + struct xe_vm_pgtable_update *entry = &entries[j]; + struct xe_pt *pt = entry->pt; + + pt->num_live -= entry->qwords; + if (pt->level) { + struct xe_pt_dir *pt_dir = as_xe_pt_dir(pt); + u32 i; + + for (i = entry->ofs; i < entry->ofs + entry->qwords; + i++) { + if (xe_pt_entry(pt_dir, i)) + xe_pt_destroy(xe_pt_entry(pt_dir, i), + vma->vm->flags, deferred); + + pt_dir->dir.entries[i] = NULL; + } + } + } +} + +static const struct xe_migrate_pt_update_ops unbind_ops = { + .populate = xe_migrate_clear_pgtable_callback, +}; + +static const struct xe_migrate_pt_update_ops userptr_unbind_ops = { + .populate = xe_migrate_clear_pgtable_callback, + .pre_commit = xe_pt_userptr_pre_commit, +}; + +/** + * __xe_pt_unbind_vma() - Disconnect and free a page-table tree for the vma + * address range. + * @gt: The gt to unbind for. + * @vma: The vma to unbind. + * @e: The engine with which to do pipelined page-table updates. + * @syncs: Entries to sync on before disconnecting the tree to be destroyed. + * @num_syncs: Number of @sync entries. + * + * This function builds a the xe_vm_pgtable_update entries abstracting the + * operations needed to detach the page-table tree to be destroyed from the + * man vm tree. + * It then takes the relevant locks and submits the operations for + * pipelined detachment of the gpu page-table from the vm main tree, + * (which can be done either by the cpu and the GPU), Finally it frees the + * detached page-table tree. + * + * Return: A valid dma-fence representing the pipelined detachment operation + * on success, an error pointer on error. + */ +struct dma_fence * +__xe_pt_unbind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, + struct xe_sync_entry *syncs, u32 num_syncs) +{ + struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; + struct xe_pt_migrate_pt_update unbind_pt_update = { + .base = { + .ops = xe_vma_is_userptr(vma) ? &userptr_unbind_ops : + &unbind_ops, + .vma = vma, + }, + }; + struct xe_vm *vm = vma->vm; + u32 num_entries; + struct dma_fence *fence = NULL; + LLIST_HEAD(deferred); + + xe_bo_assert_held(vma->bo); + xe_vm_assert_held(vm); + XE_BUG_ON(xe_gt_is_media_type(gt)); + + vm_dbg(&vma->vm->xe->drm, + "Preparing unbind, with range [%llx...%llx) engine %p.\n", + vma->start, vma->end, e); + + num_entries = xe_pt_stage_unbind(gt, vma, entries); + XE_BUG_ON(num_entries > ARRAY_SIZE(entries)); + + xe_vm_dbg_print_entries(gt_to_xe(gt), entries, num_entries); + + /* + * Even if we were already evicted and unbind to destroy, we need to + * clear again here. The eviction may have updated pagetables at a + * lower level, because it needs to be more conservative. + */ + fence = xe_migrate_update_pgtables(gt->migrate, + vm, NULL, e ? e : + vm->eng[gt->info.id], + entries, num_entries, + syncs, num_syncs, + &unbind_pt_update.base); + if (!IS_ERR(fence)) { + /* add shared fence now for pagetable delayed destroy */ + dma_resv_add_fence(&vm->resv, fence, + DMA_RESV_USAGE_BOOKKEEP); + + /* This fence will be installed by caller when doing eviction */ + if (!xe_vma_is_userptr(vma) && !vma->bo->vm) + dma_resv_add_fence(vma->bo->ttm.base.resv, fence, + DMA_RESV_USAGE_BOOKKEEP); + xe_pt_commit_unbind(vma, entries, num_entries, + unbind_pt_update.locked ? &deferred : NULL); + vma->gt_present &= ~BIT(gt->info.id); + } + + if (!vma->gt_present) + list_del_init(&vma->rebind_link); + + if (unbind_pt_update.locked) { + XE_WARN_ON(!xe_vma_is_userptr(vma)); + + if (!vma->gt_present) { + spin_lock(&vm->userptr.invalidated_lock); + list_del_init(&vma->userptr.invalidate_link); + spin_unlock(&vm->userptr.invalidated_lock); + } + up_read(&vm->userptr.notifier_lock); + xe_bo_put_commit(&deferred); + } + + return fence; +} diff --git a/drivers/gpu/drm/xe/xe_pt.h b/drivers/gpu/drm/xe/xe_pt.h new file mode 100644 index 000000000000..1152043e5c63 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ +#ifndef _XE_PT_H_ +#define _XE_PT_H_ + +#include + +#include "xe_pt_types.h" + +struct dma_fence; +struct xe_bo; +struct xe_device; +struct xe_engine; +struct xe_gt; +struct xe_sync_entry; +struct xe_vm; +struct xe_vma; + +#define xe_pt_write(xe, map, idx, data) \ + xe_map_wr(xe, map, (idx) * sizeof(u64), u64, data) + +unsigned int xe_pt_shift(unsigned int level); + +struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_gt *gt, + unsigned int level); + +int xe_pt_create_scratch(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm); + +void xe_pt_populate_empty(struct xe_gt *gt, struct xe_vm *vm, + struct xe_pt *pt); + +void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred); + +struct dma_fence * +__xe_pt_bind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, + struct xe_sync_entry *syncs, u32 num_syncs, + bool rebind); + +struct dma_fence * +__xe_pt_unbind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, + struct xe_sync_entry *syncs, u32 num_syncs); + +bool xe_pt_zap_ptes(struct xe_gt *gt, struct xe_vma *vma); + +u64 gen8_pde_encode(struct xe_bo *bo, u64 bo_offset, + const enum xe_cache_level level); + +u64 gen8_pte_encode(struct xe_vma *vma, struct xe_bo *bo, + u64 offset, enum xe_cache_level cache, + u32 flags, u32 pt_level); +#endif diff --git a/drivers/gpu/drm/xe/xe_pt_types.h b/drivers/gpu/drm/xe/xe_pt_types.h new file mode 100644 index 000000000000..2ed64c0a4485 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt_types.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PT_TYPES_H_ +#define _XE_PT_TYPES_H_ + +#include "xe_pt_walk.h" + +enum xe_cache_level { + XE_CACHE_NONE, + XE_CACHE_WT, + XE_CACHE_WB, +}; + +#define XE_VM_MAX_LEVEL 4 + +struct xe_pt { + struct xe_ptw base; + struct xe_bo *bo; + unsigned int level; + unsigned int num_live; + bool rebind; + bool is_compact; +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) + /** addr: Virtual address start address of the PT. */ + u64 addr; +#endif +}; + +struct xe_pt_entry { + struct xe_pt *pt; + u64 pte; +}; + +struct xe_vm_pgtable_update { + /** @bo: page table bo to write to */ + struct xe_bo *pt_bo; + + /** @ofs: offset inside this PTE to begin writing to (in qwords) */ + u32 ofs; + + /** @qwords: number of PTE's to write */ + u32 qwords; + + /** @pt: opaque pointer useful for the caller of xe_migrate_update_pgtables */ + struct xe_pt *pt; + + /** @pt_entries: Newly added pagetable entries */ + struct xe_pt_entry *pt_entries; + + /** @flags: Target flags */ + u32 flags; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_pt_walk.c b/drivers/gpu/drm/xe/xe_pt_walk.c new file mode 100644 index 000000000000..0def89af4372 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt_walk.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2022 Intel Corporation + */ +#include "xe_pt_walk.h" + +/** + * DOC: GPU page-table tree walking. + * The utilities in this file are similar to the CPU page-table walk + * utilities in mm/pagewalk.c. The main difference is that we distinguish + * the various levels of a page-table tree with an unsigned integer rather + * than by name. 0 is the lowest level, and page-tables with level 0 can + * not be directories pointing to lower levels, whereas all other levels + * can. The user of the utilities determines the highest level. + * + * Nomenclature: + * Each struct xe_ptw, regardless of level is referred to as a page table, and + * multiple page tables typically form a page table tree with page tables at + * intermediate levels being page directories pointing at page tables at lower + * levels. A shared page table for a given address range is a page-table which + * is neither fully within nor fully outside the address range and that can + * thus be shared by two or more address ranges. + * + * Please keep this code generic so that it can used as a drm-wide page- + * table walker should other drivers find use for it. + */ +static u64 xe_pt_addr_end(u64 addr, u64 end, unsigned int level, + const struct xe_pt_walk *walk) +{ + u64 size = 1ull << walk->shifts[level]; + u64 tmp = round_up(addr + 1, size); + + return min_t(u64, tmp, end); +} + +static bool xe_pt_next(pgoff_t *offset, u64 *addr, u64 next, u64 end, + unsigned int level, const struct xe_pt_walk *walk) +{ + pgoff_t step = 1; + + /* Shared pt walk skips to the last pagetable */ + if (unlikely(walk->shared_pt_mode)) { + unsigned int shift = walk->shifts[level]; + u64 skip_to = round_down(end, 1ull << shift); + + if (skip_to > next) { + step += (skip_to - next) >> shift; + next = skip_to; + } + } + + *addr = next; + *offset += step; + + return next != end; +} + +/** + * xe_pt_walk_range() - Walk a range of a gpu page table tree with callbacks + * for each page-table entry in all levels. + * @parent: The root page table for walk start. + * @level: The root page table level. + * @addr: Virtual address start. + * @end: Virtual address end + 1. + * @walk: Walk info. + * + * Similar to the CPU page-table walker, this is a helper to walk + * a gpu page table and call a provided callback function for each entry. + * + * Return: 0 on success, negative error code on error. The error is + * propagated from the callback and on error the walk is terminated. + */ +int xe_pt_walk_range(struct xe_ptw *parent, unsigned int level, + u64 addr, u64 end, struct xe_pt_walk *walk) +{ + pgoff_t offset = xe_pt_offset(addr, level, walk); + struct xe_ptw **entries = parent->dir ? parent->dir->entries : NULL; + const struct xe_pt_walk_ops *ops = walk->ops; + enum page_walk_action action; + struct xe_ptw *child; + int err = 0; + u64 next; + + do { + next = xe_pt_addr_end(addr, end, level, walk); + if (walk->shared_pt_mode && xe_pt_covers(addr, next, level, + walk)) + continue; +again: + action = ACTION_SUBTREE; + child = entries ? entries[offset] : NULL; + err = ops->pt_entry(parent, offset, level, addr, next, + &child, &action, walk); + if (err) + break; + + /* Probably not needed yet for gpu pagetable walk. */ + if (unlikely(action == ACTION_AGAIN)) + goto again; + + if (likely(!level || !child || action == ACTION_CONTINUE)) + continue; + + err = xe_pt_walk_range(child, level - 1, addr, next, walk); + + if (!err && ops->pt_post_descend) + err = ops->pt_post_descend(parent, offset, level, addr, + next, &child, &action, walk); + if (err) + break; + + } while (xe_pt_next(&offset, &addr, next, end, level, walk)); + + return err; +} + +/** + * xe_pt_walk_shared() - Walk shared page tables of a page-table tree. + * @parent: Root page table directory. + * @level: Level of the root. + * @addr: Start address. + * @end: Last address + 1. + * @walk: Walk info. + * + * This function is similar to xe_pt_walk_range() but it skips page tables + * that are private to the range. Since the root (or @parent) page table is + * typically also a shared page table this function is different in that it + * calls the pt_entry callback and the post_descend callback also for the + * root. The root can be detected in the callbacks by checking whether + * parent == *child. + * Walking only the shared page tables is common for unbind-type operations + * where the page-table entries for an address range are cleared or detached + * from the main page-table tree. + * + * Return: 0 on success, negative error code on error: If a callback + * returns an error, the walk will be terminated and the error returned by + * this function. + */ +int xe_pt_walk_shared(struct xe_ptw *parent, unsigned int level, + u64 addr, u64 end, struct xe_pt_walk *walk) +{ + const struct xe_pt_walk_ops *ops = walk->ops; + enum page_walk_action action = ACTION_SUBTREE; + struct xe_ptw *child = parent; + int err; + + walk->shared_pt_mode = true; + err = walk->ops->pt_entry(parent, 0, level + 1, addr, end, + &child, &action, walk); + + if (err || action != ACTION_SUBTREE) + return err; + + err = xe_pt_walk_range(parent, level, addr, end, walk); + if (!err && ops->pt_post_descend) { + err = ops->pt_post_descend(parent, 0, level + 1, addr, end, + &child, &action, walk); + } + return err; +} diff --git a/drivers/gpu/drm/xe/xe_pt_walk.h b/drivers/gpu/drm/xe/xe_pt_walk.h new file mode 100644 index 000000000000..42c51fa601ec --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pt_walk.h @@ -0,0 +1,161 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright © 2022 Intel Corporation + */ +#ifndef __XE_PT_WALK__ +#define __XE_PT_WALK__ + +#include +#include + +struct xe_ptw_dir; + +/** + * struct xe_ptw - base class for driver pagetable subclassing. + * @dir: Pointer to an array of children if any. + * + * Drivers could subclass this, and if it's a page-directory, typically + * embed the xe_ptw_dir::entries array in the same allocation. + */ +struct xe_ptw { + struct xe_ptw_dir *dir; +}; + +/** + * struct xe_ptw_dir - page directory structure + * @entries: Array holding page directory children. + * + * It is the responsibility of the user to ensure @entries is + * correctly sized. + */ +struct xe_ptw_dir { + struct xe_ptw *entries[0]; +}; + +/** + * struct xe_pt_walk - Embeddable struct for walk parameters + */ +struct xe_pt_walk { + /** @ops: The walk ops used for the pagewalk */ + const struct xe_pt_walk_ops *ops; + /** + * @shifts: Array of page-table entry shifts used for the + * different levels, starting out with the leaf level 0 + * page-shift as the first entry. It's legal for this pointer to be + * changed during the walk. + */ + const u64 *shifts; + /** @max_level: Highest populated level in @sizes */ + unsigned int max_level; + /** + * @shared_pt_mode: Whether to skip all entries that are private + * to the address range and called only for entries that are + * shared with other address ranges. Such entries are referred to + * as shared pagetables. + */ + bool shared_pt_mode; +}; + +/** + * typedef xe_pt_entry_fn - gpu page-table-walk callback-function + * @parent: The parent page.table. + * @offset: The offset (number of entries) into the page table. + * @level: The level of @parent. + * @addr: The virtual address. + * @next: The virtual address for the next call, or end address. + * @child: Pointer to pointer to child page-table at this @offset. The + * function may modify the value pointed to if, for example, allocating a + * child page table. + * @action: The walk action to take upon return. See . + * @walk: The walk parameters. + */ +typedef int (*xe_pt_entry_fn)(struct xe_ptw *parent, pgoff_t offset, + unsigned int level, u64 addr, u64 next, + struct xe_ptw **child, + enum page_walk_action *action, + struct xe_pt_walk *walk); + +/** + * struct xe_pt_walk_ops - Walk callbacks. + */ +struct xe_pt_walk_ops { + /** + * @pt_entry: Callback to be called for each page table entry prior + * to descending to the next level. The returned value of the action + * function parameter is honored. + */ + xe_pt_entry_fn pt_entry; + /** + * @pt_post_descend: Callback to be called for each page table entry + * after return from descending to the next level. The returned value + * of the action function parameter is ignored. + */ + xe_pt_entry_fn pt_post_descend; +}; + +int xe_pt_walk_range(struct xe_ptw *parent, unsigned int level, + u64 addr, u64 end, struct xe_pt_walk *walk); + +int xe_pt_walk_shared(struct xe_ptw *parent, unsigned int level, + u64 addr, u64 end, struct xe_pt_walk *walk); + +/** + * xe_pt_covers - Whether the address range covers an entire entry in @level + * @addr: Start of the range. + * @end: End of range + 1. + * @level: Page table level. + * @walk: Page table walk info. + * + * This function is a helper to aid in determining whether a leaf page table + * entry can be inserted at this @level. + * + * Return: Whether the range provided covers exactly an entry at this level. + */ +static inline bool xe_pt_covers(u64 addr, u64 end, unsigned int level, + const struct xe_pt_walk *walk) +{ + u64 pt_size = 1ull << walk->shifts[level]; + + return end - addr == pt_size && IS_ALIGNED(addr, pt_size); +} + +/** + * xe_pt_num_entries: Number of page-table entries of a given range at this + * level + * @addr: Start address. + * @end: End address. + * @level: Page table level. + * @walk: Walk info. + * + * Return: The number of page table entries at this level between @start and + * @end. + */ +static inline pgoff_t +xe_pt_num_entries(u64 addr, u64 end, unsigned int level, + const struct xe_pt_walk *walk) +{ + u64 pt_size = 1ull << walk->shifts[level]; + + return (round_up(end, pt_size) - round_down(addr, pt_size)) >> + walk->shifts[level]; +} + +/** + * xe_pt_offset: Offset of the page-table entry for a given address. + * @addr: The address. + * @level: Page table level. + * @walk: Walk info. + * + * Return: The page table entry offset for the given address in a + * page table with size indicated by @level. + */ +static inline pgoff_t +xe_pt_offset(u64 addr, unsigned int level, const struct xe_pt_walk *walk) +{ + if (level < walk->max_level) + addr &= ((1ull << walk->shifts[level + 1]) - 1); + + return addr >> walk->shifts[level]; +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c new file mode 100644 index 000000000000..6e904e97f456 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_query.c @@ -0,0 +1,387 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_macros.h" +#include "xe_query.h" +#include "xe_ggtt.h" +#include "xe_guc_hwconfig.h" + +static const enum xe_engine_class xe_to_user_engine_class[] = { + [XE_ENGINE_CLASS_RENDER] = DRM_XE_ENGINE_CLASS_RENDER, + [XE_ENGINE_CLASS_COPY] = DRM_XE_ENGINE_CLASS_COPY, + [XE_ENGINE_CLASS_VIDEO_DECODE] = DRM_XE_ENGINE_CLASS_VIDEO_DECODE, + [XE_ENGINE_CLASS_VIDEO_ENHANCE] = DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE, + [XE_ENGINE_CLASS_COMPUTE] = DRM_XE_ENGINE_CLASS_COMPUTE, +}; + +static size_t calc_hw_engine_info_size(struct xe_device *xe) +{ + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct xe_gt *gt; + u8 gt_id; + int i = 0; + + for_each_gt(gt, xe, gt_id) + for_each_hw_engine(hwe, gt, id) { + if (xe_hw_engine_is_reserved(hwe)) + continue; + i++; + } + + return i * sizeof(struct drm_xe_engine_class_instance); +} + +static int query_engines(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + size_t size = calc_hw_engine_info_size(xe); + struct drm_xe_engine_class_instance __user *query_ptr = + u64_to_user_ptr(query->data); + struct drm_xe_engine_class_instance *hw_engine_info; + struct xe_hw_engine *hwe; + enum xe_hw_engine_id id; + struct xe_gt *gt; + u8 gt_id; + int i = 0; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_ERR(xe, query->size != size)) { + return -EINVAL; + } + + hw_engine_info = kmalloc(size, GFP_KERNEL); + if (XE_IOCTL_ERR(xe, !hw_engine_info)) + return -ENOMEM; + + for_each_gt(gt, xe, gt_id) + for_each_hw_engine(hwe, gt, id) { + if (xe_hw_engine_is_reserved(hwe)) + continue; + + hw_engine_info[i].engine_class = + xe_to_user_engine_class[hwe->class]; + hw_engine_info[i].engine_instance = + hwe->logical_instance; + hw_engine_info[i++].gt_id = gt->info.id; + } + + if (copy_to_user(query_ptr, hw_engine_info, size)) { + kfree(hw_engine_info); + return -EFAULT; + } + kfree(hw_engine_info); + + return 0; +} + +static size_t calc_memory_usage_size(struct xe_device *xe) +{ + u32 num_managers = 1; + int i; + + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) + if (ttm_manager_type(&xe->ttm, i)) + num_managers++; + + return offsetof(struct drm_xe_query_mem_usage, regions[num_managers]); +} + +static int query_memory_usage(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + size_t size = calc_memory_usage_size(xe); + struct drm_xe_query_mem_usage *usage; + struct drm_xe_query_mem_usage __user *query_ptr = + u64_to_user_ptr(query->data); + struct ttm_resource_manager *man; + int ret, i; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_ERR(xe, query->size != size)) { + return -EINVAL; + } + + usage = kmalloc(size, GFP_KERNEL); + if (XE_IOCTL_ERR(xe, !usage)) + return -ENOMEM; + + usage->pad = 0; + + man = ttm_manager_type(&xe->ttm, XE_PL_TT); + usage->regions[0].mem_class = XE_MEM_REGION_CLASS_SYSMEM; + usage->regions[0].instance = 0; + usage->regions[0].pad = 0; + usage->regions[0].min_page_size = PAGE_SIZE; + usage->regions[0].max_page_size = PAGE_SIZE; + usage->regions[0].total_size = man->size << PAGE_SHIFT; + usage->regions[0].used = ttm_resource_manager_usage(man); + usage->num_regions = 1; + + for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { + man = ttm_manager_type(&xe->ttm, i); + if (man) { + usage->regions[usage->num_regions].mem_class = + XE_MEM_REGION_CLASS_VRAM; + usage->regions[usage->num_regions].instance = + usage->num_regions; + usage->regions[usage->num_regions].pad = 0; + usage->regions[usage->num_regions].min_page_size = + xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? + SZ_64K : PAGE_SIZE; + usage->regions[usage->num_regions].max_page_size = + SZ_1G; + usage->regions[usage->num_regions].total_size = + man->size; + usage->regions[usage->num_regions++].used = + ttm_resource_manager_usage(man); + } + } + + if (!copy_to_user(query_ptr, usage, size)) + ret = 0; + else + ret = -ENOSPC; + + kfree(usage); + return ret; +} + +static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) +{ + u32 num_params = XE_QUERY_CONFIG_NUM_PARAM; + size_t size = + sizeof(struct drm_xe_query_config) + num_params * sizeof(u64); + struct drm_xe_query_config __user *query_ptr = + u64_to_user_ptr(query->data); + struct drm_xe_query_config *config; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_ERR(xe, query->size != size)) { + return -EINVAL; + } + + config = kzalloc(size, GFP_KERNEL); + if (XE_IOCTL_ERR(xe, !config)) + return -ENOMEM; + + config->num_params = num_params; + config->info[XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = + xe->info.devid | (xe->info.revid << 16); + if (to_gt(xe)->mem.vram.size) + config->info[XE_QUERY_CONFIG_FLAGS] = + XE_QUERY_CONFIG_FLAGS_HAS_VRAM; + if (xe->info.enable_guc) + config->info[XE_QUERY_CONFIG_FLAGS] |= + XE_QUERY_CONFIG_FLAGS_USE_GUC; + config->info[XE_QUERY_CONFIG_MIN_ALIGNEMENT] = + xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; + config->info[XE_QUERY_CONFIG_VA_BITS] = 12 + + (9 * (xe->info.vm_max_level + 1)); + config->info[XE_QUERY_CONFIG_GT_COUNT] = xe->info.tile_count; + config->info[XE_QUERY_CONFIG_MEM_REGION_COUNT] = + hweight_long(xe->info.mem_region_mask); + + if (copy_to_user(query_ptr, config, size)) { + kfree(config); + return -EFAULT; + } + kfree(config); + + return 0; +} + +static int query_gts(struct xe_device *xe, struct drm_xe_device_query *query) +{ + struct xe_gt *gt; + size_t size = sizeof(struct drm_xe_query_gts) + + xe->info.tile_count * sizeof(struct drm_xe_query_gt); + struct drm_xe_query_gts __user *query_ptr = + u64_to_user_ptr(query->data); + struct drm_xe_query_gts *gts; + u8 id; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_ERR(xe, query->size != size)) { + return -EINVAL; + } + + gts = kzalloc(size, GFP_KERNEL); + if (XE_IOCTL_ERR(xe, !gts)) + return -ENOMEM; + + gts->num_gt = xe->info.tile_count; + for_each_gt(gt, xe, id) { + if (id == 0) + gts->gts[id].type = XE_QUERY_GT_TYPE_MAIN; + else if (xe_gt_is_media_type(gt)) + gts->gts[id].type = XE_QUERY_GT_TYPE_MEDIA; + else + gts->gts[id].type = XE_QUERY_GT_TYPE_REMOTE; + gts->gts[id].instance = id; + gts->gts[id].clock_freq = gt->info.clock_freq; + if (!IS_DGFX(xe)) + gts->gts[id].native_mem_regions = 0x1; + else + gts->gts[id].native_mem_regions = + BIT(gt->info.vram_id) << 1; + gts->gts[id].slow_mem_regions = xe->info.mem_region_mask ^ + gts->gts[id].native_mem_regions; + } + + if (copy_to_user(query_ptr, gts, size)) { + kfree(gts); + return -EFAULT; + } + kfree(gts); + + return 0; +} + +static int query_hwconfig(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + struct xe_gt *gt = xe_device_get_gt(xe, 0); + size_t size = xe_guc_hwconfig_size(>->uc.guc); + void __user *query_ptr = u64_to_user_ptr(query->data); + void *hwconfig; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_ERR(xe, query->size != size)) { + return -EINVAL; + } + + hwconfig = kzalloc(size, GFP_KERNEL); + if (XE_IOCTL_ERR(xe, !hwconfig)) + return -ENOMEM; + + xe_device_mem_access_get(xe); + xe_guc_hwconfig_copy(>->uc.guc, hwconfig); + xe_device_mem_access_put(xe); + + if (copy_to_user(query_ptr, hwconfig, size)) { + kfree(hwconfig); + return -EFAULT; + } + kfree(hwconfig); + + return 0; +} + +static size_t calc_topo_query_size(struct xe_device *xe) +{ + return xe->info.tile_count * + (3 * sizeof(struct drm_xe_query_topology_mask) + + sizeof_field(struct xe_gt, fuse_topo.g_dss_mask) + + sizeof_field(struct xe_gt, fuse_topo.c_dss_mask) + + sizeof_field(struct xe_gt, fuse_topo.eu_mask_per_dss)); +} + +static void __user *copy_mask(void __user *ptr, + struct drm_xe_query_topology_mask *topo, + void *mask, size_t mask_size) +{ + topo->num_bytes = mask_size; + + if (copy_to_user(ptr, topo, sizeof(*topo))) + return ERR_PTR(-EFAULT); + ptr += sizeof(topo); + + if (copy_to_user(ptr, mask, mask_size)) + return ERR_PTR(-EFAULT); + ptr += mask_size; + + return ptr; +} + +static int query_gt_topology(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + void __user *query_ptr = u64_to_user_ptr(query->data); + size_t size = calc_topo_query_size(xe); + struct drm_xe_query_topology_mask topo; + struct xe_gt *gt; + int id; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_ERR(xe, query->size != size)) { + return -EINVAL; + } + + for_each_gt(gt, xe, id) { + topo.gt_id = id; + + topo.type = XE_TOPO_DSS_GEOMETRY; + query_ptr = copy_mask(query_ptr, &topo, + gt->fuse_topo.g_dss_mask, + sizeof(gt->fuse_topo.g_dss_mask)); + if (IS_ERR(query_ptr)) + return PTR_ERR(query_ptr); + + topo.type = XE_TOPO_DSS_COMPUTE; + query_ptr = copy_mask(query_ptr, &topo, + gt->fuse_topo.c_dss_mask, + sizeof(gt->fuse_topo.c_dss_mask)); + if (IS_ERR(query_ptr)) + return PTR_ERR(query_ptr); + + topo.type = XE_TOPO_EU_PER_DSS; + query_ptr = copy_mask(query_ptr, &topo, + gt->fuse_topo.eu_mask_per_dss, + sizeof(gt->fuse_topo.eu_mask_per_dss)); + if (IS_ERR(query_ptr)) + return PTR_ERR(query_ptr); + } + + return 0; +} + +static int (* const xe_query_funcs[])(struct xe_device *xe, + struct drm_xe_device_query *query) = { + query_engines, + query_memory_usage, + query_config, + query_gts, + query_hwconfig, + query_gt_topology, +}; + +int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct drm_xe_device_query *query = data; + u32 idx; + + if (XE_IOCTL_ERR(xe, query->extensions != 0)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, query->query > ARRAY_SIZE(xe_query_funcs))) + return -EINVAL; + + idx = array_index_nospec(query->query, ARRAY_SIZE(xe_query_funcs)); + if (XE_IOCTL_ERR(xe, !xe_query_funcs[idx])) + return -EINVAL; + + return xe_query_funcs[idx](xe, query); +} diff --git a/drivers/gpu/drm/xe/xe_query.h b/drivers/gpu/drm/xe/xe_query.h new file mode 100644 index 000000000000..beeb7a8192b4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_query.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_QUERY_H_ +#define _XE_QUERY_H_ + +struct drm_device; +struct drm_file; + +int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file); + +#endif diff --git a/drivers/gpu/drm/xe/xe_reg_sr.c b/drivers/gpu/drm/xe/xe_reg_sr.c new file mode 100644 index 000000000000..16e025dcf2cc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_sr.c @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_reg_sr.h" + +#include +#include +#include + +#include +#include + +#include "xe_rtp_types.h" +#include "xe_device_types.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_gt_mcr.h" +#include "xe_macros.h" +#include "xe_mmio.h" + +#include "gt/intel_engine_regs.h" +#include "gt/intel_gt_regs.h" + +#define XE_REG_SR_GROW_STEP_DEFAULT 16 + +static void reg_sr_fini(struct drm_device *drm, void *arg) +{ + struct xe_reg_sr *sr = arg; + + xa_destroy(&sr->xa); + kfree(sr->pool.arr); + memset(&sr->pool, 0, sizeof(sr->pool)); +} + +int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe) +{ + xa_init(&sr->xa); + memset(&sr->pool, 0, sizeof(sr->pool)); + sr->pool.grow_step = XE_REG_SR_GROW_STEP_DEFAULT; + sr->name = name; + + return drmm_add_action_or_reset(&xe->drm, reg_sr_fini, sr); +} + +int xe_reg_sr_dump_kv(struct xe_reg_sr *sr, + struct xe_reg_sr_kv **dst) +{ + struct xe_reg_sr_kv *iter; + struct xe_reg_sr_entry *entry; + unsigned long idx; + + if (xa_empty(&sr->xa)) { + *dst = NULL; + return 0; + } + + *dst = kmalloc_array(sr->pool.used, sizeof(**dst), GFP_KERNEL); + if (!*dst) + return -ENOMEM; + + iter = *dst; + xa_for_each(&sr->xa, idx, entry) { + iter->k = idx; + iter->v = *entry; + iter++; + } + + return 0; +} + +static struct xe_reg_sr_entry *alloc_entry(struct xe_reg_sr *sr) +{ + if (sr->pool.used == sr->pool.allocated) { + struct xe_reg_sr_entry *arr; + + arr = krealloc_array(sr->pool.arr, + ALIGN(sr->pool.allocated + 1, sr->pool.grow_step), + sizeof(*arr), GFP_KERNEL); + if (!arr) + return NULL; + + sr->pool.arr = arr; + sr->pool.allocated += sr->pool.grow_step; + } + + return &sr->pool.arr[sr->pool.used++]; +} + +static bool compatible_entries(const struct xe_reg_sr_entry *e1, + const struct xe_reg_sr_entry *e2) +{ + /* + * Don't allow overwriting values: clr_bits/set_bits should be disjoint + * when operating in the same register + */ + if (e1->clr_bits & e2->clr_bits || e1->set_bits & e2->set_bits || + e1->clr_bits & e2->set_bits || e1->set_bits & e2->clr_bits) + return false; + + if (e1->masked_reg != e2->masked_reg) + return false; + + if (e1->reg_type != e2->reg_type) + return false; + + return true; +} + +int xe_reg_sr_add(struct xe_reg_sr *sr, u32 reg, + const struct xe_reg_sr_entry *e) +{ + unsigned long idx = reg; + struct xe_reg_sr_entry *pentry = xa_load(&sr->xa, idx); + int ret; + + if (pentry) { + if (!compatible_entries(pentry, e)) { + ret = -EINVAL; + goto fail; + } + + pentry->clr_bits |= e->clr_bits; + pentry->set_bits |= e->set_bits; + pentry->read_mask |= e->read_mask; + + return 0; + } + + pentry = alloc_entry(sr); + if (!pentry) { + ret = -ENOMEM; + goto fail; + } + + *pentry = *e; + ret = xa_err(xa_store(&sr->xa, idx, pentry, GFP_KERNEL)); + if (ret) + goto fail; + + return 0; + +fail: + DRM_ERROR("Discarding save-restore reg %04lx (clear: %08x, set: %08x, masked: %s): ret=%d\n", + idx, e->clr_bits, e->set_bits, + str_yes_no(e->masked_reg), ret); + + return ret; +} + +static void apply_one_mmio(struct xe_gt *gt, u32 reg, + struct xe_reg_sr_entry *entry) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 val; + + /* + * If this is a masked register, need to figure what goes on the upper + * 16 bits: it's either the clr_bits (when using FIELD_SET and WR) or + * the set_bits, when using SET. + * + * When it's not masked, we have to read it from hardware, unless we are + * supposed to set all bits. + */ + if (entry->masked_reg) + val = (entry->clr_bits ?: entry->set_bits << 16); + else if (entry->clr_bits + 1) + val = (entry->reg_type == XE_RTP_REG_MCR ? + xe_gt_mcr_unicast_read_any(gt, MCR_REG(reg)) : + xe_mmio_read32(gt, reg)) & (~entry->clr_bits); + else + val = 0; + + /* + * TODO: add selftest to validate all tables, regardless of platform: + * - Masked registers can't have set_bits with upper bits set + * - set_bits must be contained in clr_bits + */ + val |= entry->set_bits; + + drm_dbg(&xe->drm, "REG[0x%x] = 0x%08x", reg, val); + + if (entry->reg_type == XE_RTP_REG_MCR) + xe_gt_mcr_multicast_write(gt, MCR_REG(reg), val); + else + xe_mmio_write32(gt, reg, val); +} + +void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_reg_sr_entry *entry; + unsigned long reg; + int err; + + drm_dbg(&xe->drm, "Applying %s save-restore MMIOs\n", sr->name); + + err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL); + if (err) + goto err_force_wake; + + xa_for_each(&sr->xa, reg, entry) + apply_one_mmio(gt, reg, entry); + + err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL); + XE_WARN_ON(err); + + return; + +err_force_wake: + drm_err(&xe->drm, "Failed to apply, err=%d\n", err); +} + +void xe_reg_sr_apply_whitelist(struct xe_reg_sr *sr, u32 mmio_base, + struct xe_gt *gt) +{ + struct xe_device *xe = gt_to_xe(gt); + struct xe_reg_sr_entry *entry; + unsigned long reg; + unsigned int slot = 0; + int err; + + drm_dbg(&xe->drm, "Whitelisting %s registers\n", sr->name); + + err = xe_force_wake_get(>->mmio.fw, XE_FORCEWAKE_ALL); + if (err) + goto err_force_wake; + + xa_for_each(&sr->xa, reg, entry) { + xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot).reg, + reg | entry->set_bits); + slot++; + } + + /* And clear the rest just in case of garbage */ + for (; slot < RING_MAX_NONPRIV_SLOTS; slot++) + xe_mmio_write32(gt, RING_FORCE_TO_NONPRIV(mmio_base, slot).reg, + RING_NOPID(mmio_base).reg); + + err = xe_force_wake_put(>->mmio.fw, XE_FORCEWAKE_ALL); + XE_WARN_ON(err); + + return; + +err_force_wake: + drm_err(&xe->drm, "Failed to apply, err=%d\n", err); +} diff --git a/drivers/gpu/drm/xe/xe_reg_sr.h b/drivers/gpu/drm/xe/xe_reg_sr.h new file mode 100644 index 000000000000..c3a9db251e92 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_sr.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_REG_SR_ +#define _XE_REG_SR_ + +#include "xe_reg_sr_types.h" + +/* + * Reg save/restore bookkeeping + */ + +struct xe_device; +struct xe_gt; + +int xe_reg_sr_init(struct xe_reg_sr *sr, const char *name, struct xe_device *xe); +int xe_reg_sr_dump_kv(struct xe_reg_sr *sr, + struct xe_reg_sr_kv **dst); + +int xe_reg_sr_add(struct xe_reg_sr *sr, u32 reg, + const struct xe_reg_sr_entry *e); +void xe_reg_sr_apply_mmio(struct xe_reg_sr *sr, struct xe_gt *gt); +void xe_reg_sr_apply_whitelist(struct xe_reg_sr *sr, u32 mmio_base, + struct xe_gt *gt); + +#endif diff --git a/drivers/gpu/drm/xe/xe_reg_sr_types.h b/drivers/gpu/drm/xe/xe_reg_sr_types.h new file mode 100644 index 000000000000..2fa7ff3966ba --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_sr_types.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_REG_SR_TYPES_ +#define _XE_REG_SR_TYPES_ + +#include +#include + +#include "i915_reg_defs.h" + +struct xe_reg_sr_entry { + u32 clr_bits; + u32 set_bits; + /* Mask for bits to consider when reading value back */ + u32 read_mask; + /* + * "Masked registers" are marked in spec as register with the upper 16 + * bits as a mask for the bits that is being updated on the lower 16 + * bits when writing to it. + */ + u8 masked_reg; + u8 reg_type; +}; + +struct xe_reg_sr_kv { + u32 k; + struct xe_reg_sr_entry v; +}; + +struct xe_reg_sr { + struct { + struct xe_reg_sr_entry *arr; + unsigned int used; + unsigned int allocated; + unsigned int grow_step; + } pool; + struct xarray xa; + const char *name; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.c b/drivers/gpu/drm/xe/xe_reg_whitelist.c new file mode 100644 index 000000000000..2e0c87b72395 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.c @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include "xe_reg_whitelist.h" + +#include "xe_platform_types.h" +#include "xe_gt_types.h" +#include "xe_rtp.h" + +#include "../i915/gt/intel_engine_regs.h" +#include "../i915/gt/intel_gt_regs.h" + +#undef _MMIO +#undef MCR_REG +#define _MMIO(x) _XE_RTP_REG(x) +#define MCR_REG(x) _XE_RTP_MCR_REG(x) + +static bool match_not_render(const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return hwe->class != XE_ENGINE_CLASS_RENDER; +} + +static const struct xe_rtp_entry register_whitelist[] = { + { XE_RTP_NAME("WaAllowPMDepthAndInvocationCountAccessFromUMD, 1408556865"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_WHITELIST_REGISTER(PS_INVOCATION_COUNT, + RING_FORCE_TO_NONPRIV_ACCESS_RD | + RING_FORCE_TO_NONPRIV_RANGE_4) + }, + { XE_RTP_NAME("1508744258, 14012131227, 1808121037"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_WHITELIST_REGISTER(GEN7_COMMON_SLICE_CHICKEN1, 0) + }, + { XE_RTP_NAME("1806527549"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_WHITELIST_REGISTER(HIZ_CHICKEN, 0) + }, + { XE_RTP_NAME("allow_read_ctx_timestamp"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1260), FUNC(match_not_render)), + XE_WHITELIST_REGISTER(RING_CTX_TIMESTAMP(0), + RING_FORCE_TO_NONPRIV_ACCESS_RD, + XE_RTP_FLAG(ENGINE_BASE)) + }, + { XE_RTP_NAME("16014440446_part_1"), + XE_RTP_RULES(PLATFORM(PVC)), + XE_WHITELIST_REGISTER(_MMIO(0x4400), + RING_FORCE_TO_NONPRIV_DENY | + RING_FORCE_TO_NONPRIV_RANGE_64) + }, + { XE_RTP_NAME("16014440446_part_2"), + XE_RTP_RULES(PLATFORM(PVC)), + XE_WHITELIST_REGISTER(_MMIO(0x4500), + RING_FORCE_TO_NONPRIV_DENY | + RING_FORCE_TO_NONPRIV_RANGE_64) + }, + {} +}; + +/** + * xe_reg_whitelist_process_engine - process table of registers to whitelist + * @hwe: engine instance to process whitelist for + * + * Process wwhitelist table for this platform, saving in @hwe all the + * registers that need to be whitelisted by the hardware so they can be accessed + * by userspace. + */ +void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe) +{ + xe_rtp_process(register_whitelist, &hwe->reg_whitelist, hwe->gt, hwe); +} diff --git a/drivers/gpu/drm/xe/xe_reg_whitelist.h b/drivers/gpu/drm/xe/xe_reg_whitelist.h new file mode 100644 index 000000000000..6e861b1bdb01 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_reg_whitelist.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_REG_WHITELIST_ +#define _XE_REG_WHITELIST_ + +struct xe_hw_engine; + +void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_res_cursor.h b/drivers/gpu/drm/xe/xe_res_cursor.h new file mode 100644 index 000000000000..f54409850d74 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_res_cursor.h @@ -0,0 +1,226 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright 2020 Advanced Micro Devices, Inc. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR + * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef __XE_RES_CURSOR_H__ +#define __XE_RES_CURSOR_H__ + +#include + +#include +#include +#include +#include +#include + +#include "xe_bo.h" +#include "xe_macros.h" +#include "xe_ttm_vram_mgr.h" + +/* state back for walking over vram_mgr and gtt_mgr allocations */ +struct xe_res_cursor { + u64 start; + u64 size; + u64 remaining; + void *node; + u32 mem_type; + struct scatterlist *sgl; +}; + +/** + * xe_res_first - initialize a xe_res_cursor + * + * @res: TTM resource object to walk + * @start: Start of the range + * @size: Size of the range + * @cur: cursor object to initialize + * + * Start walking over the range of allocations between @start and @size. + */ +static inline void xe_res_first(struct ttm_resource *res, + u64 start, u64 size, + struct xe_res_cursor *cur) +{ + struct drm_buddy_block *block; + struct list_head *head, *next; + + cur->sgl = NULL; + if (!res) + goto fallback; + + XE_BUG_ON(start + size > res->size); + + cur->mem_type = res->mem_type; + + switch (cur->mem_type) { + case XE_PL_VRAM0: + case XE_PL_VRAM1: + head = &to_xe_ttm_vram_mgr_resource(res)->blocks; + + block = list_first_entry_or_null(head, + struct drm_buddy_block, + link); + if (!block) + goto fallback; + + while (start >= xe_ttm_vram_mgr_block_size(block)) { + start -= xe_ttm_vram_mgr_block_size(block); + + next = block->link.next; + if (next != head) + block = list_entry(next, struct drm_buddy_block, + link); + } + + cur->start = xe_ttm_vram_mgr_block_start(block) + start; + cur->size = min(xe_ttm_vram_mgr_block_size(block) - start, + size); + cur->remaining = size; + cur->node = block; + break; + default: + goto fallback; + } + + return; + +fallback: + cur->start = start; + cur->size = size; + cur->remaining = size; + cur->node = NULL; + cur->mem_type = XE_PL_TT; + XE_WARN_ON(res && start + size > res->size); + return; +} + +static inline void __xe_res_sg_next(struct xe_res_cursor *cur) +{ + struct scatterlist *sgl = cur->sgl; + u64 start = cur->start; + + while (start >= sg_dma_len(sgl)) { + start -= sg_dma_len(sgl); + sgl = sg_next(sgl); + XE_BUG_ON(!sgl); + } + + cur->start = start; + cur->size = sg_dma_len(sgl) - start; + cur->sgl = sgl; +} + +/** + * xe_res_first_sg - initialize a xe_res_cursor with a scatter gather table + * + * @sg: scatter gather table to walk + * @start: Start of the range + * @size: Size of the range + * @cur: cursor object to initialize + * + * Start walking over the range of allocations between @start and @size. + */ +static inline void xe_res_first_sg(const struct sg_table *sg, + u64 start, u64 size, + struct xe_res_cursor *cur) +{ + XE_BUG_ON(!sg); + XE_BUG_ON(!IS_ALIGNED(start, PAGE_SIZE) || + !IS_ALIGNED(size, PAGE_SIZE)); + cur->node = NULL; + cur->start = start; + cur->remaining = size; + cur->size = 0; + cur->sgl = sg->sgl; + cur->mem_type = XE_PL_TT; + __xe_res_sg_next(cur); +} + +/** + * xe_res_next - advance the cursor + * + * @cur: the cursor to advance + * @size: number of bytes to move forward + * + * Move the cursor @size bytes forwrad, walking to the next node if necessary. + */ +static inline void xe_res_next(struct xe_res_cursor *cur, u64 size) +{ + struct drm_buddy_block *block; + struct list_head *next; + u64 start; + + XE_BUG_ON(size > cur->remaining); + + cur->remaining -= size; + if (!cur->remaining) + return; + + if (cur->size > size) { + cur->size -= size; + cur->start += size; + return; + } + + if (cur->sgl) { + cur->start += size; + __xe_res_sg_next(cur); + return; + } + + switch (cur->mem_type) { + case XE_PL_VRAM0: + case XE_PL_VRAM1: + start = size - cur->size; + block = cur->node; + + next = block->link.next; + block = list_entry(next, struct drm_buddy_block, link); + + + while (start >= xe_ttm_vram_mgr_block_size(block)) { + start -= xe_ttm_vram_mgr_block_size(block); + + next = block->link.next; + block = list_entry(next, struct drm_buddy_block, link); + } + + cur->start = xe_ttm_vram_mgr_block_start(block) + start; + cur->size = min(xe_ttm_vram_mgr_block_size(block) - start, + cur->remaining); + cur->node = block; + break; + default: + return; + } +} + +/** + * xe_res_dma - return dma address of cursor at current position + * + * @cur: the cursor to return the dma address from + */ +static inline u64 xe_res_dma(const struct xe_res_cursor *cur) +{ + return cur->sgl ? sg_dma_address(cur->sgl) + cur->start : cur->start; +} +#endif diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c new file mode 100644 index 000000000000..fda7978a63e0 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_engine_types.h" +#include "xe_gt.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_ring_ops.h" +#include "xe_sched_job.h" +#include "xe_vm_types.h" + +#include "i915_reg.h" +#include "gt/intel_gpu_commands.h" +#include "gt/intel_gt_regs.h" +#include "gt/intel_lrc_reg.h" + +static u32 preparser_disable(bool state) +{ + return MI_ARB_CHECK | BIT(8) | state; +} + +static int emit_aux_table_inv(struct xe_gt *gt, u32 addr, u32 *dw, int i) +{ + dw[i++] = MI_LOAD_REGISTER_IMM(1) | MI_LRI_MMIO_REMAP_EN; + dw[i++] = addr + gt->mmio.adj_offset; + dw[i++] = AUX_INV; + dw[i++] = MI_NOOP; + + return i; +} + +static int emit_user_interrupt(u32 *dw, int i) +{ + dw[i++] = MI_USER_INTERRUPT; + dw[i++] = MI_ARB_ON_OFF | MI_ARB_ENABLE; + dw[i++] = MI_ARB_CHECK; + + return i; +} + +static int emit_store_imm_ggtt(u32 addr, u32 value, u32 *dw, int i) +{ + dw[i++] = MI_STORE_DATA_IMM | BIT(22) /* GGTT */ | 2; + dw[i++] = addr; + dw[i++] = 0; + dw[i++] = value; + + return i; +} + +static int emit_flush_imm_ggtt(u32 addr, u32 value, u32 *dw, int i) +{ + dw[i++] = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW; + dw[i++] = addr | MI_FLUSH_DW_USE_GTT; + dw[i++] = 0; + dw[i++] = value; + + return i; +} + +static int emit_bb_start(u64 batch_addr, u32 ppgtt_flag, u32 *dw, int i) +{ + dw[i++] = MI_BATCH_BUFFER_START_GEN8 | ppgtt_flag; + dw[i++] = lower_32_bits(batch_addr); + dw[i++] = upper_32_bits(batch_addr); + + return i; +} + +static int emit_flush_invalidate(u32 flag, u32 *dw, int i) +{ + dw[i] = MI_FLUSH_DW + 1; + dw[i] |= flag; + dw[i++] |= MI_INVALIDATE_TLB | MI_FLUSH_DW_OP_STOREDW | + MI_FLUSH_DW_STORE_INDEX; + + dw[i++] = LRC_PPHWSP_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT; + dw[i++] = 0; + dw[i++] = ~0U; + + return i; +} + +static int emit_pipe_invalidate(u32 mask_flags, u32 *dw, int i) +{ + u32 flags = PIPE_CONTROL_CS_STALL | + PIPE_CONTROL_COMMAND_CACHE_INVALIDATE | + PIPE_CONTROL_TLB_INVALIDATE | + PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE | + PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | + PIPE_CONTROL_VF_CACHE_INVALIDATE | + PIPE_CONTROL_CONST_CACHE_INVALIDATE | + PIPE_CONTROL_STATE_CACHE_INVALIDATE | + PIPE_CONTROL_QW_WRITE | + PIPE_CONTROL_STORE_DATA_INDEX; + + flags &= ~mask_flags; + + dw[i++] = GFX_OP_PIPE_CONTROL(6); + dw[i++] = flags; + dw[i++] = LRC_PPHWSP_SCRATCH_ADDR; + dw[i++] = 0; + dw[i++] = 0; + dw[i++] = 0; + + return i; +} + +#define MI_STORE_QWORD_IMM_GEN8_POSTED (MI_INSTR(0x20, 3) | (1 << 21)) + +static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, + u32 *dw, int i) +{ + dw[i++] = MI_STORE_QWORD_IMM_GEN8_POSTED; + dw[i++] = lower_32_bits(addr); + dw[i++] = upper_32_bits(addr); + dw[i++] = lower_32_bits(value); + dw[i++] = upper_32_bits(value); + + return i; +} + +static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw, + int i) +{ + dw[i++] = GFX_OP_PIPE_CONTROL(6); + dw[i++] = (stall_only ? PIPE_CONTROL_CS_STALL : + PIPE_CONTROL_FLUSH_ENABLE | PIPE_CONTROL_CS_STALL) | + PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE; + dw[i++] = addr; + dw[i++] = 0; + dw[i++] = value; + dw[i++] = 0; /* We're thrashing one extra dword. */ + + return i; +} + +static u32 get_ppgtt_flag(struct xe_sched_job *job) +{ + return !(job->engine->flags & ENGINE_FLAG_WA) ? BIT(8) : 0; +} + +static void __emit_job_gen12_copy(struct xe_sched_job *job, struct xe_lrc *lrc, + u64 batch_addr, u32 seqno) +{ + u32 dw[MAX_JOB_SIZE_DW], i = 0; + u32 ppgtt_flag = get_ppgtt_flag(job); + + /* XXX: Conditional flushing possible */ + dw[i++] = preparser_disable(true); + i = emit_flush_invalidate(0, dw, i); + dw[i++] = preparser_disable(false); + + i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), + seqno, dw, i); + + i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + + if (job->user_fence.used) + i = emit_store_imm_ppgtt_posted(job->user_fence.addr, + job->user_fence.value, + dw, i); + + i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, dw, i); + + i = emit_user_interrupt(dw, i); + + XE_BUG_ON(i > MAX_JOB_SIZE_DW); + + xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); +} + +static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, + u64 batch_addr, u32 seqno) +{ + u32 dw[MAX_JOB_SIZE_DW], i = 0; + u32 ppgtt_flag = get_ppgtt_flag(job); + struct xe_gt *gt = job->engine->gt; + struct xe_device *xe = gt_to_xe(gt); + bool decode = job->engine->class == XE_ENGINE_CLASS_VIDEO_DECODE; + + /* XXX: Conditional flushing possible */ + dw[i++] = preparser_disable(true); + i = emit_flush_invalidate(decode ? MI_INVALIDATE_BSD : 0, dw, i); + /* Wa_1809175790 */ + if (!xe->info.has_flat_ccs) { + if (decode) + i = emit_aux_table_inv(gt, GEN12_VD0_AUX_INV.reg, dw, i); + else + i = emit_aux_table_inv(gt, GEN12_VE0_AUX_INV.reg, dw, i); + } + dw[i++] = preparser_disable(false); + + i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), + seqno, dw, i); + + i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + + if (job->user_fence.used) + i = emit_store_imm_ppgtt_posted(job->user_fence.addr, + job->user_fence.value, + dw, i); + + i = emit_flush_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, dw, i); + + i = emit_user_interrupt(dw, i); + + XE_BUG_ON(i > MAX_JOB_SIZE_DW); + + xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); +} + +/* + * 3D-related flags that can't be set on _engines_ that lack access to the 3D + * pipeline (i.e., CCS engines). + */ +#define PIPE_CONTROL_3D_ENGINE_FLAGS (\ + PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH | \ + PIPE_CONTROL_DEPTH_CACHE_FLUSH | \ + PIPE_CONTROL_TILE_CACHE_FLUSH | \ + PIPE_CONTROL_DEPTH_STALL | \ + PIPE_CONTROL_STALL_AT_SCOREBOARD | \ + PIPE_CONTROL_PSD_SYNC | \ + PIPE_CONTROL_AMFS_FLUSH | \ + PIPE_CONTROL_VF_CACHE_INVALIDATE | \ + PIPE_CONTROL_GLOBAL_SNAPSHOT_RESET) + +/* 3D-related flags that can't be set on _platforms_ that lack a 3D pipeline */ +#define PIPE_CONTROL_3D_ARCH_FLAGS ( \ + PIPE_CONTROL_3D_ENGINE_FLAGS | \ + PIPE_CONTROL_INDIRECT_STATE_DISABLE | \ + PIPE_CONTROL_FLUSH_ENABLE | \ + PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE | \ + PIPE_CONTROL_DC_FLUSH_ENABLE) + +static void __emit_job_gen12_render_compute(struct xe_sched_job *job, + struct xe_lrc *lrc, + u64 batch_addr, u32 seqno) +{ + u32 dw[MAX_JOB_SIZE_DW], i = 0; + u32 ppgtt_flag = get_ppgtt_flag(job); + struct xe_gt *gt = job->engine->gt; + struct xe_device *xe = gt_to_xe(gt); + bool pvc = xe->info.platform == XE_PVC; + u32 mask_flags = 0; + + /* XXX: Conditional flushing possible */ + dw[i++] = preparser_disable(true); + if (pvc) + mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS; + else if (job->engine->class == XE_ENGINE_CLASS_COMPUTE) + mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; + i = emit_pipe_invalidate(mask_flags, dw, i); + /* Wa_1809175790 */ + if (!xe->info.has_flat_ccs) + i = emit_aux_table_inv(gt, GEN12_CCS_AUX_INV.reg, dw, i); + dw[i++] = preparser_disable(false); + + i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), + seqno, dw, i); + + i = emit_bb_start(batch_addr, ppgtt_flag, dw, i); + + if (job->user_fence.used) + i = emit_store_imm_ppgtt_posted(job->user_fence.addr, + job->user_fence.value, + dw, i); + + i = emit_pipe_imm_ggtt(xe_lrc_seqno_ggtt_addr(lrc), seqno, pvc, dw, i); + + i = emit_user_interrupt(dw, i); + + XE_BUG_ON(i > MAX_JOB_SIZE_DW); + + xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); +} + +static void emit_migration_job_gen12(struct xe_sched_job *job, + struct xe_lrc *lrc, u32 seqno) +{ + u32 dw[MAX_JOB_SIZE_DW], i = 0; + + i = emit_store_imm_ggtt(xe_lrc_start_seqno_ggtt_addr(lrc), + seqno, dw, i); + + i = emit_bb_start(job->batch_addr[0], BIT(8), dw, i); + + dw[i++] = preparser_disable(true); + i = emit_flush_invalidate(0, dw, i); + dw[i++] = preparser_disable(false); + + i = emit_bb_start(job->batch_addr[1], BIT(8), dw, i); + + dw[i++] = (MI_FLUSH_DW | MI_INVALIDATE_TLB | job->migrate_flush_flags | + MI_FLUSH_DW_OP_STOREDW) + 1; + dw[i++] = xe_lrc_seqno_ggtt_addr(lrc) | MI_FLUSH_DW_USE_GTT; + dw[i++] = 0; + dw[i++] = seqno; /* value */ + + i = emit_user_interrupt(dw, i); + + XE_BUG_ON(i > MAX_JOB_SIZE_DW); + + xe_lrc_write_ring(lrc, dw, i * sizeof(*dw)); +} + +static void emit_job_gen12_copy(struct xe_sched_job *job) +{ + int i; + + if (xe_sched_job_is_migration(job->engine)) { + emit_migration_job_gen12(job, job->engine->lrc, + xe_sched_job_seqno(job)); + return; + } + + for (i = 0; i < job->engine->width; ++i) + __emit_job_gen12_copy(job, job->engine->lrc + i, + job->batch_addr[i], + xe_sched_job_seqno(job)); +} + +static void emit_job_gen12_video(struct xe_sched_job *job) +{ + int i; + + /* FIXME: Not doing parallel handshake for now */ + for (i = 0; i < job->engine->width; ++i) + __emit_job_gen12_video(job, job->engine->lrc + i, + job->batch_addr[i], + xe_sched_job_seqno(job)); +} + +static void emit_job_gen12_render_compute(struct xe_sched_job *job) +{ + int i; + + for (i = 0; i < job->engine->width; ++i) + __emit_job_gen12_render_compute(job, job->engine->lrc + i, + job->batch_addr[i], + xe_sched_job_seqno(job)); +} + +static const struct xe_ring_ops ring_ops_gen12_copy = { + .emit_job = emit_job_gen12_copy, +}; + +static const struct xe_ring_ops ring_ops_gen12_video = { + .emit_job = emit_job_gen12_video, +}; + +static const struct xe_ring_ops ring_ops_gen12_render_compute = { + .emit_job = emit_job_gen12_render_compute, +}; + +const struct xe_ring_ops * +xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class) +{ + switch (class) { + case XE_ENGINE_CLASS_COPY: + return &ring_ops_gen12_copy; + case XE_ENGINE_CLASS_VIDEO_DECODE: + case XE_ENGINE_CLASS_VIDEO_ENHANCE: + return &ring_ops_gen12_video; + case XE_ENGINE_CLASS_RENDER: + case XE_ENGINE_CLASS_COMPUTE: + return &ring_ops_gen12_render_compute; + default: + return NULL; + } +} diff --git a/drivers/gpu/drm/xe/xe_ring_ops.h b/drivers/gpu/drm/xe/xe_ring_ops.h new file mode 100644 index 000000000000..e942735d76a6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ring_ops.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_RING_OPS_H_ +#define _XE_RING_OPS_H_ + +#include "xe_hw_engine_types.h" +#include "xe_ring_ops_types.h" + +struct xe_gt; + +const struct xe_ring_ops * +xe_ring_ops_get(struct xe_gt *gt, enum xe_engine_class class); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ring_ops_types.h b/drivers/gpu/drm/xe/xe_ring_ops_types.h new file mode 100644 index 000000000000..1ae56e2ee7b4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ring_ops_types.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_RING_OPS_TYPES_H_ +#define _XE_RING_OPS_TYPES_H_ + +struct xe_sched_job; + +#define MAX_JOB_SIZE_DW 48 +#define MAX_JOB_SIZE_BYTES (MAX_JOB_SIZE_DW * 4) + +/** + * struct xe_ring_ops - Ring operations + */ +struct xe_ring_ops { + /** @emit_job: Write job to ring */ + void (*emit_job)(struct xe_sched_job *job); +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_rtp.c b/drivers/gpu/drm/xe/xe_rtp.c new file mode 100644 index 000000000000..9e8d0e43c643 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_rtp.c @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_rtp.h" + +#include + +#include "xe_gt.h" +#include "xe_macros.h" +#include "xe_reg_sr.h" + +/** + * DOC: Register Table Processing + * + * Internal infrastructure to define how registers should be updated based on + * rules and actions. This can be used to define tables with multiple entries + * (one per register) that will be walked over at some point in time to apply + * the values to the registers that have matching rules. + */ + +static bool rule_matches(struct xe_gt *gt, + struct xe_hw_engine *hwe, + const struct xe_rtp_entry *entry) +{ + const struct xe_device *xe = gt_to_xe(gt); + const struct xe_rtp_rule *r; + unsigned int i; + bool match; + + for (r = entry->rules, i = 0; i < entry->n_rules; + r = &entry->rules[++i]) { + switch (r->match_type) { + case XE_RTP_MATCH_PLATFORM: + match = xe->info.platform == r->platform; + break; + case XE_RTP_MATCH_SUBPLATFORM: + match = xe->info.platform == r->platform && + xe->info.subplatform == r->subplatform; + break; + case XE_RTP_MATCH_GRAPHICS_VERSION: + /* TODO: match display */ + match = xe->info.graphics_verx100 == r->ver_start; + break; + case XE_RTP_MATCH_GRAPHICS_VERSION_RANGE: + match = xe->info.graphics_verx100 >= r->ver_start && + xe->info.graphics_verx100 <= r->ver_end; + break; + case XE_RTP_MATCH_MEDIA_VERSION: + match = xe->info.media_verx100 == r->ver_start; + break; + case XE_RTP_MATCH_MEDIA_VERSION_RANGE: + match = xe->info.media_verx100 >= r->ver_start && + xe->info.media_verx100 <= r->ver_end; + break; + case XE_RTP_MATCH_STEP: + /* TODO: match media/display */ + match = xe->info.step.graphics >= r->step_start && + xe->info.step.graphics < r->step_end; + break; + case XE_RTP_MATCH_ENGINE_CLASS: + match = hwe->class == r->engine_class; + break; + case XE_RTP_MATCH_NOT_ENGINE_CLASS: + match = hwe->class != r->engine_class; + break; + case XE_RTP_MATCH_FUNC: + match = r->match_func(gt, hwe); + break; + case XE_RTP_MATCH_INTEGRATED: + match = !xe->info.is_dgfx; + break; + case XE_RTP_MATCH_DISCRETE: + match = xe->info.is_dgfx; + break; + + default: + XE_WARN_ON(r->match_type); + } + + if (!match) + return false; + } + + return true; +} + +static void rtp_add_sr_entry(const struct xe_rtp_entry *entry, + struct xe_gt *gt, + u32 mmio_base, + struct xe_reg_sr *sr) +{ + u32 reg = entry->regval.reg + mmio_base; + struct xe_reg_sr_entry sr_entry = { + .clr_bits = entry->regval.clr_bits, + .set_bits = entry->regval.set_bits, + .read_mask = entry->regval.read_mask, + .masked_reg = entry->regval.flags & XE_RTP_FLAG_MASKED_REG, + .reg_type = entry->regval.reg_type, + }; + + xe_reg_sr_add(sr, reg, &sr_entry); +} + +/** + * xe_rtp_process - Process all rtp @entries, adding the matching ones to @sr + * @entries: Table with RTP definitions + * @sr: Where to add an entry to with the values for matching. This can be + * viewed as the "coalesced view" of multiple the tables. The bits for each + * register set are expected not to collide with previously added entries + * @gt: The GT to be used for matching rules + * @hwe: Engine instance to use for matching rules and as mmio base + * + * Walk the table pointed by @entries (with an empty sentinel) and add all + * entries with matching rules to @sr. If @hwe is not NULL, its mmio_base is + * used to calculate the right register offset + */ +void xe_rtp_process(const struct xe_rtp_entry *entries, struct xe_reg_sr *sr, + struct xe_gt *gt, struct xe_hw_engine *hwe) +{ + const struct xe_rtp_entry *entry; + + for (entry = entries; entry && entry->name; entry++) { + u32 mmio_base = 0; + + if (entry->regval.flags & XE_RTP_FLAG_FOREACH_ENGINE) { + struct xe_hw_engine *each_hwe; + enum xe_hw_engine_id id; + + for_each_hw_engine(each_hwe, gt, id) { + mmio_base = each_hwe->mmio_base; + + if (rule_matches(gt, each_hwe, entry)) + rtp_add_sr_entry(entry, gt, mmio_base, sr); + } + } else if (rule_matches(gt, hwe, entry)) { + if (entry->regval.flags & XE_RTP_FLAG_ENGINE_BASE) + mmio_base = hwe->mmio_base; + + rtp_add_sr_entry(entry, gt, mmio_base, sr); + } + } +} diff --git a/drivers/gpu/drm/xe/xe_rtp.h b/drivers/gpu/drm/xe/xe_rtp.h new file mode 100644 index 000000000000..d4e11fdde77f --- /dev/null +++ b/drivers/gpu/drm/xe/xe_rtp.h @@ -0,0 +1,340 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_RTP_ +#define _XE_RTP_ + +#include +#include + +#include "xe_rtp_types.h" + +#include "i915_reg_defs.h" + +/* + * Register table poke infrastructure + */ + +struct xe_hw_engine; +struct xe_gt; +struct xe_reg_sr; + +/* + * Helper macros - not to be used outside this header. + */ +/* This counts to 12. Any more, it will return 13th argument. */ +#define __COUNT_ARGS(_0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _n, X...) _n +#define COUNT_ARGS(X...) __COUNT_ARGS(, ##X, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) + +#define __CONCAT(a, b) a ## b +#define CONCATENATE(a, b) __CONCAT(a, b) + +#define __CALL_FOR_EACH_1(MACRO_, x, ...) MACRO_(x) +#define __CALL_FOR_EACH_2(MACRO_, x, ...) \ + MACRO_(x) __CALL_FOR_EACH_1(MACRO_, ##__VA_ARGS__) +#define __CALL_FOR_EACH_3(MACRO_, x, ...) \ + MACRO_(x) __CALL_FOR_EACH_2(MACRO_, ##__VA_ARGS__) +#define __CALL_FOR_EACH_4(MACRO_, x, ...) \ + MACRO_(x) __CALL_FOR_EACH_3(MACRO_, ##__VA_ARGS__) + +#define _CALL_FOR_EACH(NARGS_, MACRO_, x, ...) \ + CONCATENATE(__CALL_FOR_EACH_, NARGS_)(MACRO_, x, ##__VA_ARGS__) +#define CALL_FOR_EACH(MACRO_, x, ...) \ + _CALL_FOR_EACH(COUNT_ARGS(x, ##__VA_ARGS__), MACRO_, x, ##__VA_ARGS__) + +#define _XE_RTP_REG(x_) (x_), \ + .reg_type = XE_RTP_REG_REGULAR +#define _XE_RTP_MCR_REG(x_) (x_), \ + .reg_type = XE_RTP_REG_MCR + +/* + * Helper macros for concatenating prefix - do not use them directly outside + * this header + */ +#define __ADD_XE_RTP_FLAG_PREFIX(x) CONCATENATE(XE_RTP_FLAG_, x) | +#define __ADD_XE_RTP_RULE_PREFIX(x) CONCATENATE(XE_RTP_RULE_, x) , + +/* + * Macros to encode rules to match against platform, IP version, stepping, etc. + * Shouldn't be used directly - see XE_RTP_RULES() + */ + +#define _XE_RTP_RULE_PLATFORM(plat__) \ + { .match_type = XE_RTP_MATCH_PLATFORM, .platform = plat__ } + +#define _XE_RTP_RULE_SUBPLATFORM(plat__, sub__) \ + { .match_type = XE_RTP_MATCH_SUBPLATFORM, \ + .platform = plat__, .subplatform = sub__ } + +#define _XE_RTP_RULE_STEP(start__, end__) \ + { .match_type = XE_RTP_MATCH_STEP, \ + .step_start = start__, .step_end = end__ } + +#define _XE_RTP_RULE_ENGINE_CLASS(cls__) \ + { .match_type = XE_RTP_MATCH_ENGINE_CLASS, \ + .engine_class = (cls__) } + +/** + * XE_RTP_RULE_PLATFORM - Create rule matching platform + * @plat_: platform to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_PLATFORM(plat_) \ + _XE_RTP_RULE_PLATFORM(XE_##plat_) + +/** + * XE_RTP_RULE_SUBPLATFORM - Create rule matching platform and sub-platform + * @plat_: platform to match + * @sub_: sub-platform to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_SUBPLATFORM(plat_, sub_) \ + _XE_RTP_RULE_SUBPLATFORM(XE_##plat_, XE_SUBPLATFORM_##plat_##_##sub_) + +/** + * XE_RTP_RULE_STEP - Create rule matching platform stepping + * @start_: First stepping matching the rule + * @end_: First stepping that does not match the rule + * + * Note that the range matching this rule [ @start_, @end_ ), i.e. inclusive on + * the left, exclusive on the right. + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_STEP(start_, end_) \ + _XE_RTP_RULE_STEP(STEP_##start_, STEP_##end_) + +/** + * XE_RTP_RULE_ENGINE_CLASS - Create rule matching an engine class + * @cls_: Engine class to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_ENGINE_CLASS(cls_) \ + _XE_RTP_RULE_ENGINE_CLASS(XE_ENGINE_CLASS_##cls_) + +/** + * XE_RTP_RULE_FUNC - Create rule using callback function for match + * @func__: Function to call to decide if rule matches + * + * This allows more complex checks to be performed. The ``XE_RTP`` + * infrastructure will simply call the function @func_ passed to decide if this + * rule matches the device. + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_FUNC(func__) \ + { .match_type = XE_RTP_MATCH_FUNC, \ + .match_func = (func__) } + +/** + * XE_RTP_RULE_GRAPHICS_VERSION - Create rule matching graphics version + * @ver__: Graphics IP version to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_GRAPHICS_VERSION(ver__) \ + { .match_type = XE_RTP_MATCH_GRAPHICS_VERSION, \ + .ver_start = ver__, } + +/** + * XE_RTP_RULE_GRAPHICS_VERSION_RANGE - Create rule matching a range of graphics version + * @ver_start__: First graphics IP version to match + * @ver_end__: Last graphics IP version to match + * + * Note that the range matching this rule is [ @ver_start__, @ver_end__ ], i.e. + * inclusive on boths sides + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_GRAPHICS_VERSION_RANGE(ver_start__, ver_end__) \ + { .match_type = XE_RTP_MATCH_GRAPHICS_VERSION_RANGE, \ + .ver_start = ver_start__, .ver_end = ver_end__, } + +/** + * XE_RTP_RULE_MEDIA_VERSION - Create rule matching media version + * @ver__: Graphics IP version to match + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_MEDIA_VERSION(ver__) \ + { .match_type = XE_RTP_MATCH_MEDIA_VERSION, \ + .ver_start = ver__, } + +/** + * XE_RTP_RULE_MEDIA_VERSION_RANGE - Create rule matching a range of media version + * @ver_start__: First media IP version to match + * @ver_end__: Last media IP version to match + * + * Note that the range matching this rule is [ @ver_start__, @ver_end__ ], i.e. + * inclusive on boths sides + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_MEDIA_VERSION_RANGE(ver_start__, ver_end__) \ + { .match_type = XE_RTP_MATCH_MEDIA_VERSION_RANGE, \ + .ver_start = ver_start__, .ver_end = ver_end__, } + +/** + * XE_RTP_RULE_IS_INTEGRATED - Create a rule matching integrated graphics devices + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_IS_INTEGRATED \ + { .match_type = XE_RTP_MATCH_INTEGRATED } + +/** + * XE_RTP_RULE_IS_DISCRETE - Create a rule matching discrete graphics devices + * + * Refer to XE_RTP_RULES() for expected usage. + */ +#define XE_RTP_RULE_IS_DISCRETE \ + { .match_type = XE_RTP_MATCH_DISCRETE } + +/** + * XE_RTP_WR - Helper to write a value to the register, overriding all the bits + * @reg_: Register + * @val_: Value to set + * @...: Additional fields to override in the struct xe_rtp_regval entry + * + * The correspondent notation in bspec is: + * + * REGNAME = VALUE + */ +#define XE_RTP_WR(reg_, val_, ...) \ + .regval = { .reg = reg_, .clr_bits = ~0u, .set_bits = (val_), \ + .read_mask = (~0u), ##__VA_ARGS__ } + +/** + * XE_RTP_SET - Set bits from @val_ in the register. + * @reg_: Register + * @val_: Bits to set in the register + * @...: Additional fields to override in the struct xe_rtp_regval entry + * + * For masked registers this translates to a single write, while for other + * registers it's a RMW. The correspondent bspec notation is (example for bits 2 + * and 5, but could be any): + * + * REGNAME[2] = 1 + * REGNAME[5] = 1 + */ +#define XE_RTP_SET(reg_, val_, ...) \ + .regval = { .reg = reg_, .clr_bits = (val_), .set_bits = (val_), \ + .read_mask = (val_), ##__VA_ARGS__ } + +/** + * XE_RTP_CLR: Clear bits from @val_ in the register. + * @reg_: Register + * @val_: Bits to clear in the register + * @...: Additional fields to override in the struct xe_rtp_regval entry + * + * For masked registers this translates to a single write, while for other + * registers it's a RMW. The correspondent bspec notation is (example for bits 2 + * and 5, but could be any): + * + * REGNAME[2] = 0 + * REGNAME[5] = 0 + */ +#define XE_RTP_CLR(reg_, val_, ...) \ + .regval = { .reg = reg_, .clr_bits = (val_), .set_bits = 0, \ + .read_mask = (val_), ##__VA_ARGS__ } + +/** + * XE_RTP_FIELD_SET: Set a bit range, defined by @mask_bits_, to the value in + * @reg_: Register + * @mask_bits_: Mask of bits to be changed in the register, forming a field + * @val_: Value to set in the field denoted by @mask_bits_ + * @...: Additional fields to override in the struct xe_rtp_regval entry + * + * For masked registers this translates to a single write, while for other + * registers it's a RMW. The correspondent bspec notation is: + * + * REGNAME[:] = VALUE + */ +#define XE_RTP_FIELD_SET(reg_, mask_bits_, val_, ...) \ + .regval = { .reg = reg_, .clr_bits = (mask_bits_), .set_bits = (val_),\ + .read_mask = (mask_bits_), ##__VA_ARGS__ } + +#define XE_RTP_FIELD_SET_NO_READ_MASK(reg_, mask_bits_, val_, ...) \ + .regval = { .reg = reg_, .clr_bits = (mask_bits_), .set_bits = (val_),\ + .read_mask = 0, ##__VA_ARGS__ } + +/** + * XE_WHITELIST_REGISTER - Add register to userspace whitelist + * @reg_: Register + * @flags_: Whitelist-specific flags to set + * @...: Additional fields to override in the struct xe_rtp_regval entry + * + * Add a register to the whitelist, allowing userspace to modify the ster with + * regular user privileges. + */ +#define XE_WHITELIST_REGISTER(reg_, flags_, ...) \ + /* TODO fail build if ((flags) & ~(RING_FORCE_TO_NONPRIV_MASK_VALID)) */\ + .regval = { .reg = reg_, .set_bits = (flags_), \ + .clr_bits = RING_FORCE_TO_NONPRIV_MASK_VALID, \ + ##__VA_ARGS__ } + +/** + * XE_RTP_NAME - Helper to set the name in xe_rtp_entry + * @s_: Name describing this rule, often a HW-specific number + * + * TODO: maybe move this behind a debug config? + */ +#define XE_RTP_NAME(s_) .name = (s_) + +/** + * XE_RTP_FLAG - Helper to add multiple flags to a struct xe_rtp_regval entry + * @f1_: Last part of a ``XE_RTP_FLAG_*`` + * @...: Additional flags, defined like @f1_ + * + * Helper to automatically add a ``XE_RTP_FLAG_`` prefix to @f1_ so it can be + * easily used to define struct xe_rtp_regval entries. Example: + * + * .. code-block:: c + * + * const struct xe_rtp_entry wa_entries[] = { + * ... + * { XE_RTP_NAME("test-entry"), + * XE_RTP_FLAG(FOREACH_ENGINE, MASKED_REG), + * ... + * }, + * ... + * }; + */ +#define XE_RTP_FLAG(f1_, ...) \ + .flags = (CALL_FOR_EACH(__ADD_XE_RTP_FLAG_PREFIX, f1_, ##__VA_ARGS__) 0) + +/** + * XE_RTP_RULES - Helper to set multiple rules to a struct xe_rtp_entry entry + * @r1: Last part of XE_RTP_MATCH_* + * @...: Additional rules, defined like @r1 + * + * At least one rule is needed and up to 4 are supported. Multiple rules are + * AND'ed together, i.e. all the rules must evaluate to true for the entry to + * be processed. See XE_RTP_MATCH_* for the possible match rules. Example: + * + * .. code-block:: c + * + * const struct xe_rtp_entry wa_entries[] = { + * ... + * { XE_RTP_NAME("test-entry"), + * XE_RTP_RULES(SUBPLATFORM(DG2, G10), STEP(A0, B0)), + * ... + * }, + * ... + * }; + */ +#define XE_RTP_RULES(r1, ...) \ + .n_rules = COUNT_ARGS(r1, ##__VA_ARGS__), \ + .rules = (struct xe_rtp_rule[]) { \ + CALL_FOR_EACH(__ADD_XE_RTP_RULE_PREFIX, r1, ##__VA_ARGS__) \ + } + +void xe_rtp_process(const struct xe_rtp_entry *entries, struct xe_reg_sr *sr, + struct xe_gt *gt, struct xe_hw_engine *hwe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_rtp_types.h b/drivers/gpu/drm/xe/xe_rtp_types.h new file mode 100644 index 000000000000..b55b556a2495 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_rtp_types.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_RTP_TYPES_ +#define _XE_RTP_TYPES_ + +#include + +#include "i915_reg_defs.h" + +struct xe_hw_engine; +struct xe_gt; + +enum { + XE_RTP_REG_REGULAR, + XE_RTP_REG_MCR, +}; + +/** + * struct xe_rtp_regval - register and value for rtp table + */ +struct xe_rtp_regval { + /** @reg: Register */ + u32 reg; + /* + * TODO: maybe we need a union here with a func pointer for cases + * that are too specific to be generalized + */ + /** @clr_bits: bits to clear when updating register */ + u32 clr_bits; + /** @set_bits: bits to set when updating register */ + u32 set_bits; +#define XE_RTP_NOCHECK .read_mask = 0 + /** @read_mask: mask for bits to consider when reading value back */ + u32 read_mask; +#define XE_RTP_FLAG_FOREACH_ENGINE BIT(0) +#define XE_RTP_FLAG_MASKED_REG BIT(1) +#define XE_RTP_FLAG_ENGINE_BASE BIT(2) + /** @flags: flags to apply on rule evaluation or action */ + u8 flags; + /** @reg_type: register type, see ``XE_RTP_REG_*`` */ + u8 reg_type; +}; + +enum { + XE_RTP_MATCH_PLATFORM, + XE_RTP_MATCH_SUBPLATFORM, + XE_RTP_MATCH_GRAPHICS_VERSION, + XE_RTP_MATCH_GRAPHICS_VERSION_RANGE, + XE_RTP_MATCH_MEDIA_VERSION, + XE_RTP_MATCH_MEDIA_VERSION_RANGE, + XE_RTP_MATCH_INTEGRATED, + XE_RTP_MATCH_DISCRETE, + XE_RTP_MATCH_STEP, + XE_RTP_MATCH_ENGINE_CLASS, + XE_RTP_MATCH_NOT_ENGINE_CLASS, + XE_RTP_MATCH_FUNC, +}; + +/** struct xe_rtp_rule - match rule for processing entry */ +struct xe_rtp_rule { + u8 match_type; + + /* match filters */ + union { + /* MATCH_PLATFORM / MATCH_SUBPLATFORM */ + struct { + u8 platform; + u8 subplatform; + }; + /* + * MATCH_GRAPHICS_VERSION / XE_RTP_MATCH_GRAPHICS_VERSION_RANGE / + * MATCH_MEDIA_VERSION / XE_RTP_MATCH_MEDIA_VERSION_RANGE + */ + struct { + u32 ver_start; +#define XE_RTP_END_VERSION_UNDEFINED U32_MAX + u32 ver_end; + }; + /* MATCH_STEP */ + struct { + u8 step_start; + u8 step_end; + }; + /* MATCH_ENGINE_CLASS / MATCH_NOT_ENGINE_CLASS */ + struct { + u8 engine_class; + }; + /* MATCH_FUNC */ + bool (*match_func)(const struct xe_gt *gt, + const struct xe_hw_engine *hwe); + }; +}; + +/** struct xe_rtp_entry - Entry in an rtp table */ +struct xe_rtp_entry { + const char *name; + const struct xe_rtp_regval regval; + const struct xe_rtp_rule *rules; + unsigned int n_rules; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c new file mode 100644 index 000000000000..7403410cd806 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sa.c @@ -0,0 +1,96 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_map.h" +#include "xe_sa.h" + +static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg) +{ + struct xe_sa_manager *sa_manager = arg; + struct xe_bo *bo = sa_manager->bo; + + if (!bo) { + drm_err(drm, "no bo for sa manager\n"); + return; + } + + drm_suballoc_manager_fini(&sa_manager->base); + + if (bo->vmap.is_iomem) + kvfree(sa_manager->cpu_ptr); + + xe_bo_unpin_map_no_vm(bo); + sa_manager->bo = NULL; +} + +int xe_sa_bo_manager_init(struct xe_gt *gt, + struct xe_sa_manager *sa_manager, + u32 size, u32 align) +{ + struct xe_device *xe = gt_to_xe(gt); + u32 managed_size = size - SZ_4K; + struct xe_bo *bo; + + sa_manager->bo = NULL; + + bo = xe_bo_create_pin_map(xe, gt, NULL, size, ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_GGTT_BIT); + if (IS_ERR(bo)) { + drm_err(&xe->drm, "failed to allocate bo for sa manager: %ld\n", + PTR_ERR(bo)); + return PTR_ERR(bo); + } + sa_manager->bo = bo; + + drm_suballoc_manager_init(&sa_manager->base, managed_size, align); + sa_manager->gpu_addr = xe_bo_ggtt_addr(bo); + + if (bo->vmap.is_iomem) { + sa_manager->cpu_ptr = kvzalloc(managed_size, GFP_KERNEL); + if (!sa_manager->cpu_ptr) { + xe_bo_unpin_map_no_vm(sa_manager->bo); + sa_manager->bo = NULL; + return -ENOMEM; + } + } else { + sa_manager->cpu_ptr = bo->vmap.vaddr; + memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size); + } + + return drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini, + sa_manager); +} + +struct drm_suballoc *xe_sa_bo_new(struct xe_sa_manager *sa_manager, + unsigned size) +{ + return drm_suballoc_new(&sa_manager->base, size, GFP_KERNEL, true, 0); +} + +void xe_sa_bo_flush_write(struct drm_suballoc *sa_bo) +{ + struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); + struct xe_device *xe = gt_to_xe(sa_manager->bo->gt); + + if (!sa_manager->bo->vmap.is_iomem) + return; + + xe_map_memcpy_to(xe, &sa_manager->bo->vmap, drm_suballoc_soffset(sa_bo), + xe_sa_bo_cpu_addr(sa_bo), + drm_suballoc_size(sa_bo)); +} + +void xe_sa_bo_free(struct drm_suballoc *sa_bo, + struct dma_fence *fence) +{ + drm_suballoc_free(sa_bo, fence); +} diff --git a/drivers/gpu/drm/xe/xe_sa.h b/drivers/gpu/drm/xe/xe_sa.h new file mode 100644 index 000000000000..742282ef7179 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sa.h @@ -0,0 +1,42 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ +#ifndef _XE_SA_H_ +#define _XE_SA_H_ + +#include "xe_sa_types.h" + +struct dma_fence; +struct xe_bo; +struct xe_gt; + +int xe_sa_bo_manager_init(struct xe_gt *gt, + struct xe_sa_manager *sa_manager, + u32 size, u32 align); + +struct drm_suballoc *xe_sa_bo_new(struct xe_sa_manager *sa_manager, + u32 size); +void xe_sa_bo_flush_write(struct drm_suballoc *sa_bo); +void xe_sa_bo_free(struct drm_suballoc *sa_bo, + struct dma_fence *fence); + +static inline struct xe_sa_manager * +to_xe_sa_manager(struct drm_suballoc_manager *mng) +{ + return container_of(mng, struct xe_sa_manager, base); +} + +static inline u64 xe_sa_bo_gpu_addr(struct drm_suballoc *sa) +{ + return to_xe_sa_manager(sa->manager)->gpu_addr + + drm_suballoc_soffset(sa); +} + +static inline void *xe_sa_bo_cpu_addr(struct drm_suballoc *sa) +{ + return to_xe_sa_manager(sa->manager)->cpu_ptr + + drm_suballoc_soffset(sa); +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_sa_types.h b/drivers/gpu/drm/xe/xe_sa_types.h new file mode 100644 index 000000000000..2ef896aeca1d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sa_types.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ +#ifndef _XE_SA_TYPES_H_ +#define _XE_SA_TYPES_H_ + +#include + +struct xe_bo; + +struct xe_sa_manager { + struct drm_suballoc_manager base; + struct xe_bo *bo; + u64 gpu_addr; + void *cpu_ptr; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c new file mode 100644 index 000000000000..ab81bfe17e8a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sched_job.c @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_sched_job.h" + +#include +#include + +#include "xe_device_types.h" +#include "xe_engine.h" +#include "xe_gt.h" +#include "xe_hw_engine_types.h" +#include "xe_hw_fence.h" +#include "xe_lrc.h" +#include "xe_macros.h" +#include "xe_trace.h" +#include "xe_vm.h" + +static struct kmem_cache *xe_sched_job_slab; +static struct kmem_cache *xe_sched_job_parallel_slab; + +int __init xe_sched_job_module_init(void) +{ + xe_sched_job_slab = + kmem_cache_create("xe_sched_job", + sizeof(struct xe_sched_job) + + sizeof(u64), 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!xe_sched_job_slab) + return -ENOMEM; + + xe_sched_job_parallel_slab = + kmem_cache_create("xe_sched_job_parallel", + sizeof(struct xe_sched_job) + + sizeof(u64) * + XE_HW_ENGINE_MAX_INSTANCE , 0, + SLAB_HWCACHE_ALIGN, NULL); + if (!xe_sched_job_parallel_slab) { + kmem_cache_destroy(xe_sched_job_slab); + return -ENOMEM; + } + + return 0; +} + +void xe_sched_job_module_exit(void) +{ + kmem_cache_destroy(xe_sched_job_slab); + kmem_cache_destroy(xe_sched_job_parallel_slab); +} + +static struct xe_sched_job *job_alloc(bool parallel) +{ + return kmem_cache_zalloc(parallel ? xe_sched_job_parallel_slab : + xe_sched_job_slab, GFP_KERNEL); +} + +bool xe_sched_job_is_migration(struct xe_engine *e) +{ + return e->vm && (e->vm->flags & XE_VM_FLAG_MIGRATION) && + !(e->flags & ENGINE_FLAG_WA); +} + +static void job_free(struct xe_sched_job *job) +{ + struct xe_engine *e = job->engine; + bool is_migration = xe_sched_job_is_migration(e); + + kmem_cache_free(xe_engine_is_parallel(job->engine) || is_migration ? + xe_sched_job_parallel_slab : xe_sched_job_slab, job); +} + +struct xe_sched_job *xe_sched_job_create(struct xe_engine *e, + u64 *batch_addr) +{ + struct xe_sched_job *job; + struct dma_fence **fences; + bool is_migration = xe_sched_job_is_migration(e); + int err; + int i, j; + u32 width; + + /* Migration and kernel engines have their own locking */ + if (!(e->flags & (ENGINE_FLAG_KERNEL | ENGINE_FLAG_VM | + ENGINE_FLAG_WA))) { + lockdep_assert_held(&e->vm->lock); + if (!xe_vm_no_dma_fences(e->vm)) + xe_vm_assert_held(e->vm); + } + + job = job_alloc(xe_engine_is_parallel(e) || is_migration); + if (!job) + return ERR_PTR(-ENOMEM); + + job->engine = e; + kref_init(&job->refcount); + xe_engine_get(job->engine); + + err = drm_sched_job_init(&job->drm, e->entity, 1, NULL); + if (err) + goto err_free; + + if (!xe_engine_is_parallel(e)) { + job->fence = xe_lrc_create_seqno_fence(e->lrc); + if (IS_ERR(job->fence)) { + err = PTR_ERR(job->fence); + goto err_sched_job; + } + } else { + struct dma_fence_array *cf; + + fences = kmalloc_array(e->width, sizeof(*fences), GFP_KERNEL); + if (!fences) { + err = -ENOMEM; + goto err_sched_job; + } + + for (j = 0; j < e->width; ++j) { + fences[j] = xe_lrc_create_seqno_fence(e->lrc + j); + if (IS_ERR(fences[j])) { + err = PTR_ERR(fences[j]); + goto err_fences; + } + } + + cf = dma_fence_array_create(e->width, fences, + e->parallel.composite_fence_ctx, + e->parallel.composite_fence_seqno++, + false); + if (!cf) { + --e->parallel.composite_fence_seqno; + err = -ENOMEM; + goto err_fences; + } + + /* Sanity check */ + for (j = 0; j < e->width; ++j) + XE_BUG_ON(cf->base.seqno != fences[j]->seqno); + + job->fence = &cf->base; + } + + width = e->width; + if (is_migration) + width = 2; + + for (i = 0; i < width; ++i) + job->batch_addr[i] = batch_addr[i]; + + trace_xe_sched_job_create(job); + return job; + +err_fences: + for (j = j - 1; j >= 0; --j) { + --e->lrc[j].fence_ctx.next_seqno; + dma_fence_put(fences[j]); + } + kfree(fences); +err_sched_job: + drm_sched_job_cleanup(&job->drm); +err_free: + xe_engine_put(e); + job_free(job); + return ERR_PTR(err); +} + +/** + * xe_sched_job_destroy - Destroy XE schedule job + * @ref: reference to XE schedule job + * + * Called when ref == 0, drop a reference to job's xe_engine + fence, cleanup + * base DRM schedule job, and free memory for XE schedule job. + */ +void xe_sched_job_destroy(struct kref *ref) +{ + struct xe_sched_job *job = + container_of(ref, struct xe_sched_job, refcount); + + xe_engine_put(job->engine); + dma_fence_put(job->fence); + drm_sched_job_cleanup(&job->drm); + job_free(job); +} + +void xe_sched_job_set_error(struct xe_sched_job *job, int error) +{ + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) + return; + + dma_fence_set_error(job->fence, error); + + if (dma_fence_is_array(job->fence)) { + struct dma_fence_array *array = + to_dma_fence_array(job->fence); + struct dma_fence **child = array->fences; + unsigned int nchild = array->num_fences; + + do { + struct dma_fence *current_fence = *child++; + + if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + ¤t_fence->flags)) + continue; + dma_fence_set_error(current_fence, error); + } while (--nchild); + } + + trace_xe_sched_job_set_error(job); + + dma_fence_enable_sw_signaling(job->fence); + xe_hw_fence_irq_run(job->engine->fence_irq); +} + +bool xe_sched_job_started(struct xe_sched_job *job) +{ + struct xe_lrc *lrc = job->engine->lrc; + + return xe_lrc_start_seqno(lrc) >= xe_sched_job_seqno(job); +} + +bool xe_sched_job_completed(struct xe_sched_job *job) +{ + struct xe_lrc *lrc = job->engine->lrc; + + /* + * Can safely check just LRC[0] seqno as that is last seqno written when + * parallel handshake is done. + */ + + return xe_lrc_seqno(lrc) >= xe_sched_job_seqno(job); +} + +void xe_sched_job_arm(struct xe_sched_job *job) +{ + drm_sched_job_arm(&job->drm); +} + +void xe_sched_job_push(struct xe_sched_job *job) +{ + xe_sched_job_get(job); + trace_xe_sched_job_exec(job); + drm_sched_entity_push_job(&job->drm); + xe_sched_job_put(job); +} diff --git a/drivers/gpu/drm/xe/xe_sched_job.h b/drivers/gpu/drm/xe/xe_sched_job.h new file mode 100644 index 000000000000..5315ad8656c2 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sched_job.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_SCHED_JOB_H_ +#define _XE_SCHED_JOB_H_ + +#include "xe_sched_job_types.h" + +#define XE_SCHED_HANG_LIMIT 1 +#define XE_SCHED_JOB_TIMEOUT LONG_MAX + +int xe_sched_job_module_init(void); +void xe_sched_job_module_exit(void); + +struct xe_sched_job *xe_sched_job_create(struct xe_engine *e, + u64 *batch_addr); +void xe_sched_job_destroy(struct kref *ref); + +/** + * xe_sched_job_get - get reference to XE schedule job + * @job: XE schedule job object + * + * Increment XE schedule job's reference count + */ +static inline struct xe_sched_job *xe_sched_job_get(struct xe_sched_job *job) +{ + kref_get(&job->refcount); + return job; +} + +/** + * xe_sched_job_put - put reference to XE schedule job + * @job: XE schedule job object + * + * Decrement XE schedule job's reference count, call xe_sched_job_destroy when + * reference count == 0. + */ +static inline void xe_sched_job_put(struct xe_sched_job *job) +{ + kref_put(&job->refcount, xe_sched_job_destroy); +} + +void xe_sched_job_set_error(struct xe_sched_job *job, int error); +static inline bool xe_sched_job_is_error(struct xe_sched_job *job) +{ + return job->fence->error < 0; +} + +bool xe_sched_job_started(struct xe_sched_job *job); +bool xe_sched_job_completed(struct xe_sched_job *job); + +void xe_sched_job_arm(struct xe_sched_job *job); +void xe_sched_job_push(struct xe_sched_job *job); + +static inline struct xe_sched_job * +to_xe_sched_job(struct drm_sched_job *drm) +{ + return container_of(drm, struct xe_sched_job, drm); +} + +static inline u32 xe_sched_job_seqno(struct xe_sched_job *job) +{ + return job->fence->seqno; +} + +static inline void +xe_sched_job_add_migrate_flush(struct xe_sched_job *job, u32 flags) +{ + job->migrate_flush_flags = flags; +} + +bool xe_sched_job_is_migration(struct xe_engine *e); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sched_job_types.h b/drivers/gpu/drm/xe/xe_sched_job_types.h new file mode 100644 index 000000000000..fd1d75996127 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sched_job_types.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_SCHED_JOB_TYPES_H_ +#define _XE_SCHED_JOB_TYPES_H_ + +#include + +#include + +struct xe_engine; + +/** + * struct xe_sched_job - XE schedule job (batch buffer tracking) + */ +struct xe_sched_job { + /** @drm: base DRM scheduler job */ + struct drm_sched_job drm; + /** @engine: XE submission engine */ + struct xe_engine *engine; + /** @refcount: ref count of this job */ + struct kref refcount; + /** + * @fence: dma fence to indicate completion. 1 way relationship - job + * can safely reference fence, fence cannot safely reference job. + */ +#define JOB_FLAG_SUBMIT DMA_FENCE_FLAG_USER_BITS + struct dma_fence *fence; + /** @user_fence: write back value when BB is complete */ + struct { + /** @used: user fence is used */ + bool used; + /** @addr: address to write to */ + u64 addr; + /** @value: write back value */ + u64 value; + } user_fence; + /** @migrate_flush_flags: Additional flush flags for migration jobs */ + u32 migrate_flush_flags; + /** @batch_addr: batch buffer address of job */ + u64 batch_addr[0]; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_step.c b/drivers/gpu/drm/xe/xe_step.c new file mode 100644 index 000000000000..ca77d0971529 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_step.c @@ -0,0 +1,189 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_step.h" + +#include "xe_device.h" +#include "xe_platform_types.h" + +/* + * Provide mapping between PCI's revision ID to the individual GMD + * (Graphics/Media/Display) stepping values that can be compared numerically. + * + * Some platforms may have unusual ways of mapping PCI revision ID to GMD + * steppings. E.g., in some cases a higher PCI revision may translate to a + * lower stepping of the GT and/or display IP. + * + * Also note that some revisions/steppings may have been set aside as + * placeholders but never materialized in real hardware; in those cases there + * may be jumps in the revision IDs or stepping values in the tables below. + */ + +/* + * Some platforms always have the same stepping value for GT and display; + * use a macro to define these to make it easier to identify the platforms + * where the two steppings can deviate. + */ +#define COMMON_GT_MEDIA_STEP(x_) \ + .graphics = STEP_##x_, \ + .media = STEP_##x_ + +#define COMMON_STEP(x_) \ + COMMON_GT_MEDIA_STEP(x_), \ + .graphics = STEP_##x_, \ + .media = STEP_##x_, \ + .display = STEP_##x_ + +__diag_push(); +__diag_ignore_all("-Woverride-init", "Allow field overrides in table"); + +/* Same GT stepping between tgl_uy_revids and tgl_revids don't mean the same HW */ +static const struct xe_step_info tgl_revids[] = { + [0] = { COMMON_GT_MEDIA_STEP(A0), .display = STEP_B0 }, + [1] = { COMMON_GT_MEDIA_STEP(B0), .display = STEP_D0 }, +}; + +static const struct xe_step_info dg1_revids[] = { + [0] = { COMMON_STEP(A0) }, + [1] = { COMMON_STEP(B0) }, +}; + +static const struct xe_step_info adls_revids[] = { + [0x0] = { COMMON_GT_MEDIA_STEP(A0), .display = STEP_A0 }, + [0x1] = { COMMON_GT_MEDIA_STEP(A0), .display = STEP_A2 }, + [0x4] = { COMMON_GT_MEDIA_STEP(B0), .display = STEP_B0 }, + [0x8] = { COMMON_GT_MEDIA_STEP(C0), .display = STEP_B0 }, + [0xC] = { COMMON_GT_MEDIA_STEP(D0), .display = STEP_C0 }, +}; + +static const struct xe_step_info dg2_g10_revid_step_tbl[] = { + [0x0] = { COMMON_GT_MEDIA_STEP(A0), .display = STEP_A0 }, + [0x1] = { COMMON_GT_MEDIA_STEP(A1), .display = STEP_A0 }, + [0x4] = { COMMON_GT_MEDIA_STEP(B0), .display = STEP_B0 }, + [0x8] = { COMMON_GT_MEDIA_STEP(C0), .display = STEP_C0 }, +}; + +static const struct xe_step_info dg2_g11_revid_step_tbl[] = { + [0x0] = { COMMON_GT_MEDIA_STEP(A0), .display = STEP_B0 }, + [0x4] = { COMMON_GT_MEDIA_STEP(B0), .display = STEP_C0 }, + [0x5] = { COMMON_GT_MEDIA_STEP(B1), .display = STEP_C0 }, +}; + +static const struct xe_step_info dg2_g12_revid_step_tbl[] = { + [0x0] = { COMMON_GT_MEDIA_STEP(A0), .display = STEP_C0 }, + [0x1] = { COMMON_GT_MEDIA_STEP(A1), .display = STEP_C0 }, +}; + +static const struct xe_step_info pvc_revid_step_tbl[] = { + [0x3] = { .graphics = STEP_A0 }, + [0x5] = { .graphics = STEP_B0 }, + [0x6] = { .graphics = STEP_B1 }, + [0x7] = { .graphics = STEP_C0 }, +}; + +static const int pvc_basedie_subids[] = { + [0x0] = STEP_A0, + [0x3] = STEP_B0, + [0x4] = STEP_B1, + [0x5] = STEP_B3, +}; + +__diag_pop(); + +struct xe_step_info xe_step_get(struct xe_device *xe) +{ + const struct xe_step_info *revids = NULL; + struct xe_step_info step = {}; + u16 revid = xe->info.revid; + int size = 0; + const int *basedie_info = NULL; + int basedie_size = 0; + int baseid = 0; + + if (xe->info.platform == XE_PVC) { + baseid = FIELD_GET(GENMASK(5, 3), xe->info.revid); + revid = FIELD_GET(GENMASK(2, 0), xe->info.revid); + revids = pvc_revid_step_tbl; + size = ARRAY_SIZE(pvc_revid_step_tbl); + basedie_info = pvc_basedie_subids; + basedie_size = ARRAY_SIZE(pvc_basedie_subids); + } else if (xe->info.subplatform == XE_SUBPLATFORM_DG2_G10) { + revids = dg2_g10_revid_step_tbl; + size = ARRAY_SIZE(dg2_g10_revid_step_tbl); + } else if (xe->info.subplatform == XE_SUBPLATFORM_DG2_G11) { + revids = dg2_g11_revid_step_tbl; + size = ARRAY_SIZE(dg2_g11_revid_step_tbl); + } else if (xe->info.subplatform == XE_SUBPLATFORM_DG2_G12) { + revids = dg2_g12_revid_step_tbl; + size = ARRAY_SIZE(dg2_g12_revid_step_tbl); + } else if (xe->info.platform == XE_ALDERLAKE_S) { + revids = adls_revids; + size = ARRAY_SIZE(adls_revids); + } else if (xe->info.platform == XE_DG1) { + revids = dg1_revids; + size = ARRAY_SIZE(dg1_revids); + } else if (xe->info.platform == XE_TIGERLAKE) { + revids = tgl_revids; + size = ARRAY_SIZE(tgl_revids); + } + + /* Not using the stepping scheme for the platform yet. */ + if (!revids) + return step; + + if (revid < size && revids[revid].graphics != STEP_NONE) { + step = revids[revid]; + } else { + drm_warn(&xe->drm, "Unknown revid 0x%02x\n", revid); + + /* + * If we hit a gap in the revid array, use the information for + * the next revid. + * + * This may be wrong in all sorts of ways, especially if the + * steppings in the array are not monotonically increasing, but + * it's better than defaulting to 0. + */ + while (revid < size && revids[revid].graphics == STEP_NONE) + revid++; + + if (revid < size) { + drm_dbg(&xe->drm, "Using steppings for revid 0x%02x\n", + revid); + step = revids[revid]; + } else { + drm_dbg(&xe->drm, "Using future steppings\n"); + step.graphics = STEP_FUTURE; + step.display = STEP_FUTURE; + } + } + + drm_WARN_ON(&xe->drm, step.graphics == STEP_NONE); + + if (basedie_info && basedie_size) { + if (baseid < basedie_size && basedie_info[baseid] != STEP_NONE) { + step.basedie = basedie_info[baseid]; + } else { + drm_warn(&xe->drm, "Unknown baseid 0x%02x\n", baseid); + step.basedie = STEP_FUTURE; + } + } + + return step; +} + +#define STEP_NAME_CASE(name) \ + case STEP_##name: \ + return #name; + +const char *xe_step_name(enum xe_step step) +{ + switch (step) { + STEP_NAME_LIST(STEP_NAME_CASE); + + default: + return "**"; + } +} diff --git a/drivers/gpu/drm/xe/xe_step.h b/drivers/gpu/drm/xe/xe_step.h new file mode 100644 index 000000000000..0c596c8579fb --- /dev/null +++ b/drivers/gpu/drm/xe/xe_step.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_STEP_H_ +#define _XE_STEP_H_ + +#include + +#include "xe_step_types.h" + +struct xe_device; + +struct xe_step_info xe_step_get(struct xe_device *xe); +const char *xe_step_name(enum xe_step step); + +#endif diff --git a/drivers/gpu/drm/xe/xe_step_types.h b/drivers/gpu/drm/xe/xe_step_types.h new file mode 100644 index 000000000000..b7859f9647ca --- /dev/null +++ b/drivers/gpu/drm/xe/xe_step_types.h @@ -0,0 +1,51 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_STEP_TYPES_H_ +#define _XE_STEP_TYPES_H_ + +#include + +struct xe_step_info { + u8 graphics; + u8 media; + u8 display; + u8 basedie; +}; + +#define STEP_ENUM_VAL(name) STEP_##name, + +#define STEP_NAME_LIST(func) \ + func(A0) \ + func(A1) \ + func(A2) \ + func(B0) \ + func(B1) \ + func(B2) \ + func(B3) \ + func(C0) \ + func(C1) \ + func(D0) \ + func(D1) \ + func(E0) \ + func(F0) \ + func(G0) \ + func(H0) \ + func(I0) \ + func(I1) \ + func(J0) + +/* + * Symbolic steppings that do not match the hardware. These are valid both as gt + * and display steppings as symbolic names. + */ +enum xe_step { + STEP_NONE = 0, + STEP_NAME_LIST(STEP_ENUM_VAL) + STEP_FUTURE, + STEP_FOREVER, +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c new file mode 100644 index 000000000000..0fbd8d0978cf --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_sync.h" + +#include +#include +#include +#include +#include +#include + +#include "xe_device_types.h" +#include "xe_sched_job_types.h" +#include "xe_macros.h" + +#define SYNC_FLAGS_TYPE_MASK 0x3 +#define SYNC_FLAGS_FENCE_INSTALLED 0x10000 + +struct user_fence { + struct xe_device *xe; + struct kref refcount; + struct dma_fence_cb cb; + struct work_struct worker; + struct mm_struct *mm; + u64 __user *addr; + u64 value; +}; + +static void user_fence_destroy(struct kref *kref) +{ + struct user_fence *ufence = container_of(kref, struct user_fence, + refcount); + + mmdrop(ufence->mm); + kfree(ufence); +} + +static void user_fence_get(struct user_fence *ufence) +{ + kref_get(&ufence->refcount); +} + +static void user_fence_put(struct user_fence *ufence) +{ + kref_put(&ufence->refcount, user_fence_destroy); +} + +static struct user_fence *user_fence_create(struct xe_device *xe, u64 addr, + u64 value) +{ + struct user_fence *ufence; + + ufence = kmalloc(sizeof(*ufence), GFP_KERNEL); + if (!ufence) + return NULL; + + ufence->xe = xe; + kref_init(&ufence->refcount); + ufence->addr = u64_to_user_ptr(addr); + ufence->value = value; + ufence->mm = current->mm; + mmgrab(ufence->mm); + + return ufence; +} + +static void user_fence_worker(struct work_struct *w) +{ + struct user_fence *ufence = container_of(w, struct user_fence, worker); + + if (mmget_not_zero(ufence->mm)) { + kthread_use_mm(ufence->mm); + if (copy_to_user(ufence->addr, &ufence->value, sizeof(ufence->value))) + XE_WARN_ON("Copy to user failed"); + kthread_unuse_mm(ufence->mm); + mmput(ufence->mm); + } + + wake_up_all(&ufence->xe->ufence_wq); + user_fence_put(ufence); +} + +static void kick_ufence(struct user_fence *ufence, struct dma_fence *fence) +{ + INIT_WORK(&ufence->worker, user_fence_worker); + queue_work(ufence->xe->ordered_wq, &ufence->worker); + dma_fence_put(fence); +} + +static void user_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) +{ + struct user_fence *ufence = container_of(cb, struct user_fence, cb); + + kick_ufence(ufence, fence); +} + +int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, + struct xe_sync_entry *sync, + struct drm_xe_sync __user *sync_user, + bool exec, bool no_dma_fences) +{ + struct drm_xe_sync sync_in; + int err; + + if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) + return -EFAULT; + + if (XE_IOCTL_ERR(xe, sync_in.flags & + ~(SYNC_FLAGS_TYPE_MASK | DRM_XE_SYNC_SIGNAL))) + return -EINVAL; + + switch (sync_in.flags & SYNC_FLAGS_TYPE_MASK) { + case DRM_XE_SYNC_SYNCOBJ: + if (XE_IOCTL_ERR(xe, no_dma_fences)) + return -ENOTSUPP; + + if (XE_IOCTL_ERR(xe, upper_32_bits(sync_in.addr))) + return -EINVAL; + + sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); + if (XE_IOCTL_ERR(xe, !sync->syncobj)) + return -ENOENT; + + if (!(sync_in.flags & DRM_XE_SYNC_SIGNAL)) { + sync->fence = drm_syncobj_fence_get(sync->syncobj); + if (XE_IOCTL_ERR(xe, !sync->fence)) + return -EINVAL; + } + break; + + case DRM_XE_SYNC_TIMELINE_SYNCOBJ: + if (XE_IOCTL_ERR(xe, no_dma_fences)) + return -ENOTSUPP; + + if (XE_IOCTL_ERR(xe, upper_32_bits(sync_in.addr))) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, sync_in.timeline_value == 0)) + return -EINVAL; + + sync->syncobj = drm_syncobj_find(xef->drm, sync_in.handle); + if (XE_IOCTL_ERR(xe, !sync->syncobj)) + return -ENOENT; + + if (sync_in.flags & DRM_XE_SYNC_SIGNAL) { + sync->chain_fence = dma_fence_chain_alloc(); + if (!sync->chain_fence) + return -ENOMEM; + } else { + sync->fence = drm_syncobj_fence_get(sync->syncobj); + if (XE_IOCTL_ERR(xe, !sync->fence)) + return -EINVAL; + + err = dma_fence_chain_find_seqno(&sync->fence, + sync_in.timeline_value); + if (err) + return err; + } + break; + + case DRM_XE_SYNC_DMA_BUF: + if (XE_IOCTL_ERR(xe, "TODO")) + return -EINVAL; + break; + + case DRM_XE_SYNC_USER_FENCE: + if (XE_IOCTL_ERR(xe, !(sync_in.flags & DRM_XE_SYNC_SIGNAL))) + return -ENOTSUPP; + + if (XE_IOCTL_ERR(xe, sync_in.addr & 0x7)) + return -EINVAL; + + if (exec) { + sync->addr = sync_in.addr; + } else { + sync->ufence = user_fence_create(xe, sync_in.addr, + sync_in.timeline_value); + if (XE_IOCTL_ERR(xe, !sync->ufence)) + return -ENOMEM; + } + + break; + + default: + return -EINVAL; + } + + sync->flags = sync_in.flags; + sync->timeline_value = sync_in.timeline_value; + + return 0; +} + +int xe_sync_entry_wait(struct xe_sync_entry *sync) +{ + if (sync->fence) + dma_fence_wait(sync->fence, true); + + return 0; +} + +int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) +{ + int err; + + if (sync->fence) { + err = drm_sched_job_add_dependency(&job->drm, + dma_fence_get(sync->fence)); + if (err) { + dma_fence_put(sync->fence); + return err; + } + } + + return 0; +} + +bool xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, + struct dma_fence *fence) +{ + if (!(sync->flags & DRM_XE_SYNC_SIGNAL) || + sync->flags & SYNC_FLAGS_FENCE_INSTALLED) + return false; + + if (sync->chain_fence) { + drm_syncobj_add_point(sync->syncobj, sync->chain_fence, + fence, sync->timeline_value); + /* + * The chain's ownership is transferred to the + * timeline. + */ + sync->chain_fence = NULL; + } else if (sync->syncobj) { + drm_syncobj_replace_fence(sync->syncobj, fence); + } else if (sync->ufence) { + int err; + + dma_fence_get(fence); + user_fence_get(sync->ufence); + err = dma_fence_add_callback(fence, &sync->ufence->cb, + user_fence_cb); + if (err == -ENOENT) { + kick_ufence(sync->ufence, fence); + } else if (err) { + XE_WARN_ON("failed to add user fence"); + user_fence_put(sync->ufence); + dma_fence_put(fence); + } + } else if ((sync->flags & SYNC_FLAGS_TYPE_MASK) == + DRM_XE_SYNC_USER_FENCE) { + job->user_fence.used = true; + job->user_fence.addr = sync->addr; + job->user_fence.value = sync->timeline_value; + } + + /* TODO: external BO? */ + + sync->flags |= SYNC_FLAGS_FENCE_INSTALLED; + + return true; +} + +void xe_sync_entry_cleanup(struct xe_sync_entry *sync) +{ + if (sync->syncobj) + drm_syncobj_put(sync->syncobj); + if (sync->fence) + dma_fence_put(sync->fence); + if (sync->chain_fence) + dma_fence_put(&sync->chain_fence->base); + if (sync->ufence) + user_fence_put(sync->ufence); +} diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h new file mode 100644 index 000000000000..4cbcf7a19911 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sync.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_SYNC_H_ +#define _XE_SYNC_H_ + +#include "xe_sync_types.h" + +struct xe_device; +struct xe_file; +struct xe_sched_job; + +int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, + struct xe_sync_entry *sync, + struct drm_xe_sync __user *sync_user, + bool exec, bool compute_mode); +int xe_sync_entry_wait(struct xe_sync_entry *sync); +int xe_sync_entry_add_deps(struct xe_sync_entry *sync, + struct xe_sched_job *job); +bool xe_sync_entry_signal(struct xe_sync_entry *sync, + struct xe_sched_job *job, + struct dma_fence *fence); +void xe_sync_entry_cleanup(struct xe_sync_entry *sync); + +#endif diff --git a/drivers/gpu/drm/xe/xe_sync_types.h b/drivers/gpu/drm/xe/xe_sync_types.h new file mode 100644 index 000000000000..24fccc26cb53 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sync_types.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_SYNC_TYPES_H_ +#define _XE_SYNC_TYPES_H_ + +#include + +struct drm_syncobj; +struct dma_fence; +struct dma_fence_chain; +struct drm_xe_sync; +struct user_fence; + +struct xe_sync_entry { + struct drm_syncobj *syncobj; + struct dma_fence *fence; + struct dma_fence_chain *chain_fence; + struct user_fence *ufence; + u64 addr; + u64 timeline_value; + u32 flags; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_trace.c b/drivers/gpu/drm/xe/xe_trace.c new file mode 100644 index 000000000000..2570c0b859c4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef __CHECKER__ +#define CREATE_TRACE_POINTS +#include "xe_trace.h" +#endif diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h new file mode 100644 index 000000000000..a5f963f1f6eb --- /dev/null +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -0,0 +1,513 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright © 2022 Intel Corporation + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM xe + +#if !defined(_XE_TRACE_H_) || defined(TRACE_HEADER_MULTI_READ) +#define _XE_TRACE_H_ + +#include +#include + +#include "xe_bo_types.h" +#include "xe_engine_types.h" +#include "xe_gpu_scheduler_types.h" +#include "xe_gt_types.h" +#include "xe_guc_engine_types.h" +#include "xe_sched_job.h" +#include "xe_vm_types.h" + +DECLARE_EVENT_CLASS(xe_bo, + TP_PROTO(struct xe_bo *bo), + TP_ARGS(bo), + + TP_STRUCT__entry( + __field(size_t, size) + __field(u32, flags) + __field(u64, vm) + ), + + TP_fast_assign( + __entry->size = bo->size; + __entry->flags = bo->flags; + __entry->vm = (u64)bo->vm; + ), + + TP_printk("size=%ld, flags=0x%02x, vm=0x%016llx", + __entry->size, __entry->flags, __entry->vm) +); + +DEFINE_EVENT(xe_bo, xe_bo_cpu_fault, + TP_PROTO(struct xe_bo *bo), + TP_ARGS(bo) +); + +DEFINE_EVENT(xe_bo, xe_bo_move, + TP_PROTO(struct xe_bo *bo), + TP_ARGS(bo) +); + +DECLARE_EVENT_CLASS(xe_engine, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e), + + TP_STRUCT__entry( + __field(enum xe_engine_class, class) + __field(u32, logical_mask) + __field(u8, gt_id) + __field(u16, width) + __field(u16, guc_id) + __field(u32, guc_state) + __field(u32, flags) + ), + + TP_fast_assign( + __entry->class = e->class; + __entry->logical_mask = e->logical_mask; + __entry->gt_id = e->gt->info.id; + __entry->width = e->width; + __entry->guc_id = e->guc->id; + __entry->guc_state = atomic_read(&e->guc->state); + __entry->flags = e->flags; + ), + + TP_printk("%d:0x%x, gt=%d, width=%d, guc_id=%d, guc_state=0x%x, flags=0x%x", + __entry->class, __entry->logical_mask, + __entry->gt_id, __entry->width, __entry->guc_id, + __entry->guc_state, __entry->flags) +); + +DEFINE_EVENT(xe_engine, xe_engine_create, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_supress_resume, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_submit, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_scheduling_enable, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_scheduling_disable, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_scheduling_done, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_register, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_deregister, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_deregister_done, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_close, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_kill, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_cleanup_entity, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_destroy, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_reset, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_memory_cat_error, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_stop, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DEFINE_EVENT(xe_engine, xe_engine_resubmit, + TP_PROTO(struct xe_engine *e), + TP_ARGS(e) +); + +DECLARE_EVENT_CLASS(xe_sched_job, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job), + + TP_STRUCT__entry( + __field(u32, seqno) + __field(u16, guc_id) + __field(u32, guc_state) + __field(u32, flags) + __field(int, error) + __field(u64, fence) + __field(u64, batch_addr) + ), + + TP_fast_assign( + __entry->seqno = xe_sched_job_seqno(job); + __entry->guc_id = job->engine->guc->id; + __entry->guc_state = + atomic_read(&job->engine->guc->state); + __entry->flags = job->engine->flags; + __entry->error = job->fence->error; + __entry->fence = (u64)job->fence; + __entry->batch_addr = (u64)job->batch_addr[0]; + ), + + TP_printk("fence=0x%016llx, seqno=%u, guc_id=%d, batch_addr=0x%012llx, guc_state=0x%x, flags=0x%x, error=%d", + __entry->fence, __entry->seqno, __entry->guc_id, + __entry->batch_addr, __entry->guc_state, + __entry->flags, __entry->error) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_create, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_exec, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_run, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_free, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_timedout, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_set_error, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DEFINE_EVENT(xe_sched_job, xe_sched_job_ban, + TP_PROTO(struct xe_sched_job *job), + TP_ARGS(job) +); + +DECLARE_EVENT_CLASS(xe_sched_msg, + TP_PROTO(struct xe_sched_msg *msg), + TP_ARGS(msg), + + TP_STRUCT__entry( + __field(u32, opcode) + __field(u16, guc_id) + ), + + TP_fast_assign( + __entry->opcode = msg->opcode; + __entry->guc_id = + ((struct xe_engine *)msg->private_data)->guc->id; + ), + + TP_printk("guc_id=%d, opcode=%u", __entry->guc_id, + __entry->opcode) +); + +DEFINE_EVENT(xe_sched_msg, xe_sched_msg_add, + TP_PROTO(struct xe_sched_msg *msg), + TP_ARGS(msg) +); + +DEFINE_EVENT(xe_sched_msg, xe_sched_msg_recv, + TP_PROTO(struct xe_sched_msg *msg), + TP_ARGS(msg) +); + +DECLARE_EVENT_CLASS(xe_hw_fence, + TP_PROTO(struct xe_hw_fence *fence), + TP_ARGS(fence), + + TP_STRUCT__entry( + __field(u64, ctx) + __field(u32, seqno) + __field(u64, fence) + ), + + TP_fast_assign( + __entry->ctx = fence->dma.context; + __entry->seqno = fence->dma.seqno; + __entry->fence = (u64)fence; + ), + + TP_printk("ctx=0x%016llx, fence=0x%016llx, seqno=%u", + __entry->ctx, __entry->fence, __entry->seqno) +); + +DEFINE_EVENT(xe_hw_fence, xe_hw_fence_create, + TP_PROTO(struct xe_hw_fence *fence), + TP_ARGS(fence) +); + +DEFINE_EVENT(xe_hw_fence, xe_hw_fence_signal, + TP_PROTO(struct xe_hw_fence *fence), + TP_ARGS(fence) +); + +DEFINE_EVENT(xe_hw_fence, xe_hw_fence_try_signal, + TP_PROTO(struct xe_hw_fence *fence), + TP_ARGS(fence) +); + +DEFINE_EVENT(xe_hw_fence, xe_hw_fence_free, + TP_PROTO(struct xe_hw_fence *fence), + TP_ARGS(fence) +); + +DECLARE_EVENT_CLASS(xe_vma, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma), + + TP_STRUCT__entry( + __field(u64, vma) + __field(u32, asid) + __field(u64, start) + __field(u64, end) + __field(u64, ptr) + ), + + TP_fast_assign( + __entry->vma = (u64)vma; + __entry->asid = vma->vm->usm.asid; + __entry->start = vma->start; + __entry->end = vma->end; + __entry->ptr = (u64)vma->userptr.ptr; + ), + + TP_printk("vma=0x%016llx, asid=0x%05x, start=0x%012llx, end=0x%012llx, ptr=0x%012llx,", + __entry->vma, __entry->asid, __entry->start, + __entry->end, __entry->ptr) +) + +DEFINE_EVENT(xe_vma, xe_vma_flush, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_pagefault, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_acc, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_fail, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_bind, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_pf_bind, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_unbind, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_userptr_rebind_worker, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_userptr_rebind_exec, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_rebind_worker, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_rebind_exec, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_userptr_invalidate, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_usm_invalidate, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_evict, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DEFINE_EVENT(xe_vma, xe_vma_userptr_invalidate_complete, + TP_PROTO(struct xe_vma *vma), + TP_ARGS(vma) +); + +DECLARE_EVENT_CLASS(xe_vm, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm), + + TP_STRUCT__entry( + __field(u64, vm) + __field(u32, asid) + ), + + TP_fast_assign( + __entry->vm = (u64)vm; + __entry->asid = vm->usm.asid; + ), + + TP_printk("vm=0x%016llx, asid=0x%05x", __entry->vm, + __entry->asid) +); + +DEFINE_EVENT(xe_vm, xe_vm_create, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_free, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_cpu_bind, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_restart, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_rebind_worker_enter, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_rebind_worker_retry, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +DEFINE_EVENT(xe_vm, xe_vm_rebind_worker_exit, + TP_PROTO(struct xe_vm *vm), + TP_ARGS(vm) +); + +TRACE_EVENT(xe_guc_ct_h2g_flow_control, + TP_PROTO(u32 _head, u32 _tail, u32 size, u32 space, u32 len), + TP_ARGS(_head, _tail, size, space, len), + + TP_STRUCT__entry( + __field(u32, _head) + __field(u32, _tail) + __field(u32, size) + __field(u32, space) + __field(u32, len) + ), + + TP_fast_assign( + __entry->_head = _head; + __entry->_tail = _tail; + __entry->size = size; + __entry->space = space; + __entry->len = len; + ), + + TP_printk("head=%u, tail=%u, size=%u, space=%u, len=%u", + __entry->_head, __entry->_tail, __entry->size, + __entry->space, __entry->len) +); + +TRACE_EVENT(xe_guc_ct_g2h_flow_control, + TP_PROTO(u32 _head, u32 _tail, u32 size, u32 space, u32 len), + TP_ARGS(_head, _tail, size, space, len), + + TP_STRUCT__entry( + __field(u32, _head) + __field(u32, _tail) + __field(u32, size) + __field(u32, space) + __field(u32, len) + ), + + TP_fast_assign( + __entry->_head = _head; + __entry->_tail = _tail; + __entry->size = size; + __entry->space = space; + __entry->len = len; + ), + + TP_printk("head=%u, tail=%u, size=%u, space=%u, len=%u", + __entry->_head, __entry->_tail, __entry->size, + __entry->space, __entry->len) +); + +#endif + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_PATH ../../drivers/gpu/drm/xe +#define TRACE_INCLUDE_FILE xe_trace +#include diff --git a/drivers/gpu/drm/xe/xe_ttm_gtt_mgr.c b/drivers/gpu/drm/xe/xe_ttm_gtt_mgr.c new file mode 100644 index 000000000000..a0ba8bba84d1 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_gtt_mgr.c @@ -0,0 +1,130 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021-2022 Intel Corporation + * Copyright (C) 2021-2002 Red Hat + */ + +#include + +#include +#include +#include + +#include "xe_bo.h" +#include "xe_gt.h" +#include "xe_ttm_gtt_mgr.h" + +struct xe_ttm_gtt_node { + struct ttm_buffer_object *tbo; + struct ttm_range_mgr_node base; +}; + +static inline struct xe_ttm_gtt_mgr * +to_gtt_mgr(struct ttm_resource_manager *man) +{ + return container_of(man, struct xe_ttm_gtt_mgr, manager); +} + +static inline struct xe_ttm_gtt_node * +to_xe_ttm_gtt_node(struct ttm_resource *res) +{ + return container_of(res, struct xe_ttm_gtt_node, base.base); +} + +static int xe_ttm_gtt_mgr_new(struct ttm_resource_manager *man, + struct ttm_buffer_object *tbo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + struct xe_ttm_gtt_node *node; + int r; + + node = kzalloc(struct_size(node, base.mm_nodes, 1), GFP_KERNEL); + if (!node) + return -ENOMEM; + + node->tbo = tbo; + ttm_resource_init(tbo, place, &node->base.base); + + if (!(place->flags & TTM_PL_FLAG_TEMPORARY) && + ttm_resource_manager_usage(man) > (man->size << PAGE_SHIFT)) { + r = -ENOSPC; + goto err_fini; + } + + node->base.mm_nodes[0].start = 0; + node->base.mm_nodes[0].size = PFN_UP(node->base.base.size); + node->base.base.start = XE_BO_INVALID_OFFSET; + + *res = &node->base.base; + + return 0; + +err_fini: + ttm_resource_fini(man, &node->base.base); + kfree(node); + return r; +} + +static void xe_ttm_gtt_mgr_del(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct xe_ttm_gtt_node *node = to_xe_ttm_gtt_node(res); + + ttm_resource_fini(man, res); + kfree(node); +} + +static void xe_ttm_gtt_mgr_debug(struct ttm_resource_manager *man, + struct drm_printer *printer) +{ + +} + +static const struct ttm_resource_manager_func xe_ttm_gtt_mgr_func = { + .alloc = xe_ttm_gtt_mgr_new, + .free = xe_ttm_gtt_mgr_del, + .debug = xe_ttm_gtt_mgr_debug +}; + +static void ttm_gtt_mgr_fini(struct drm_device *drm, void *arg) +{ + struct xe_ttm_gtt_mgr *mgr = arg; + struct xe_device *xe = gt_to_xe(mgr->gt); + struct ttm_resource_manager *man = &mgr->manager; + int err; + + ttm_resource_manager_set_used(man, false); + + err = ttm_resource_manager_evict_all(&xe->ttm, man); + if (err) + return; + + ttm_resource_manager_cleanup(man); + ttm_set_driver_manager(&xe->ttm, XE_PL_TT, NULL); +} + +int xe_ttm_gtt_mgr_init(struct xe_gt *gt, struct xe_ttm_gtt_mgr *mgr, + u64 gtt_size) +{ + struct xe_device *xe = gt_to_xe(gt); + struct ttm_resource_manager *man = &mgr->manager; + int err; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + + mgr->gt = gt; + man->use_tt = true; + man->func = &xe_ttm_gtt_mgr_func; + + ttm_resource_manager_init(man, &xe->ttm, gtt_size >> PAGE_SHIFT); + + ttm_set_driver_manager(&xe->ttm, XE_PL_TT, &mgr->manager); + ttm_resource_manager_set_used(man, true); + + err = drmm_add_action_or_reset(&xe->drm, ttm_gtt_mgr_fini, mgr); + if (err) + return err; + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_ttm_gtt_mgr.h b/drivers/gpu/drm/xe/xe_ttm_gtt_mgr.h new file mode 100644 index 000000000000..d1d57cb9c2b8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_gtt_mgr.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TTGM_GTT_MGR_H_ +#define _XE_TTGM_GTT_MGR_H_ + +#include "xe_ttm_gtt_mgr_types.h" + +struct xe_gt; + +int xe_ttm_gtt_mgr_init(struct xe_gt *gt, struct xe_ttm_gtt_mgr *mgr, + u64 gtt_size); + +#endif diff --git a/drivers/gpu/drm/xe/xe_ttm_gtt_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_gtt_mgr_types.h new file mode 100644 index 000000000000..c66737488326 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_gtt_mgr_types.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TTM_GTT_MGR_TYPES_H_ +#define _XE_TTM_GTT_MGR_TYPES_H_ + +#include + +struct xe_gt; + +struct xe_ttm_gtt_mgr { + struct xe_gt *gt; + struct ttm_resource_manager manager; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c new file mode 100644 index 000000000000..e391e81d3640 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -0,0 +1,403 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021-2022 Intel Corporation + * Copyright (C) 2021-2002 Red Hat + */ + +#include + +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_res_cursor.h" +#include "xe_ttm_vram_mgr.h" + +static inline struct xe_ttm_vram_mgr * +to_vram_mgr(struct ttm_resource_manager *man) +{ + return container_of(man, struct xe_ttm_vram_mgr, manager); +} + +static inline struct xe_gt * +mgr_to_gt(struct xe_ttm_vram_mgr *mgr) +{ + return mgr->gt; +} + +static inline struct drm_buddy_block * +xe_ttm_vram_mgr_first_block(struct list_head *list) +{ + return list_first_entry_or_null(list, struct drm_buddy_block, link); +} + +static inline bool xe_is_vram_mgr_blocks_contiguous(struct list_head *head) +{ + struct drm_buddy_block *block; + u64 start, size; + + block = xe_ttm_vram_mgr_first_block(head); + if (!block) + return false; + + while (head != block->link.next) { + start = xe_ttm_vram_mgr_block_start(block); + size = xe_ttm_vram_mgr_block_size(block); + + block = list_entry(block->link.next, struct drm_buddy_block, + link); + if (start + size != xe_ttm_vram_mgr_block_start(block)) + return false; + } + + return true; +} + +static int xe_ttm_vram_mgr_new(struct ttm_resource_manager *man, + struct ttm_buffer_object *tbo, + const struct ttm_place *place, + struct ttm_resource **res) +{ + u64 max_bytes, cur_size, min_block_size; + struct xe_ttm_vram_mgr *mgr = to_vram_mgr(man); + struct xe_ttm_vram_mgr_resource *vres; + u64 size, remaining_size, lpfn, fpfn; + struct drm_buddy *mm = &mgr->mm; + struct drm_buddy_block *block; + unsigned long pages_per_block; + int r; + + lpfn = (u64)place->lpfn << PAGE_SHIFT; + if (!lpfn) + lpfn = man->size; + + fpfn = (u64)place->fpfn << PAGE_SHIFT; + + max_bytes = mgr->gt->mem.vram.size; + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + pages_per_block = ~0ul; + } else { +#ifdef CONFIG_TRANSPARENT_HUGEPAGE + pages_per_block = HPAGE_PMD_NR; +#else + /* default to 2MB */ + pages_per_block = 2UL << (20UL - PAGE_SHIFT); +#endif + + pages_per_block = max_t(uint32_t, pages_per_block, + tbo->page_alignment); + } + + vres = kzalloc(sizeof(*vres), GFP_KERNEL); + if (!vres) + return -ENOMEM; + + ttm_resource_init(tbo, place, &vres->base); + remaining_size = vres->base.size; + + /* bail out quickly if there's likely not enough VRAM for this BO */ + if (ttm_resource_manager_usage(man) > max_bytes) { + r = -ENOSPC; + goto error_fini; + } + + INIT_LIST_HEAD(&vres->blocks); + + if (place->flags & TTM_PL_FLAG_TOPDOWN) + vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; + + if (fpfn || lpfn != man->size) + /* Allocate blocks in desired range */ + vres->flags |= DRM_BUDDY_RANGE_ALLOCATION; + + mutex_lock(&mgr->lock); + while (remaining_size) { + if (tbo->page_alignment) + min_block_size = tbo->page_alignment << PAGE_SHIFT; + else + min_block_size = mgr->default_page_size; + + XE_BUG_ON(min_block_size < mm->chunk_size); + + /* Limit maximum size to 2GiB due to SG table limitations */ + size = min(remaining_size, 2ULL << 30); + + if (size >= pages_per_block << PAGE_SHIFT) + min_block_size = pages_per_block << PAGE_SHIFT; + + cur_size = size; + + if (fpfn + size != place->lpfn << PAGE_SHIFT) { + /* + * Except for actual range allocation, modify the size and + * min_block_size conforming to continuous flag enablement + */ + if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { + size = roundup_pow_of_two(size); + min_block_size = size; + /* + * Modify the size value if size is not + * aligned with min_block_size + */ + } else if (!IS_ALIGNED(size, min_block_size)) { + size = round_up(size, min_block_size); + } + } + + r = drm_buddy_alloc_blocks(mm, fpfn, + lpfn, + size, + min_block_size, + &vres->blocks, + vres->flags); + if (unlikely(r)) + goto error_free_blocks; + + if (size > remaining_size) + remaining_size = 0; + else + remaining_size -= size; + } + mutex_unlock(&mgr->lock); + + if (cur_size != size) { + struct drm_buddy_block *block; + struct list_head *trim_list; + u64 original_size; + LIST_HEAD(temp); + + trim_list = &vres->blocks; + original_size = vres->base.size; + + /* + * If size value is rounded up to min_block_size, trim the last + * block to the required size + */ + if (!list_is_singular(&vres->blocks)) { + block = list_last_entry(&vres->blocks, typeof(*block), link); + list_move_tail(&block->link, &temp); + trim_list = &temp; + /* + * Compute the original_size value by subtracting the + * last block size with (aligned size - original size) + */ + original_size = xe_ttm_vram_mgr_block_size(block) - + (size - cur_size); + } + + mutex_lock(&mgr->lock); + drm_buddy_block_trim(mm, + original_size, + trim_list); + mutex_unlock(&mgr->lock); + + if (!list_empty(&temp)) + list_splice_tail(trim_list, &vres->blocks); + } + + vres->base.start = 0; + list_for_each_entry(block, &vres->blocks, link) { + unsigned long start; + + start = xe_ttm_vram_mgr_block_start(block) + + xe_ttm_vram_mgr_block_size(block); + start >>= PAGE_SHIFT; + + if (start > PFN_UP(vres->base.size)) + start -= PFN_UP(vres->base.size); + else + start = 0; + vres->base.start = max(vres->base.start, start); + } + + if (xe_is_vram_mgr_blocks_contiguous(&vres->blocks)) + vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS; + + *res = &vres->base; + return 0; + +error_free_blocks: + drm_buddy_free_list(mm, &vres->blocks); + mutex_unlock(&mgr->lock); +error_fini: + ttm_resource_fini(man, &vres->base); + kfree(vres); + + return r; +} + +static void xe_ttm_vram_mgr_del(struct ttm_resource_manager *man, + struct ttm_resource *res) +{ + struct xe_ttm_vram_mgr_resource *vres = + to_xe_ttm_vram_mgr_resource(res); + struct xe_ttm_vram_mgr *mgr = to_vram_mgr(man); + struct drm_buddy *mm = &mgr->mm; + + mutex_lock(&mgr->lock); + drm_buddy_free_list(mm, &vres->blocks); + mutex_unlock(&mgr->lock); + + ttm_resource_fini(man, res); + + kfree(vres); +} + +static void xe_ttm_vram_mgr_debug(struct ttm_resource_manager *man, + struct drm_printer *printer) +{ + struct xe_ttm_vram_mgr *mgr = to_vram_mgr(man); + struct drm_buddy *mm = &mgr->mm; + + mutex_lock(&mgr->lock); + drm_buddy_print(mm, printer); + mutex_unlock(&mgr->lock); + drm_printf(printer, "man size:%llu\n", man->size); +} + +static const struct ttm_resource_manager_func xe_ttm_vram_mgr_func = { + .alloc = xe_ttm_vram_mgr_new, + .free = xe_ttm_vram_mgr_del, + .debug = xe_ttm_vram_mgr_debug +}; + +static void ttm_vram_mgr_fini(struct drm_device *drm, void *arg) +{ + struct xe_ttm_vram_mgr *mgr = arg; + struct xe_device *xe = gt_to_xe(mgr->gt); + struct ttm_resource_manager *man = &mgr->manager; + int err; + + ttm_resource_manager_set_used(man, false); + + err = ttm_resource_manager_evict_all(&xe->ttm, man); + if (err) + return; + + drm_buddy_fini(&mgr->mm); + + ttm_resource_manager_cleanup(man); + ttm_set_driver_manager(&xe->ttm, XE_PL_VRAM0 + mgr->gt->info.vram_id, + NULL); +} + +int xe_ttm_vram_mgr_init(struct xe_gt *gt, struct xe_ttm_vram_mgr *mgr) +{ + struct xe_device *xe = gt_to_xe(gt); + struct ttm_resource_manager *man = &mgr->manager; + int err; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + + mgr->gt = gt; + man->func = &xe_ttm_vram_mgr_func; + + ttm_resource_manager_init(man, &xe->ttm, gt->mem.vram.size); + err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE); + if (err) + return err; + + mutex_init(&mgr->lock); + mgr->default_page_size = PAGE_SIZE; + + ttm_set_driver_manager(&xe->ttm, XE_PL_VRAM0 + gt->info.vram_id, + &mgr->manager); + ttm_resource_manager_set_used(man, true); + + err = drmm_add_action_or_reset(&xe->drm, ttm_vram_mgr_fini, mgr); + if (err) + return err; + + return 0; +} + +int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, + struct ttm_resource *res, + u64 offset, u64 length, + struct device *dev, + enum dma_data_direction dir, + struct sg_table **sgt) +{ + struct xe_gt *gt = xe_device_get_gt(xe, res->mem_type - XE_PL_VRAM0); + struct xe_res_cursor cursor; + struct scatterlist *sg; + int num_entries = 0; + int i, r; + + *sgt = kmalloc(sizeof(**sgt), GFP_KERNEL); + if (!*sgt) + return -ENOMEM; + + /* Determine the number of DRM_BUDDY blocks to export */ + xe_res_first(res, offset, length, &cursor); + while (cursor.remaining) { + num_entries++; + xe_res_next(&cursor, cursor.size); + } + + r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL); + if (r) + goto error_free; + + /* Initialize scatterlist nodes of sg_table */ + for_each_sgtable_sg((*sgt), sg, i) + sg->length = 0; + + /* + * Walk down DRM_BUDDY blocks to populate scatterlist nodes + * @note: Use iterator api to get first the DRM_BUDDY block + * and the number of bytes from it. Access the following + * DRM_BUDDY block(s) if more buffer needs to exported + */ + xe_res_first(res, offset, length, &cursor); + for_each_sgtable_sg((*sgt), sg, i) { + phys_addr_t phys = cursor.start + gt->mem.vram.io_start; + size_t size = cursor.size; + dma_addr_t addr; + + addr = dma_map_resource(dev, phys, size, dir, + DMA_ATTR_SKIP_CPU_SYNC); + r = dma_mapping_error(dev, addr); + if (r) + goto error_unmap; + + sg_set_page(sg, NULL, size, 0); + sg_dma_address(sg) = addr; + sg_dma_len(sg) = size; + + xe_res_next(&cursor, cursor.size); + } + + return 0; + +error_unmap: + for_each_sgtable_sg((*sgt), sg, i) { + if (!sg->length) + continue; + + dma_unmap_resource(dev, sg->dma_address, + sg->length, dir, + DMA_ATTR_SKIP_CPU_SYNC); + } + sg_free_table(*sgt); + +error_free: + kfree(*sgt); + return r; +} + +void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, + struct sg_table *sgt) +{ + struct scatterlist *sg; + int i; + + for_each_sgtable_sg(sgt, sg, i) + dma_unmap_resource(dev, sg->dma_address, + sg->length, dir, + DMA_ATTR_SKIP_CPU_SYNC); + sg_free_table(sgt); + kfree(sgt); +} diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h new file mode 100644 index 000000000000..537fccec4318 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h @@ -0,0 +1,41 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TTM_VRAM_MGR_H_ +#define _XE_TTM_VRAM_MGR_H_ + +#include "xe_ttm_vram_mgr_types.h" + +enum dma_data_direction; +struct xe_device; +struct xe_gt; + +int xe_ttm_vram_mgr_init(struct xe_gt *gt, struct xe_ttm_vram_mgr *mgr); +int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, + struct ttm_resource *res, + u64 offset, u64 length, + struct device *dev, + enum dma_data_direction dir, + struct sg_table **sgt); +void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, + struct sg_table *sgt); + +static inline u64 xe_ttm_vram_mgr_block_start(struct drm_buddy_block *block) +{ + return drm_buddy_block_offset(block); +} + +static inline u64 xe_ttm_vram_mgr_block_size(struct drm_buddy_block *block) +{ + return PAGE_SIZE << drm_buddy_block_order(block); +} + +static inline struct xe_ttm_vram_mgr_resource * +to_xe_ttm_vram_mgr_resource(struct ttm_resource *res) +{ + return container_of(res, struct xe_ttm_vram_mgr_resource, base); +} + +#endif diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h new file mode 100644 index 000000000000..39b93c71c21b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr_types.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TTM_VRAM_MGR_TYPES_H_ +#define _XE_TTM_VRAM_MGR_TYPES_H_ + +#include +#include + +struct xe_gt; + +/** + * struct xe_ttm_vram_mgr - XE TTM VRAM manager + * + * Manages placement of TTM resource in VRAM. + */ +struct xe_ttm_vram_mgr { + /** @gt: Graphics tile which the VRAM belongs to */ + struct xe_gt *gt; + /** @manager: Base TTM resource manager */ + struct ttm_resource_manager manager; + /** @mm: DRM buddy allocator which manages the VRAM */ + struct drm_buddy mm; + /** @default_page_size: default page size */ + u64 default_page_size; + /** @lock: protects allocations of VRAM */ + struct mutex lock; +}; + +/** + * struct xe_ttm_vram_mgr_resource - XE TTM VRAM resource + */ +struct xe_ttm_vram_mgr_resource { + /** @base: Base TTM resource */ + struct ttm_resource base; + /** @blocks: list of DRM buddy blocks */ + struct list_head blocks; + /** @flags: flags associated with the resource */ + unsigned long flags; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_tuning.c b/drivers/gpu/drm/xe/xe_tuning.c new file mode 100644 index 000000000000..e043db037368 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tuning.c @@ -0,0 +1,39 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_wa.h" + +#include "xe_platform_types.h" +#include "xe_gt_types.h" +#include "xe_rtp.h" + +#include "gt/intel_gt_regs.h" + +#undef _MMIO +#undef MCR_REG +#define _MMIO(x) _XE_RTP_REG(x) +#define MCR_REG(x) _XE_RTP_MCR_REG(x) + +static const struct xe_rtp_entry gt_tunings[] = { + { XE_RTP_NAME("Tuning: 32B Access Enable"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_SET(XEHP_SQCM, EN_32B_ACCESS) + }, + {} +}; + +static const struct xe_rtp_entry context_tunings[] = { + { XE_RTP_NAME("1604555607"), + XE_RTP_RULES(GRAPHICS_VERSION(1200)), + XE_RTP_FIELD_SET_NO_READ_MASK(XEHP_FF_MODE2, FF_MODE2_TDS_TIMER_MASK, + FF_MODE2_TDS_TIMER_128) + }, + {} +}; + +void xe_tuning_process_gt(struct xe_gt *gt) +{ + xe_rtp_process(gt_tunings, >->reg_sr, gt, NULL); +} diff --git a/drivers/gpu/drm/xe/xe_tuning.h b/drivers/gpu/drm/xe/xe_tuning.h new file mode 100644 index 000000000000..66dbc93192bd --- /dev/null +++ b/drivers/gpu/drm/xe/xe_tuning.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_TUNING_ +#define _XE_TUNING_ + +struct xe_gt; + +void xe_tuning_process_gt(struct xe_gt *gt); + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc.c b/drivers/gpu/drm/xe/xe_uc.c new file mode 100644 index 000000000000..938d14698003 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc.c @@ -0,0 +1,226 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_device.h" +#include "xe_huc.h" +#include "xe_gt.h" +#include "xe_guc.h" +#include "xe_guc_pc.h" +#include "xe_guc_submit.h" +#include "xe_uc.h" +#include "xe_uc_fw.h" +#include "xe_wopcm.h" + +static struct xe_gt * +uc_to_gt(struct xe_uc *uc) +{ + return container_of(uc, struct xe_gt, uc); +} + +static struct xe_device * +uc_to_xe(struct xe_uc *uc) +{ + return gt_to_xe(uc_to_gt(uc)); +} + +/* Should be called once at driver load only */ +int xe_uc_init(struct xe_uc *uc) +{ + int ret; + + /* GuC submission not enabled, nothing to do */ + if (!xe_device_guc_submission_enabled(uc_to_xe(uc))) + return 0; + + ret = xe_guc_init(&uc->guc); + if (ret) + goto err; + + ret = xe_huc_init(&uc->huc); + if (ret) + goto err; + + ret = xe_wopcm_init(&uc->wopcm); + if (ret) + goto err; + + ret = xe_guc_submit_init(&uc->guc); + if (ret) + goto err; + + return 0; + +err: + /* If any uC firmwares not found, fall back to execlists */ + xe_device_guc_submission_disable(uc_to_xe(uc)); + + return ret; +} + +/** + * xe_uc_init_post_hwconfig - init Uc post hwconfig load + * @uc: The UC object + * + * Return: 0 on success, negative error code on error. + */ +int xe_uc_init_post_hwconfig(struct xe_uc *uc) +{ + /* GuC submission not enabled, nothing to do */ + if (!xe_device_guc_submission_enabled(uc_to_xe(uc))) + return 0; + + return xe_guc_init_post_hwconfig(&uc->guc); +} + +static int uc_reset(struct xe_uc *uc) +{ + struct xe_device *xe = uc_to_xe(uc); + int ret; + + ret = xe_guc_reset(&uc->guc); + if (ret) { + drm_err(&xe->drm, "Failed to reset GuC, ret = %d\n", ret); + return ret; + } + + return 0; +} + +static int uc_sanitize(struct xe_uc *uc) +{ + xe_huc_sanitize(&uc->huc); + xe_guc_sanitize(&uc->guc); + + return uc_reset(uc); +} + +/** + * xe_uc_init_hwconfig - minimally init Uc, read and parse hwconfig + * @uc: The UC object + * + * Return: 0 on success, negative error code on error. + */ +int xe_uc_init_hwconfig(struct xe_uc *uc) +{ + int ret; + + /* GuC submission not enabled, nothing to do */ + if (!xe_device_guc_submission_enabled(uc_to_xe(uc))) + return 0; + + ret = xe_guc_min_load_for_hwconfig(&uc->guc); + if (ret) + return ret; + + return 0; +} + +/* + * Should be called during driver load, after every GT reset, and after every + * suspend to reload / auth the firmwares. + */ +int xe_uc_init_hw(struct xe_uc *uc) +{ + int ret; + + /* GuC submission not enabled, nothing to do */ + if (!xe_device_guc_submission_enabled(uc_to_xe(uc))) + return 0; + + ret = uc_sanitize(uc); + if (ret) + return ret; + + ret = xe_huc_upload(&uc->huc); + if (ret) + return ret; + + ret = xe_guc_upload(&uc->guc); + if (ret) + return ret; + + ret = xe_guc_enable_communication(&uc->guc); + if (ret) + return ret; + + ret = xe_gt_record_default_lrcs(uc_to_gt(uc)); + if (ret) + return ret; + + ret = xe_guc_post_load_init(&uc->guc); + if (ret) + return ret; + + ret = xe_guc_pc_start(&uc->guc.pc); + if (ret) + return ret; + + /* We don't fail the driver load if HuC fails to auth, but let's warn */ + ret = xe_huc_auth(&uc->huc); + XE_WARN_ON(ret); + + return 0; +} + +int xe_uc_reset_prepare(struct xe_uc *uc) +{ + /* GuC submission not enabled, nothing to do */ + if (!xe_device_guc_submission_enabled(uc_to_xe(uc))) + return 0; + + return xe_guc_reset_prepare(&uc->guc); +} + +void xe_uc_stop_prepare(struct xe_uc *uc) +{ + xe_guc_stop_prepare(&uc->guc); +} + +int xe_uc_stop(struct xe_uc *uc) +{ + /* GuC submission not enabled, nothing to do */ + if (!xe_device_guc_submission_enabled(uc_to_xe(uc))) + return 0; + + return xe_guc_stop(&uc->guc); +} + +int xe_uc_start(struct xe_uc *uc) +{ + /* GuC submission not enabled, nothing to do */ + if (!xe_device_guc_submission_enabled(uc_to_xe(uc))) + return 0; + + return xe_guc_start(&uc->guc); +} + +static void uc_reset_wait(struct xe_uc *uc) +{ + int ret; + +again: + xe_guc_reset_wait(&uc->guc); + + ret = xe_uc_reset_prepare(uc); + if (ret) + goto again; +} + +int xe_uc_suspend(struct xe_uc *uc) +{ + int ret; + + /* GuC submission not enabled, nothing to do */ + if (!xe_device_guc_submission_enabled(uc_to_xe(uc))) + return 0; + + uc_reset_wait(uc); + + ret = xe_uc_stop(uc); + if (ret) + return ret; + + return xe_guc_suspend(&uc->guc); +} diff --git a/drivers/gpu/drm/xe/xe_uc.h b/drivers/gpu/drm/xe/xe_uc.h new file mode 100644 index 000000000000..380e722f95fc --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc.h @@ -0,0 +1,21 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_H_ +#define _XE_UC_H_ + +#include "xe_uc_types.h" + +int xe_uc_init(struct xe_uc *uc); +int xe_uc_init_hwconfig(struct xe_uc *uc); +int xe_uc_init_post_hwconfig(struct xe_uc *uc); +int xe_uc_init_hw(struct xe_uc *uc); +int xe_uc_reset_prepare(struct xe_uc *uc); +void xe_uc_stop_prepare(struct xe_uc *uc); +int xe_uc_stop(struct xe_uc *uc); +int xe_uc_start(struct xe_uc *uc); +int xe_uc_suspend(struct xe_uc *uc); + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc_debugfs.c b/drivers/gpu/drm/xe/xe_uc_debugfs.c new file mode 100644 index 000000000000..0a39ec5a6e99 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_debugfs.c @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include + +#include "xe_gt.h" +#include "xe_guc_debugfs.h" +#include "xe_huc_debugfs.h" +#include "xe_macros.h" +#include "xe_uc_debugfs.h" + +void xe_uc_debugfs_register(struct xe_uc *uc, struct dentry *parent) +{ + struct dentry *root; + + root = debugfs_create_dir("uc", parent); + if (IS_ERR(root)) { + XE_WARN_ON("Create UC directory failed"); + return; + } + + xe_guc_debugfs_register(&uc->guc, root); + xe_huc_debugfs_register(&uc->huc, root); +} diff --git a/drivers/gpu/drm/xe/xe_uc_debugfs.h b/drivers/gpu/drm/xe/xe_uc_debugfs.h new file mode 100644 index 000000000000..a13382df2bd7 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_debugfs.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_DEBUGFS_H_ +#define _XE_UC_DEBUGFS_H_ + +struct dentry; +struct xe_uc; + +void xe_uc_debugfs_register(struct xe_uc *uc, struct dentry *parent); + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c new file mode 100644 index 000000000000..86c47b7f0901 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include + +#include + +#include "xe_bo.h" +#include "xe_device_types.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_guc_reg.h" +#include "xe_map.h" +#include "xe_mmio.h" +#include "xe_uc_fw.h" + +static struct xe_gt * +__uc_fw_to_gt(struct xe_uc_fw *uc_fw, enum xe_uc_fw_type type) +{ + if (type == XE_UC_FW_TYPE_GUC) + return container_of(uc_fw, struct xe_gt, uc.guc.fw); + + XE_BUG_ON(type != XE_UC_FW_TYPE_HUC); + return container_of(uc_fw, struct xe_gt, uc.huc.fw); +} + +static struct xe_gt *uc_fw_to_gt(struct xe_uc_fw *uc_fw) +{ + return __uc_fw_to_gt(uc_fw, uc_fw->type); +} + +static struct xe_device *uc_fw_to_xe(struct xe_uc_fw *uc_fw) +{ + return gt_to_xe(uc_fw_to_gt(uc_fw)); +} + +/* + * List of required GuC and HuC binaries per-platform. + * Must be ordered based on platform + revid, from newer to older. + */ +#define XE_GUC_FIRMWARE_DEFS(fw_def, guc_def) \ + fw_def(METEORLAKE, 0, guc_def(mtl, 70, 5, 2)) \ + fw_def(ALDERLAKE_P, 0, guc_def(adlp, 70, 5, 2)) \ + fw_def(ALDERLAKE_S, 0, guc_def(tgl, 70, 5, 2)) \ + fw_def(PVC, 0, guc_def(pvc, 70, 5, 2)) \ + fw_def(DG2, 0, guc_def(dg2, 70, 5, 2)) \ + fw_def(DG1, 0, guc_def(dg1, 70, 5, 2)) \ + fw_def(TIGERLAKE, 0, guc_def(tgl, 70, 5, 2)) + +#define XE_HUC_FIRMWARE_DEFS(fw_def, huc_def) \ + fw_def(DG1, 0, huc_def(dg1, 7, 9, 3)) \ + fw_def(TIGERLAKE, 0, huc_def(tgl, 7, 9, 3)) + +#define __MAKE_UC_FW_PATH_MAJOR(prefix_, name_, major_) \ + "xe/" \ + __stringify(prefix_) "_" name_ "_" \ + __stringify(major_) ".bin" + +#define __MAKE_UC_FW_PATH(prefix_, name_, major_, minor_, patch_) \ + "xe/" \ + __stringify(prefix_) name_ \ + __stringify(major_) "." \ + __stringify(minor_) "." \ + __stringify(patch_) ".bin" + +#define MAKE_GUC_FW_PATH(prefix_, major_, minor_, patch_) \ + __MAKE_UC_FW_PATH_MAJOR(prefix_, "guc", major_) + +#define MAKE_HUC_FW_PATH(prefix_, major_, minor_, bld_num_) \ + __MAKE_UC_FW_PATH(prefix_, "_huc_", major_, minor_, bld_num_) + +/* All blobs need to be declared via MODULE_FIRMWARE() */ +#define XE_UC_MODULE_FW(platform_, revid_, uc_) \ + MODULE_FIRMWARE(uc_); + +XE_GUC_FIRMWARE_DEFS(XE_UC_MODULE_FW, MAKE_GUC_FW_PATH) +XE_HUC_FIRMWARE_DEFS(XE_UC_MODULE_FW, MAKE_HUC_FW_PATH) + +/* The below structs and macros are used to iterate across the list of blobs */ +struct __packed uc_fw_blob { + u8 major; + u8 minor; + const char *path; +}; + +#define UC_FW_BLOB(major_, minor_, path_) \ + { .major = major_, .minor = minor_, .path = path_ } + +#define GUC_FW_BLOB(prefix_, major_, minor_, patch_) \ + UC_FW_BLOB(major_, minor_, \ + MAKE_GUC_FW_PATH(prefix_, major_, minor_, patch_)) + +#define HUC_FW_BLOB(prefix_, major_, minor_, bld_num_) \ + UC_FW_BLOB(major_, minor_, \ + MAKE_HUC_FW_PATH(prefix_, major_, minor_, bld_num_)) + +struct __packed uc_fw_platform_requirement { + enum xe_platform p; + u8 rev; /* first platform rev using this FW */ + const struct uc_fw_blob blob; +}; + +#define MAKE_FW_LIST(platform_, revid_, uc_) \ +{ \ + .p = XE_##platform_, \ + .rev = revid_, \ + .blob = uc_, \ +}, + +struct fw_blobs_by_type { + const struct uc_fw_platform_requirement *blobs; + u32 count; +}; + +static void +uc_fw_auto_select(struct xe_device *xe, struct xe_uc_fw *uc_fw) +{ + static const struct uc_fw_platform_requirement blobs_guc[] = { + XE_GUC_FIRMWARE_DEFS(MAKE_FW_LIST, GUC_FW_BLOB) + }; + static const struct uc_fw_platform_requirement blobs_huc[] = { + XE_HUC_FIRMWARE_DEFS(MAKE_FW_LIST, HUC_FW_BLOB) + }; + static const struct fw_blobs_by_type blobs_all[XE_UC_FW_NUM_TYPES] = { + [XE_UC_FW_TYPE_GUC] = { blobs_guc, ARRAY_SIZE(blobs_guc) }, + [XE_UC_FW_TYPE_HUC] = { blobs_huc, ARRAY_SIZE(blobs_huc) }, + }; + static const struct uc_fw_platform_requirement *fw_blobs; + enum xe_platform p = xe->info.platform; + u32 fw_count; + u8 rev = xe->info.revid; + int i; + + XE_BUG_ON(uc_fw->type >= ARRAY_SIZE(blobs_all)); + fw_blobs = blobs_all[uc_fw->type].blobs; + fw_count = blobs_all[uc_fw->type].count; + + for (i = 0; i < fw_count && p <= fw_blobs[i].p; i++) { + if (p == fw_blobs[i].p && rev >= fw_blobs[i].rev) { + const struct uc_fw_blob *blob = &fw_blobs[i].blob; + + uc_fw->path = blob->path; + uc_fw->major_ver_wanted = blob->major; + uc_fw->minor_ver_wanted = blob->minor; + break; + } + } +} + +/** + * xe_uc_fw_copy_rsa - copy fw RSA to buffer + * + * @uc_fw: uC firmware + * @dst: dst buffer + * @max_len: max number of bytes to copy + * + * Return: number of copied bytes. + */ +size_t xe_uc_fw_copy_rsa(struct xe_uc_fw *uc_fw, void *dst, u32 max_len) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + u32 size = min_t(u32, uc_fw->rsa_size, max_len); + + XE_BUG_ON(size % 4); + XE_BUG_ON(!xe_uc_fw_is_available(uc_fw)); + + xe_map_memcpy_from(xe, dst, &uc_fw->bo->vmap, + xe_uc_fw_rsa_offset(uc_fw), size); + + return size; +} + +static void uc_fw_fini(struct drm_device *drm, void *arg) +{ + struct xe_uc_fw *uc_fw = arg; + + if (!xe_uc_fw_is_available(uc_fw)) + return; + + xe_bo_unpin_map_no_vm(uc_fw->bo); + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_SELECTED); +} + +int xe_uc_fw_init(struct xe_uc_fw *uc_fw) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + struct xe_gt *gt = uc_fw_to_gt(uc_fw); + struct device *dev = xe->drm.dev; + const struct firmware *fw = NULL; + struct uc_css_header *css; + struct xe_bo *obj; + size_t size; + int err; + + /* + * we use FIRMWARE_UNINITIALIZED to detect checks against uc_fw->status + * before we're looked at the HW caps to see if we have uc support + */ + BUILD_BUG_ON(XE_UC_FIRMWARE_UNINITIALIZED); + XE_BUG_ON(uc_fw->status); + XE_BUG_ON(uc_fw->path); + + uc_fw_auto_select(xe, uc_fw); + xe_uc_fw_change_status(uc_fw, uc_fw->path ? *uc_fw->path ? + XE_UC_FIRMWARE_SELECTED : + XE_UC_FIRMWARE_DISABLED : + XE_UC_FIRMWARE_NOT_SUPPORTED); + + /* Transform no huc in the list into firmware disabled */ + if (uc_fw->type == XE_UC_FW_TYPE_HUC && !xe_uc_fw_is_supported(uc_fw)) { + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_DISABLED); + err = -ENOPKG; + return err; + } + err = request_firmware(&fw, uc_fw->path, dev); + if (err) + goto fail; + + /* Check the size of the blob before examining buffer contents */ + if (unlikely(fw->size < sizeof(struct uc_css_header))) { + drm_warn(&xe->drm, "%s firmware %s: invalid size: %zu < %zu\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + fw->size, sizeof(struct uc_css_header)); + err = -ENODATA; + goto fail; + } + + css = (struct uc_css_header *)fw->data; + + /* Check integrity of size values inside CSS header */ + size = (css->header_size_dw - css->key_size_dw - css->modulus_size_dw - + css->exponent_size_dw) * sizeof(u32); + if (unlikely(size != sizeof(struct uc_css_header))) { + drm_warn(&xe->drm, + "%s firmware %s: unexpected header size: %zu != %zu\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + fw->size, sizeof(struct uc_css_header)); + err = -EPROTO; + goto fail; + } + + /* uCode size must calculated from other sizes */ + uc_fw->ucode_size = (css->size_dw - css->header_size_dw) * sizeof(u32); + + /* now RSA */ + uc_fw->rsa_size = css->key_size_dw * sizeof(u32); + + /* At least, it should have header, uCode and RSA. Size of all three. */ + size = sizeof(struct uc_css_header) + uc_fw->ucode_size + + uc_fw->rsa_size; + if (unlikely(fw->size < size)) { + drm_warn(&xe->drm, "%s firmware %s: invalid size: %zu < %zu\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + fw->size, size); + err = -ENOEXEC; + goto fail; + } + + /* Get version numbers from the CSS header */ + uc_fw->major_ver_found = FIELD_GET(CSS_SW_VERSION_UC_MAJOR, + css->sw_version); + uc_fw->minor_ver_found = FIELD_GET(CSS_SW_VERSION_UC_MINOR, + css->sw_version); + + if (uc_fw->major_ver_found != uc_fw->major_ver_wanted || + uc_fw->minor_ver_found < uc_fw->minor_ver_wanted) { + drm_notice(&xe->drm, "%s firmware %s: unexpected version: %u.%u != %u.%u\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + uc_fw->major_ver_found, uc_fw->minor_ver_found, + uc_fw->major_ver_wanted, uc_fw->minor_ver_wanted); + if (!xe_uc_fw_is_overridden(uc_fw)) { + err = -ENOEXEC; + goto fail; + } + } + + if (uc_fw->type == XE_UC_FW_TYPE_GUC) + uc_fw->private_data_size = css->private_data_size; + + obj = xe_bo_create_from_data(xe, gt, fw->data, fw->size, + ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_GGTT_BIT); + if (IS_ERR(obj)) { + drm_notice(&xe->drm, "%s firmware %s: failed to create / populate bo", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path); + err = PTR_ERR(obj); + goto fail; + } + + uc_fw->bo = obj; + uc_fw->size = fw->size; + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_AVAILABLE); + + release_firmware(fw); + + err = drmm_add_action_or_reset(&xe->drm, uc_fw_fini, uc_fw); + if (err) + return err; + + return 0; + +fail: + xe_uc_fw_change_status(uc_fw, err == -ENOENT ? + XE_UC_FIRMWARE_MISSING : + XE_UC_FIRMWARE_ERROR); + + drm_notice(&xe->drm, "%s firmware %s: fetch failed with error %d\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, err); + drm_info(&xe->drm, "%s firmware(s) can be downloaded from %s\n", + xe_uc_fw_type_repr(uc_fw->type), XE_UC_FIRMWARE_URL); + + release_firmware(fw); /* OK even if fw is NULL */ + return err; +} + +static u32 uc_fw_ggtt_offset(struct xe_uc_fw *uc_fw) +{ + return xe_bo_ggtt_addr(uc_fw->bo); +} + +static int uc_fw_xfer(struct xe_uc_fw *uc_fw, u32 offset, u32 dma_flags) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + struct xe_gt *gt = uc_fw_to_gt(uc_fw); + u32 src_offset; + int ret; + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + + /* Set the source address for the uCode */ + src_offset = uc_fw_ggtt_offset(uc_fw); + xe_mmio_write32(gt, DMA_ADDR_0_LOW.reg, lower_32_bits(src_offset)); + xe_mmio_write32(gt, DMA_ADDR_0_HIGH.reg, upper_32_bits(src_offset)); + + /* Set the DMA destination */ + xe_mmio_write32(gt, DMA_ADDR_1_LOW.reg, offset); + xe_mmio_write32(gt, DMA_ADDR_1_HIGH.reg, DMA_ADDRESS_SPACE_WOPCM); + + /* + * Set the transfer size. The header plus uCode will be copied to WOPCM + * via DMA, excluding any other components + */ + xe_mmio_write32(gt, DMA_COPY_SIZE.reg, + sizeof(struct uc_css_header) + uc_fw->ucode_size); + + /* Start the DMA */ + xe_mmio_write32(gt, DMA_CTRL.reg, + _MASKED_BIT_ENABLE(dma_flags | START_DMA)); + + /* Wait for DMA to finish */ + ret = xe_mmio_wait32(gt, DMA_CTRL.reg, 0, START_DMA, 100); + if (ret) + drm_err(&xe->drm, "DMA for %s fw failed, DMA_CTRL=%u\n", + xe_uc_fw_type_repr(uc_fw->type), + xe_mmio_read32(gt, DMA_CTRL.reg)); + + /* Disable the bits once DMA is over */ + xe_mmio_write32(gt, DMA_CTRL.reg, _MASKED_BIT_DISABLE(dma_flags)); + + return ret; +} + +int xe_uc_fw_upload(struct xe_uc_fw *uc_fw, u32 offset, u32 dma_flags) +{ + struct xe_device *xe = uc_fw_to_xe(uc_fw); + int err; + + /* make sure the status was cleared the last time we reset the uc */ + XE_BUG_ON(xe_uc_fw_is_loaded(uc_fw)); + + if (!xe_uc_fw_is_loadable(uc_fw)) + return -ENOEXEC; + + /* Call custom loader */ + err = uc_fw_xfer(uc_fw, offset, dma_flags); + if (err) + goto fail; + + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_TRANSFERRED); + return 0; + +fail: + drm_err(&xe->drm, "Failed to load %s firmware %s (%d)\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path, + err); + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_LOAD_FAIL); + return err; +} + + +void xe_uc_fw_print(struct xe_uc_fw *uc_fw, struct drm_printer *p) +{ + drm_printf(p, "%s firmware: %s\n", + xe_uc_fw_type_repr(uc_fw->type), uc_fw->path); + drm_printf(p, "\tstatus: %s\n", + xe_uc_fw_status_repr(uc_fw->status)); + drm_printf(p, "\tversion: wanted %u.%u, found %u.%u\n", + uc_fw->major_ver_wanted, uc_fw->minor_ver_wanted, + uc_fw->major_ver_found, uc_fw->minor_ver_found); + drm_printf(p, "\tuCode: %u bytes\n", uc_fw->ucode_size); + drm_printf(p, "\tRSA: %u bytes\n", uc_fw->rsa_size); +} diff --git a/drivers/gpu/drm/xe/xe_uc_fw.h b/drivers/gpu/drm/xe/xe_uc_fw.h new file mode 100644 index 000000000000..b0df5064b27d --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_fw.h @@ -0,0 +1,180 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_FW_H_ +#define _XE_UC_FW_H_ + +#include + +#include "xe_uc_fw_types.h" +#include "xe_uc_fw_abi.h" +#include "xe_macros.h" + +struct drm_printer; + +int xe_uc_fw_init(struct xe_uc_fw *uc_fw); +size_t xe_uc_fw_copy_rsa(struct xe_uc_fw *uc_fw, void *dst, u32 max_len); +int xe_uc_fw_upload(struct xe_uc_fw *uc_fw, u32 offset, u32 dma_flags); +void xe_uc_fw_print(struct xe_uc_fw *uc_fw, struct drm_printer *p); + +static inline u32 xe_uc_fw_rsa_offset(struct xe_uc_fw *uc_fw) +{ + return sizeof(struct uc_css_header) + uc_fw->ucode_size; +} + +static inline void xe_uc_fw_change_status(struct xe_uc_fw *uc_fw, + enum xe_uc_fw_status status) +{ + uc_fw->__status = status; +} + +static inline +const char *xe_uc_fw_status_repr(enum xe_uc_fw_status status) +{ + switch (status) { + case XE_UC_FIRMWARE_NOT_SUPPORTED: + return "N/A"; + case XE_UC_FIRMWARE_UNINITIALIZED: + return "UNINITIALIZED"; + case XE_UC_FIRMWARE_DISABLED: + return "DISABLED"; + case XE_UC_FIRMWARE_SELECTED: + return "SELECTED"; + case XE_UC_FIRMWARE_MISSING: + return "MISSING"; + case XE_UC_FIRMWARE_ERROR: + return "ERROR"; + case XE_UC_FIRMWARE_AVAILABLE: + return "AVAILABLE"; + case XE_UC_FIRMWARE_INIT_FAIL: + return "INIT FAIL"; + case XE_UC_FIRMWARE_LOADABLE: + return "LOADABLE"; + case XE_UC_FIRMWARE_LOAD_FAIL: + return "LOAD FAIL"; + case XE_UC_FIRMWARE_TRANSFERRED: + return "TRANSFERRED"; + case XE_UC_FIRMWARE_RUNNING: + return "RUNNING"; + } + return ""; +} + +static inline int xe_uc_fw_status_to_error(enum xe_uc_fw_status status) +{ + switch (status) { + case XE_UC_FIRMWARE_NOT_SUPPORTED: + return -ENODEV; + case XE_UC_FIRMWARE_UNINITIALIZED: + return -EACCES; + case XE_UC_FIRMWARE_DISABLED: + return -EPERM; + case XE_UC_FIRMWARE_MISSING: + return -ENOENT; + case XE_UC_FIRMWARE_ERROR: + return -ENOEXEC; + case XE_UC_FIRMWARE_INIT_FAIL: + case XE_UC_FIRMWARE_LOAD_FAIL: + return -EIO; + case XE_UC_FIRMWARE_SELECTED: + return -ESTALE; + case XE_UC_FIRMWARE_AVAILABLE: + case XE_UC_FIRMWARE_LOADABLE: + case XE_UC_FIRMWARE_TRANSFERRED: + case XE_UC_FIRMWARE_RUNNING: + return 0; + } + return -EINVAL; +} + +static inline const char *xe_uc_fw_type_repr(enum xe_uc_fw_type type) +{ + switch (type) { + case XE_UC_FW_TYPE_GUC: + return "GuC"; + case XE_UC_FW_TYPE_HUC: + return "HuC"; + } + return "uC"; +} + +static inline enum xe_uc_fw_status +__xe_uc_fw_status(struct xe_uc_fw *uc_fw) +{ + /* shouldn't call this before checking hw/blob availability */ + XE_BUG_ON(uc_fw->status == XE_UC_FIRMWARE_UNINITIALIZED); + return uc_fw->status; +} + +static inline bool xe_uc_fw_is_supported(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) != XE_UC_FIRMWARE_NOT_SUPPORTED; +} + +static inline bool xe_uc_fw_is_enabled(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) > XE_UC_FIRMWARE_DISABLED; +} + +static inline bool xe_uc_fw_is_disabled(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) == XE_UC_FIRMWARE_DISABLED; +} + +static inline bool xe_uc_fw_is_available(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_AVAILABLE; +} + +static inline bool xe_uc_fw_is_loadable(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_LOADABLE; +} + +static inline bool xe_uc_fw_is_loaded(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) >= XE_UC_FIRMWARE_TRANSFERRED; +} + +static inline bool xe_uc_fw_is_running(struct xe_uc_fw *uc_fw) +{ + return __xe_uc_fw_status(uc_fw) == XE_UC_FIRMWARE_RUNNING; +} + +static inline bool xe_uc_fw_is_overridden(const struct xe_uc_fw *uc_fw) +{ + return uc_fw->user_overridden; +} + +static inline void xe_uc_fw_sanitize(struct xe_uc_fw *uc_fw) +{ + if (xe_uc_fw_is_loaded(uc_fw)) + xe_uc_fw_change_status(uc_fw, XE_UC_FIRMWARE_LOADABLE); +} + +static inline u32 __xe_uc_fw_get_upload_size(struct xe_uc_fw *uc_fw) +{ + return sizeof(struct uc_css_header) + uc_fw->ucode_size; +} + +/** + * xe_uc_fw_get_upload_size() - Get size of firmware needed to be uploaded. + * @uc_fw: uC firmware. + * + * Get the size of the firmware and header that will be uploaded to WOPCM. + * + * Return: Upload firmware size, or zero on firmware fetch failure. + */ +static inline u32 xe_uc_fw_get_upload_size(struct xe_uc_fw *uc_fw) +{ + if (!xe_uc_fw_is_available(uc_fw)) + return 0; + + return __xe_uc_fw_get_upload_size(uc_fw); +} + +#define XE_UC_FIRMWARE_URL "https://git.kernel.org/pub/scm/linux/kernel/git/firmware/linux-firmware.git/tree/xe" + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc_fw_abi.h b/drivers/gpu/drm/xe/xe_uc_fw_abi.h new file mode 100644 index 000000000000..dafd26cb0c41 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_fw_abi.h @@ -0,0 +1,81 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_FW_ABI_H +#define _XE_UC_FW_ABI_H + +#include +#include + +/** + * DOC: Firmware Layout + * + * The GuC/HuC firmware layout looks like this:: + * + * +======================================================================+ + * | Firmware blob | + * +===============+===============+============+============+============+ + * | CSS header | uCode | RSA key | modulus | exponent | + * +===============+===============+============+============+============+ + * <-header size-> <---header size continued -----------> + * <--- size -----------------------------------------------------------> + * <-key size-> + * <-mod size-> + * <-exp size-> + * + * The firmware may or may not have modulus key and exponent data. The header, + * uCode and RSA signature are must-have components that will be used by driver. + * Length of each components, which is all in dwords, can be found in header. + * In the case that modulus and exponent are not present in fw, a.k.a truncated + * image, the length value still appears in header. + * + * Driver will do some basic fw size validation based on the following rules: + * + * 1. Header, uCode and RSA are must-have components. + * 2. All firmware components, if they present, are in the sequence illustrated + * in the layout table above. + * 3. Length info of each component can be found in header, in dwords. + * 4. Modulus and exponent key are not required by driver. They may not appear + * in fw. So driver will load a truncated firmware in this case. + */ + +struct uc_css_header { + u32 module_type; + /* + * header_size includes all non-uCode bits, including css_header, rsa + * key, modulus key and exponent data. + */ + u32 header_size_dw; + u32 header_version; + u32 module_id; + u32 module_vendor; + u32 date; +#define CSS_DATE_DAY (0xFF << 0) +#define CSS_DATE_MONTH (0xFF << 8) +#define CSS_DATE_YEAR (0xFFFF << 16) + u32 size_dw; /* uCode plus header_size_dw */ + u32 key_size_dw; + u32 modulus_size_dw; + u32 exponent_size_dw; + u32 time; +#define CSS_TIME_HOUR (0xFF << 0) +#define CSS_DATE_MIN (0xFF << 8) +#define CSS_DATE_SEC (0xFFFF << 16) + char username[8]; + char buildnumber[12]; + u32 sw_version; +#define CSS_SW_VERSION_UC_MAJOR (0xFF << 16) +#define CSS_SW_VERSION_UC_MINOR (0xFF << 8) +#define CSS_SW_VERSION_UC_PATCH (0xFF << 0) + u32 reserved0[13]; + union { + u32 private_data_size; /* only applies to GuC */ + u32 reserved1; + }; + u32 header_info; +} __packed; +static_assert(sizeof(struct uc_css_header) == 128); + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc_fw_types.h b/drivers/gpu/drm/xe/xe_uc_fw_types.h new file mode 100644 index 000000000000..1cfd30a655df --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_fw_types.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_FW_TYPES_H_ +#define _XE_UC_FW_TYPES_H_ + +#include + +struct xe_bo; + +/* + * +------------+---------------------------------------------------+ + * | PHASE | FIRMWARE STATUS TRANSITIONS | + * +============+===================================================+ + * | | UNINITIALIZED | + * +------------+- / | \ -+ + * | | DISABLED <--/ | \--> NOT_SUPPORTED | + * | init_early | V | + * | | SELECTED | + * +------------+- / | \ -+ + * | | MISSING <--/ | \--> ERROR | + * | fetch | V | + * | | AVAILABLE | + * +------------+- | \ -+ + * | | | \--> INIT FAIL | + * | init | V | + * | | /------> LOADABLE <----<-----------\ | + * +------------+- \ / \ \ \ -+ + * | | LOAD FAIL <--< \--> TRANSFERRED \ | + * | upload | \ / \ / | + * | | \---------/ \--> RUNNING | + * +------------+---------------------------------------------------+ + */ + +/* + * FIXME: Ported from the i915 and this is state machine is way too complicated. + * Circle back and simplify this. + */ +enum xe_uc_fw_status { + XE_UC_FIRMWARE_NOT_SUPPORTED = -1, /* no uc HW */ + XE_UC_FIRMWARE_UNINITIALIZED = 0, /* used to catch checks done too early */ + XE_UC_FIRMWARE_DISABLED, /* disabled */ + XE_UC_FIRMWARE_SELECTED, /* selected the blob we want to load */ + XE_UC_FIRMWARE_MISSING, /* blob not found on the system */ + XE_UC_FIRMWARE_ERROR, /* invalid format or version */ + XE_UC_FIRMWARE_AVAILABLE, /* blob found and copied in mem */ + XE_UC_FIRMWARE_INIT_FAIL, /* failed to prepare fw objects for load */ + XE_UC_FIRMWARE_LOADABLE, /* all fw-required objects are ready */ + XE_UC_FIRMWARE_LOAD_FAIL, /* failed to xfer or init/auth the fw */ + XE_UC_FIRMWARE_TRANSFERRED, /* dma xfer done */ + XE_UC_FIRMWARE_RUNNING /* init/auth done */ +}; + +enum xe_uc_fw_type { + XE_UC_FW_TYPE_GUC = 0, + XE_UC_FW_TYPE_HUC +}; +#define XE_UC_FW_NUM_TYPES 2 + +/** + * struct xe_uc_fw - XE micro controller firmware + */ +struct xe_uc_fw { + /** @type: type uC firmware */ + enum xe_uc_fw_type type; + union { + /** @status: firmware load status */ + const enum xe_uc_fw_status status; + /** + * @__status: private firmware load status - only to be used + * by firmware laoding code + */ + enum xe_uc_fw_status __status; + }; + /** @path: path to uC firmware */ + const char *path; + /** @user_overridden: user provided path to uC firmware via modparam */ + bool user_overridden; + /** @size: size of uC firmware including css header */ + size_t size; + + /** @bo: XE BO for uC firmware */ + struct xe_bo *bo; + + /* + * The firmware build process will generate a version header file with + * major and minor version defined. The versions are built into CSS + * header of firmware. The xe kernel driver set the minimal firmware + * version required per platform. + */ + + /** @major_ver_wanted: major firmware version wanted by platform */ + u16 major_ver_wanted; + /** @minor_ver_wanted: minor firmware version wanted by platform */ + u16 minor_ver_wanted; + /** @major_ver_found: major version found in firmware blob */ + u16 major_ver_found; + /** @minor_ver_found: major version found in firmware blob */ + u16 minor_ver_found; + + /** @rsa_size: RSA size */ + u32 rsa_size; + /** @ucode_size: micro kernel size */ + u32 ucode_size; + + /** @private_data_size: size of private data found in uC css header */ + u32 private_data_size; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_uc_types.h b/drivers/gpu/drm/xe/xe_uc_types.h new file mode 100644 index 000000000000..49bef6498b85 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_uc_types.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_UC_TYPES_H_ +#define _XE_UC_TYPES_H_ + +#include "xe_guc_types.h" +#include "xe_huc_types.h" +#include "xe_wopcm_types.h" + +/** + * struct xe_uc - XE micro controllers + */ +struct xe_uc { + /** @guc: Graphics micro controller */ + struct xe_guc guc; + /** @huc: HuC */ + struct xe_huc huc; + /** @wopcm: WOPCM */ + struct xe_wopcm wopcm; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c new file mode 100644 index 000000000000..d47a8617c5b6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -0,0 +1,3407 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include "xe_vm.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include "xe_bo.h" +#include "xe_device.h" +#include "xe_engine.h" +#include "xe_gt.h" +#include "xe_gt_pagefault.h" +#include "xe_migrate.h" +#include "xe_pm.h" +#include "xe_preempt_fence.h" +#include "xe_pt.h" +#include "xe_res_cursor.h" +#include "xe_trace.h" +#include "xe_sync.h" + +#define TEST_VM_ASYNC_OPS_ERROR + +/** + * xe_vma_userptr_check_repin() - Advisory check for repin needed + * @vma: The userptr vma + * + * Check if the userptr vma has been invalidated since last successful + * repin. The check is advisory only and can the function can be called + * without the vm->userptr.notifier_lock held. There is no guarantee that the + * vma userptr will remain valid after a lockless check, so typically + * the call needs to be followed by a proper check under the notifier_lock. + * + * Return: 0 if userptr vma is valid, -EAGAIN otherwise; repin recommended. + */ +int xe_vma_userptr_check_repin(struct xe_vma *vma) +{ + return mmu_interval_check_retry(&vma->userptr.notifier, + vma->userptr.notifier_seq) ? + -EAGAIN : 0; +} + +int xe_vma_userptr_pin_pages(struct xe_vma *vma) +{ + struct xe_vm *vm = vma->vm; + struct xe_device *xe = vm->xe; + const unsigned long num_pages = + (vma->end - vma->start + 1) >> PAGE_SHIFT; + struct page **pages; + bool in_kthread = !current->mm; + unsigned long notifier_seq; + int pinned, ret, i; + bool read_only = vma->pte_flags & PTE_READ_ONLY; + + lockdep_assert_held(&vm->lock); + XE_BUG_ON(!xe_vma_is_userptr(vma)); +retry: + if (vma->destroyed) + return 0; + + notifier_seq = mmu_interval_read_begin(&vma->userptr.notifier); + if (notifier_seq == vma->userptr.notifier_seq) + return 0; + + pages = kvmalloc_array(num_pages, sizeof(*pages), GFP_KERNEL); + if (!pages) + return -ENOMEM; + + if (vma->userptr.sg) { + dma_unmap_sgtable(xe->drm.dev, + vma->userptr.sg, + read_only ? DMA_TO_DEVICE : + DMA_BIDIRECTIONAL, 0); + sg_free_table(vma->userptr.sg); + vma->userptr.sg = NULL; + } + + pinned = ret = 0; + if (in_kthread) { + if (!mmget_not_zero(vma->userptr.notifier.mm)) { + ret = -EFAULT; + goto mm_closed; + } + kthread_use_mm(vma->userptr.notifier.mm); + } + + while (pinned < num_pages) { + ret = get_user_pages_fast(vma->userptr.ptr + pinned * PAGE_SIZE, + num_pages - pinned, + read_only ? 0 : FOLL_WRITE, + &pages[pinned]); + if (ret < 0) { + if (in_kthread) + ret = 0; + break; + } + + pinned += ret; + ret = 0; + } + + if (in_kthread) { + kthread_unuse_mm(vma->userptr.notifier.mm); + mmput(vma->userptr.notifier.mm); + } +mm_closed: + if (ret) + goto out; + + ret = sg_alloc_table_from_pages(&vma->userptr.sgt, pages, pinned, + 0, (u64)pinned << PAGE_SHIFT, + GFP_KERNEL); + if (ret) { + vma->userptr.sg = NULL; + goto out; + } + vma->userptr.sg = &vma->userptr.sgt; + + ret = dma_map_sgtable(xe->drm.dev, vma->userptr.sg, + read_only ? DMA_TO_DEVICE : + DMA_BIDIRECTIONAL, + DMA_ATTR_SKIP_CPU_SYNC | + DMA_ATTR_NO_KERNEL_MAPPING); + if (ret) { + sg_free_table(vma->userptr.sg); + vma->userptr.sg = NULL; + goto out; + } + + for (i = 0; i < pinned; ++i) { + if (!read_only) { + lock_page(pages[i]); + set_page_dirty(pages[i]); + unlock_page(pages[i]); + } + + mark_page_accessed(pages[i]); + } + +out: + release_pages(pages, pinned); + kvfree(pages); + + if (!(ret < 0)) { + vma->userptr.notifier_seq = notifier_seq; + if (xe_vma_userptr_check_repin(vma) == -EAGAIN) + goto retry; + } + + return ret < 0 ? ret : 0; +} + +static bool preempt_fences_waiting(struct xe_vm *vm) +{ + struct xe_engine *e; + + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + + list_for_each_entry(e, &vm->preempt.engines, compute.link) { + if (!e->compute.pfence || (e->compute.pfence && + test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, + &e->compute.pfence->flags))) { + return true; + } + } + + return false; +} + +static void free_preempt_fences(struct list_head *list) +{ + struct list_head *link, *next; + + list_for_each_safe(link, next, list) + xe_preempt_fence_free(to_preempt_fence_from_link(link)); +} + +static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list, + unsigned int *count) +{ + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + + if (*count >= vm->preempt.num_engines) + return 0; + + for (; *count < vm->preempt.num_engines; ++(*count)) { + struct xe_preempt_fence *pfence = xe_preempt_fence_alloc(); + + if (IS_ERR(pfence)) + return PTR_ERR(pfence); + + list_move_tail(xe_preempt_fence_link(pfence), list); + } + + return 0; +} + +static int wait_for_existing_preempt_fences(struct xe_vm *vm) +{ + struct xe_engine *e; + + xe_vm_assert_held(vm); + + list_for_each_entry(e, &vm->preempt.engines, compute.link) { + if (e->compute.pfence) { + long timeout = dma_fence_wait(e->compute.pfence, false); + + if (timeout < 0) + return -ETIME; + dma_fence_put(e->compute.pfence); + e->compute.pfence = NULL; + } + } + + return 0; +} + +static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list) +{ + struct list_head *link; + struct xe_engine *e; + + list_for_each_entry(e, &vm->preempt.engines, compute.link) { + struct dma_fence *fence; + + link = list->next; + XE_BUG_ON(link == list); + + fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link), + e, e->compute.context, + ++e->compute.seqno); + dma_fence_put(e->compute.pfence); + e->compute.pfence = fence; + } +} + +static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo) +{ + struct xe_engine *e; + struct ww_acquire_ctx ww; + int err; + + err = xe_bo_lock(bo, &ww, vm->preempt.num_engines, true); + if (err) + return err; + + list_for_each_entry(e, &vm->preempt.engines, compute.link) + if (e->compute.pfence) { + dma_resv_add_fence(bo->ttm.base.resv, + e->compute.pfence, + DMA_RESV_USAGE_BOOKKEEP); + } + + xe_bo_unlock(bo, &ww); + return 0; +} + +/** + * xe_vm_fence_all_extobjs() - Add a fence to vm's external objects' resv + * @vm: The vm. + * @fence: The fence to add. + * @usage: The resv usage for the fence. + * + * Loops over all of the vm's external object bindings and adds a @fence + * with the given @usage to all of the external object's reservation + * objects. + */ +void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence, + enum dma_resv_usage usage) +{ + struct xe_vma *vma; + + list_for_each_entry(vma, &vm->extobj.list, extobj.link) + dma_resv_add_fence(vma->bo->ttm.base.resv, fence, usage); +} + +static void resume_and_reinstall_preempt_fences(struct xe_vm *vm) +{ + struct xe_engine *e; + + lockdep_assert_held(&vm->lock); + xe_vm_assert_held(vm); + + list_for_each_entry(e, &vm->preempt.engines, compute.link) { + e->ops->resume(e); + + dma_resv_add_fence(&vm->resv, e->compute.pfence, + DMA_RESV_USAGE_BOOKKEEP); + xe_vm_fence_all_extobjs(vm, e->compute.pfence, + DMA_RESV_USAGE_BOOKKEEP); + } +} + +int xe_vm_add_compute_engine(struct xe_vm *vm, struct xe_engine *e) +{ + struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV]; + struct ttm_validate_buffer *tv; + struct ww_acquire_ctx ww; + struct list_head objs; + struct dma_fence *pfence; + int err; + bool wait; + + XE_BUG_ON(!xe_vm_in_compute_mode(vm)); + + down_write(&vm->lock); + + err = xe_vm_lock_dma_resv(vm, &ww, tv_onstack, &tv, &objs, true, 1); + if (err) + goto out_unlock_outer; + + pfence = xe_preempt_fence_create(e, e->compute.context, + ++e->compute.seqno); + if (!pfence) { + err = -ENOMEM; + goto out_unlock; + } + + list_add(&e->compute.link, &vm->preempt.engines); + ++vm->preempt.num_engines; + e->compute.pfence = pfence; + + down_read(&vm->userptr.notifier_lock); + + dma_resv_add_fence(&vm->resv, pfence, + DMA_RESV_USAGE_BOOKKEEP); + + xe_vm_fence_all_extobjs(vm, pfence, DMA_RESV_USAGE_BOOKKEEP); + + /* + * Check to see if a preemption on VM is in flight or userptr + * invalidation, if so trigger this preempt fence to sync state with + * other preempt fences on the VM. + */ + wait = __xe_vm_userptr_needs_repin(vm) || preempt_fences_waiting(vm); + if (wait) + dma_fence_enable_sw_signaling(pfence); + + up_read(&vm->userptr.notifier_lock); + +out_unlock: + xe_vm_unlock_dma_resv(vm, tv_onstack, tv, &ww, &objs); +out_unlock_outer: + up_write(&vm->lock); + + return err; +} + +/** + * __xe_vm_userptr_needs_repin() - Check whether the VM does have userptrs + * that need repinning. + * @vm: The VM. + * + * This function checks for whether the VM has userptrs that need repinning, + * and provides a release-type barrier on the userptr.notifier_lock after + * checking. + * + * Return: 0 if there are no userptrs needing repinning, -EAGAIN if there are. + */ +int __xe_vm_userptr_needs_repin(struct xe_vm *vm) +{ + lockdep_assert_held_read(&vm->userptr.notifier_lock); + + return (list_empty(&vm->userptr.repin_list) && + list_empty(&vm->userptr.invalidated)) ? 0 : -EAGAIN; +} + +/** + * xe_vm_lock_dma_resv() - Lock the vm dma_resv object and the dma_resv + * objects of the vm's external buffer objects. + * @vm: The vm. + * @ww: Pointer to a struct ww_acquire_ctx locking context. + * @tv_onstack: Array size XE_ONSTACK_TV of storage for the struct + * ttm_validate_buffers used for locking. + * @tv: Pointer to a pointer that on output contains the actual storage used. + * @objs: List head for the buffer objects locked. + * @intr: Whether to lock interruptible. + * @num_shared: Number of dma-fence slots to reserve in the locked objects. + * + * Locks the vm dma-resv objects and all the dma-resv objects of the + * buffer objects on the vm external object list. The TTM utilities require + * a list of struct ttm_validate_buffers pointing to the actual buffer + * objects to lock. Storage for those struct ttm_validate_buffers should + * be provided in @tv_onstack, and is typically reserved on the stack + * of the caller. If the size of @tv_onstack isn't sufficient, then + * storage will be allocated internally using kvmalloc(). + * + * The function performs deadlock handling internally, and after a + * successful return the ww locking transaction should be considered + * sealed. + * + * Return: 0 on success, Negative error code on error. In particular if + * @intr is set to true, -EINTR or -ERESTARTSYS may be returned. In case + * of error, any locking performed has been reverted. + */ +int xe_vm_lock_dma_resv(struct xe_vm *vm, struct ww_acquire_ctx *ww, + struct ttm_validate_buffer *tv_onstack, + struct ttm_validate_buffer **tv, + struct list_head *objs, + bool intr, + unsigned int num_shared) +{ + struct ttm_validate_buffer *tv_vm, *tv_bo; + struct xe_vma *vma, *next; + LIST_HEAD(dups); + int err; + + lockdep_assert_held(&vm->lock); + + if (vm->extobj.entries < XE_ONSTACK_TV) { + tv_vm = tv_onstack; + } else { + tv_vm = kvmalloc_array(vm->extobj.entries + 1, sizeof(*tv_vm), + GFP_KERNEL); + if (!tv_vm) + return -ENOMEM; + } + tv_bo = tv_vm + 1; + + INIT_LIST_HEAD(objs); + list_for_each_entry(vma, &vm->extobj.list, extobj.link) { + tv_bo->num_shared = num_shared; + tv_bo->bo = &vma->bo->ttm; + + list_add_tail(&tv_bo->head, objs); + tv_bo++; + } + tv_vm->num_shared = num_shared; + tv_vm->bo = xe_vm_ttm_bo(vm); + list_add_tail(&tv_vm->head, objs); + err = ttm_eu_reserve_buffers(ww, objs, intr, &dups); + if (err) + goto out_err; + + spin_lock(&vm->notifier.list_lock); + list_for_each_entry_safe(vma, next, &vm->notifier.rebind_list, + notifier.rebind_link) { + xe_bo_assert_held(vma->bo); + + list_del_init(&vma->notifier.rebind_link); + if (vma->gt_present && !vma->destroyed) + list_move_tail(&vma->rebind_link, &vm->rebind_list); + } + spin_unlock(&vm->notifier.list_lock); + + *tv = tv_vm; + return 0; + +out_err: + if (tv_vm != tv_onstack) + kvfree(tv_vm); + + return err; +} + +/** + * xe_vm_unlock_dma_resv() - Unlock reservation objects locked by + * xe_vm_lock_dma_resv() + * @vm: The vm. + * @tv_onstack: The @tv_onstack array given to xe_vm_lock_dma_resv(). + * @tv: The value of *@tv given by xe_vm_lock_dma_resv(). + * @ww: The ww_acquire_context used for locking. + * @objs: The list returned from xe_vm_lock_dma_resv(). + * + * Unlocks the reservation objects and frees any memory allocated by + * xe_vm_lock_dma_resv(). + */ +void xe_vm_unlock_dma_resv(struct xe_vm *vm, + struct ttm_validate_buffer *tv_onstack, + struct ttm_validate_buffer *tv, + struct ww_acquire_ctx *ww, + struct list_head *objs) +{ + /* + * Nothing should've been able to enter the list while we were locked, + * since we've held the dma-resvs of all the vm's external objects, + * and holding the dma_resv of an object is required for list + * addition, and we shouldn't add ourselves. + */ + XE_WARN_ON(!list_empty(&vm->notifier.rebind_list)); + + ttm_eu_backoff_reservation(ww, objs); + if (tv && tv != tv_onstack) + kvfree(tv); +} + +static void preempt_rebind_work_func(struct work_struct *w) +{ + struct xe_vm *vm = container_of(w, struct xe_vm, preempt.rebind_work); + struct xe_vma *vma; + struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV]; + struct ttm_validate_buffer *tv; + struct ww_acquire_ctx ww; + struct list_head objs; + struct dma_fence *rebind_fence; + unsigned int fence_count = 0; + LIST_HEAD(preempt_fences); + int err; + long wait; + int __maybe_unused tries = 0; + + XE_BUG_ON(!xe_vm_in_compute_mode(vm)); + trace_xe_vm_rebind_worker_enter(vm); + + if (xe_vm_is_closed(vm)) { + trace_xe_vm_rebind_worker_exit(vm); + return; + } + + down_write(&vm->lock); + +retry: + if (vm->async_ops.error) + goto out_unlock_outer; + + /* + * Extreme corner where we exit a VM error state with a munmap style VM + * unbind inflight which requires a rebind. In this case the rebind + * needs to install some fences into the dma-resv slots. The worker to + * do this queued, let that worker make progress by dropping vm->lock + * and trying this again. + */ + if (vm->async_ops.munmap_rebind_inflight) { + up_write(&vm->lock); + flush_work(&vm->async_ops.work); + goto retry; + } + + if (xe_vm_userptr_check_repin(vm)) { + err = xe_vm_userptr_pin(vm); + if (err) + goto out_unlock_outer; + } + + err = xe_vm_lock_dma_resv(vm, &ww, tv_onstack, &tv, &objs, + false, vm->preempt.num_engines); + if (err) + goto out_unlock_outer; + + /* Fresh preempt fences already installed. Everyting is running. */ + if (!preempt_fences_waiting(vm)) + goto out_unlock; + + /* + * This makes sure vm is completely suspended and also balances + * xe_engine suspend- and resume; we resume *all* vm engines below. + */ + err = wait_for_existing_preempt_fences(vm); + if (err) + goto out_unlock; + + err = alloc_preempt_fences(vm, &preempt_fences, &fence_count); + if (err) + goto out_unlock; + + list_for_each_entry(vma, &vm->rebind_list, rebind_link) { + if (xe_vma_is_userptr(vma) || vma->destroyed) + continue; + + err = xe_bo_validate(vma->bo, vm, false); + if (err) + goto out_unlock; + } + + rebind_fence = xe_vm_rebind(vm, true); + if (IS_ERR(rebind_fence)) { + err = PTR_ERR(rebind_fence); + goto out_unlock; + } + + if (rebind_fence) { + dma_fence_wait(rebind_fence, false); + dma_fence_put(rebind_fence); + } + + /* Wait on munmap style VM unbinds */ + wait = dma_resv_wait_timeout(&vm->resv, + DMA_RESV_USAGE_KERNEL, + false, MAX_SCHEDULE_TIMEOUT); + if (wait <= 0) { + err = -ETIME; + goto out_unlock; + } + +#define retry_required(__tries, __vm) \ + (IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) ? \ + (!(__tries)++ || __xe_vm_userptr_needs_repin(__vm)) : \ + __xe_vm_userptr_needs_repin(__vm)) + + down_read(&vm->userptr.notifier_lock); + if (retry_required(tries, vm)) { + up_read(&vm->userptr.notifier_lock); + err = -EAGAIN; + goto out_unlock; + } + +#undef retry_required + + /* Point of no return. */ + arm_preempt_fences(vm, &preempt_fences); + resume_and_reinstall_preempt_fences(vm); + up_read(&vm->userptr.notifier_lock); + +out_unlock: + xe_vm_unlock_dma_resv(vm, tv_onstack, tv, &ww, &objs); +out_unlock_outer: + if (err == -EAGAIN) { + trace_xe_vm_rebind_worker_retry(vm); + goto retry; + } + up_write(&vm->lock); + + free_preempt_fences(&preempt_fences); + + XE_WARN_ON(err < 0); /* TODO: Kill VM or put in error state */ + trace_xe_vm_rebind_worker_exit(vm); +} + +struct async_op_fence; +static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, + struct xe_engine *e, struct xe_sync_entry *syncs, + u32 num_syncs, struct async_op_fence *afence); + +static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni, + const struct mmu_notifier_range *range, + unsigned long cur_seq) +{ + struct xe_vma *vma = container_of(mni, struct xe_vma, userptr.notifier); + struct xe_vm *vm = vma->vm; + struct dma_resv_iter cursor; + struct dma_fence *fence; + long err; + + XE_BUG_ON(!xe_vma_is_userptr(vma)); + trace_xe_vma_userptr_invalidate(vma); + + if (!mmu_notifier_range_blockable(range)) + return false; + + down_write(&vm->userptr.notifier_lock); + mmu_interval_set_seq(mni, cur_seq); + + /* No need to stop gpu access if the userptr is not yet bound. */ + if (!vma->userptr.initial_bind) { + up_write(&vm->userptr.notifier_lock); + return true; + } + + /* + * Tell exec and rebind worker they need to repin and rebind this + * userptr. + */ + if (!xe_vm_in_fault_mode(vm) && !vma->destroyed && vma->gt_present) { + spin_lock(&vm->userptr.invalidated_lock); + list_move_tail(&vma->userptr.invalidate_link, + &vm->userptr.invalidated); + spin_unlock(&vm->userptr.invalidated_lock); + } + + up_write(&vm->userptr.notifier_lock); + + /* + * Preempt fences turn into schedule disables, pipeline these. + * Note that even in fault mode, we need to wait for binds and + * unbinds to complete, and those are attached as BOOKMARK fences + * to the vm. + */ + dma_resv_iter_begin(&cursor, &vm->resv, + DMA_RESV_USAGE_BOOKKEEP); + dma_resv_for_each_fence_unlocked(&cursor, fence) + dma_fence_enable_sw_signaling(fence); + dma_resv_iter_end(&cursor); + + err = dma_resv_wait_timeout(&vm->resv, + DMA_RESV_USAGE_BOOKKEEP, + false, MAX_SCHEDULE_TIMEOUT); + XE_WARN_ON(err <= 0); + + if (xe_vm_in_fault_mode(vm)) { + err = xe_vm_invalidate_vma(vma); + XE_WARN_ON(err); + } + + trace_xe_vma_userptr_invalidate_complete(vma); + + return true; +} + +static const struct mmu_interval_notifier_ops vma_userptr_notifier_ops = { + .invalidate = vma_userptr_invalidate, +}; + +int xe_vm_userptr_pin(struct xe_vm *vm) +{ + struct xe_vma *vma, *next; + int err = 0; + LIST_HEAD(tmp_evict); + + lockdep_assert_held_write(&vm->lock); + + /* Collect invalidated userptrs */ + spin_lock(&vm->userptr.invalidated_lock); + list_for_each_entry_safe(vma, next, &vm->userptr.invalidated, + userptr.invalidate_link) { + list_del_init(&vma->userptr.invalidate_link); + list_move_tail(&vma->userptr_link, &vm->userptr.repin_list); + } + spin_unlock(&vm->userptr.invalidated_lock); + + /* Pin and move to temporary list */ + list_for_each_entry_safe(vma, next, &vm->userptr.repin_list, userptr_link) { + err = xe_vma_userptr_pin_pages(vma); + if (err < 0) + goto out_err; + + list_move_tail(&vma->userptr_link, &tmp_evict); + } + + /* Take lock and move to rebind_list for rebinding. */ + err = dma_resv_lock_interruptible(&vm->resv, NULL); + if (err) + goto out_err; + + list_for_each_entry_safe(vma, next, &tmp_evict, userptr_link) { + list_del_init(&vma->userptr_link); + list_move_tail(&vma->rebind_link, &vm->rebind_list); + } + + dma_resv_unlock(&vm->resv); + + return 0; + +out_err: + list_splice_tail(&tmp_evict, &vm->userptr.repin_list); + + return err; +} + +/** + * xe_vm_userptr_check_repin() - Check whether the VM might have userptrs + * that need repinning. + * @vm: The VM. + * + * This function does an advisory check for whether the VM has userptrs that + * need repinning. + * + * Return: 0 if there are no indications of userptrs needing repinning, + * -EAGAIN if there are. + */ +int xe_vm_userptr_check_repin(struct xe_vm *vm) +{ + return (list_empty_careful(&vm->userptr.repin_list) && + list_empty_careful(&vm->userptr.invalidated)) ? 0 : -EAGAIN; +} + +static struct dma_fence * +xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e, + struct xe_sync_entry *syncs, u32 num_syncs); + +struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) +{ + struct dma_fence *fence = NULL; + struct xe_vma *vma, *next; + + lockdep_assert_held(&vm->lock); + if (xe_vm_no_dma_fences(vm) && !rebind_worker) + return NULL; + + xe_vm_assert_held(vm); + list_for_each_entry_safe(vma, next, &vm->rebind_list, rebind_link) { + XE_WARN_ON(!vma->gt_present); + + list_del_init(&vma->rebind_link); + dma_fence_put(fence); + if (rebind_worker) + trace_xe_vma_rebind_worker(vma); + else + trace_xe_vma_rebind_exec(vma); + fence = xe_vm_bind_vma(vma, NULL, NULL, 0); + if (IS_ERR(fence)) + return fence; + } + + return fence; +} + +static struct xe_vma *xe_vma_create(struct xe_vm *vm, + struct xe_bo *bo, + u64 bo_offset_or_userptr, + u64 start, u64 end, + bool read_only, + u64 gt_mask) +{ + struct xe_vma *vma; + struct xe_gt *gt; + u8 id; + + XE_BUG_ON(start >= end); + XE_BUG_ON(end >= vm->size); + + vma = kzalloc(sizeof(*vma), GFP_KERNEL); + if (!vma) { + vma = ERR_PTR(-ENOMEM); + return vma; + } + + INIT_LIST_HEAD(&vma->rebind_link); + INIT_LIST_HEAD(&vma->unbind_link); + INIT_LIST_HEAD(&vma->userptr_link); + INIT_LIST_HEAD(&vma->userptr.invalidate_link); + INIT_LIST_HEAD(&vma->notifier.rebind_link); + INIT_LIST_HEAD(&vma->extobj.link); + + vma->vm = vm; + vma->start = start; + vma->end = end; + if (read_only) + vma->pte_flags = PTE_READ_ONLY; + + if (gt_mask) { + vma->gt_mask = gt_mask; + } else { + for_each_gt(gt, vm->xe, id) + if (!xe_gt_is_media_type(gt)) + vma->gt_mask |= 0x1 << id; + } + + if (vm->xe->info.platform == XE_PVC) + vma->use_atomic_access_pte_bit = true; + + if (bo) { + xe_bo_assert_held(bo); + vma->bo_offset = bo_offset_or_userptr; + vma->bo = xe_bo_get(bo); + list_add_tail(&vma->bo_link, &bo->vmas); + } else /* userptr */ { + u64 size = end - start + 1; + int err; + + vma->userptr.ptr = bo_offset_or_userptr; + + err = mmu_interval_notifier_insert(&vma->userptr.notifier, + current->mm, + vma->userptr.ptr, size, + &vma_userptr_notifier_ops); + if (err) { + kfree(vma); + vma = ERR_PTR(err); + return vma; + } + + vma->userptr.notifier_seq = LONG_MAX; + xe_vm_get(vm); + } + + return vma; +} + +static bool vm_remove_extobj(struct xe_vma *vma) +{ + if (!list_empty(&vma->extobj.link)) { + vma->vm->extobj.entries--; + list_del_init(&vma->extobj.link); + return true; + } + return false; +} + +static void xe_vma_destroy_late(struct xe_vma *vma) +{ + struct xe_vm *vm = vma->vm; + struct xe_device *xe = vm->xe; + bool read_only = vma->pte_flags & PTE_READ_ONLY; + + if (xe_vma_is_userptr(vma)) { + if (vma->userptr.sg) { + dma_unmap_sgtable(xe->drm.dev, + vma->userptr.sg, + read_only ? DMA_TO_DEVICE : + DMA_BIDIRECTIONAL, 0); + sg_free_table(vma->userptr.sg); + vma->userptr.sg = NULL; + } + + /* + * Since userptr pages are not pinned, we can't remove + * the notifer until we're sure the GPU is not accessing + * them anymore + */ + mmu_interval_notifier_remove(&vma->userptr.notifier); + xe_vm_put(vm); + } else { + xe_bo_put(vma->bo); + } + + kfree(vma); +} + +static void vma_destroy_work_func(struct work_struct *w) +{ + struct xe_vma *vma = + container_of(w, struct xe_vma, destroy_work); + + xe_vma_destroy_late(vma); +} + +static struct xe_vma * +bo_has_vm_references_locked(struct xe_bo *bo, struct xe_vm *vm, + struct xe_vma *ignore) +{ + struct xe_vma *vma; + + list_for_each_entry(vma, &bo->vmas, bo_link) { + if (vma != ignore && vma->vm == vm && !vma->destroyed) + return vma; + } + + return NULL; +} + +static bool bo_has_vm_references(struct xe_bo *bo, struct xe_vm *vm, + struct xe_vma *ignore) +{ + struct ww_acquire_ctx ww; + bool ret; + + xe_bo_lock(bo, &ww, 0, false); + ret = !!bo_has_vm_references_locked(bo, vm, ignore); + xe_bo_unlock(bo, &ww); + + return ret; +} + +static void __vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma) +{ + list_add(&vma->extobj.link, &vm->extobj.list); + vm->extobj.entries++; +} + +static void vm_insert_extobj(struct xe_vm *vm, struct xe_vma *vma) +{ + struct xe_bo *bo = vma->bo; + + lockdep_assert_held_write(&vm->lock); + + if (bo_has_vm_references(bo, vm, vma)) + return; + + __vm_insert_extobj(vm, vma); +} + +static void vma_destroy_cb(struct dma_fence *fence, + struct dma_fence_cb *cb) +{ + struct xe_vma *vma = container_of(cb, struct xe_vma, destroy_cb); + + INIT_WORK(&vma->destroy_work, vma_destroy_work_func); + queue_work(system_unbound_wq, &vma->destroy_work); +} + +static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) +{ + struct xe_vm *vm = vma->vm; + + lockdep_assert_held_write(&vm->lock); + XE_BUG_ON(!list_empty(&vma->unbind_link)); + + if (xe_vma_is_userptr(vma)) { + XE_WARN_ON(!vma->destroyed); + spin_lock(&vm->userptr.invalidated_lock); + list_del_init(&vma->userptr.invalidate_link); + spin_unlock(&vm->userptr.invalidated_lock); + list_del(&vma->userptr_link); + } else { + xe_bo_assert_held(vma->bo); + list_del(&vma->bo_link); + + spin_lock(&vm->notifier.list_lock); + list_del(&vma->notifier.rebind_link); + spin_unlock(&vm->notifier.list_lock); + + if (!vma->bo->vm && vm_remove_extobj(vma)) { + struct xe_vma *other; + + other = bo_has_vm_references_locked(vma->bo, vm, NULL); + + if (other) + __vm_insert_extobj(vm, other); + } + } + + xe_vm_assert_held(vm); + if (!list_empty(&vma->rebind_link)) + list_del(&vma->rebind_link); + + if (fence) { + int ret = dma_fence_add_callback(fence, &vma->destroy_cb, + vma_destroy_cb); + + if (ret) { + XE_WARN_ON(ret != -ENOENT); + xe_vma_destroy_late(vma); + } + } else { + xe_vma_destroy_late(vma); + } +} + +static void xe_vma_destroy_unlocked(struct xe_vma *vma) +{ + struct ttm_validate_buffer tv[2]; + struct ww_acquire_ctx ww; + struct xe_bo *bo = vma->bo; + LIST_HEAD(objs); + LIST_HEAD(dups); + int err; + + memset(tv, 0, sizeof(tv)); + tv[0].bo = xe_vm_ttm_bo(vma->vm); + list_add(&tv[0].head, &objs); + + if (bo) { + tv[1].bo = &xe_bo_get(bo)->ttm; + list_add(&tv[1].head, &objs); + } + err = ttm_eu_reserve_buffers(&ww, &objs, false, &dups); + XE_WARN_ON(err); + + xe_vma_destroy(vma, NULL); + + ttm_eu_backoff_reservation(&ww, &objs); + if (bo) + xe_bo_put(bo); +} + +static struct xe_vma *to_xe_vma(const struct rb_node *node) +{ + BUILD_BUG_ON(offsetof(struct xe_vma, vm_node) != 0); + return (struct xe_vma *)node; +} + +static int xe_vma_cmp(const struct xe_vma *a, const struct xe_vma *b) +{ + if (a->end < b->start) { + return -1; + } else if (b->end < a->start) { + return 1; + } else { + return 0; + } +} + +static bool xe_vma_less_cb(struct rb_node *a, const struct rb_node *b) +{ + return xe_vma_cmp(to_xe_vma(a), to_xe_vma(b)) < 0; +} + +int xe_vma_cmp_vma_cb(const void *key, const struct rb_node *node) +{ + struct xe_vma *cmp = to_xe_vma(node); + const struct xe_vma *own = key; + + if (own->start > cmp->end) + return 1; + + if (own->end < cmp->start) + return -1; + + return 0; +} + +struct xe_vma * +xe_vm_find_overlapping_vma(struct xe_vm *vm, const struct xe_vma *vma) +{ + struct rb_node *node; + + if (xe_vm_is_closed(vm)) + return NULL; + + XE_BUG_ON(vma->end >= vm->size); + lockdep_assert_held(&vm->lock); + + node = rb_find(vma, &vm->vmas, xe_vma_cmp_vma_cb); + + return node ? to_xe_vma(node) : NULL; +} + +static void xe_vm_insert_vma(struct xe_vm *vm, struct xe_vma *vma) +{ + XE_BUG_ON(vma->vm != vm); + lockdep_assert_held(&vm->lock); + + rb_add(&vma->vm_node, &vm->vmas, xe_vma_less_cb); +} + +static void xe_vm_remove_vma(struct xe_vm *vm, struct xe_vma *vma) +{ + XE_BUG_ON(vma->vm != vm); + lockdep_assert_held(&vm->lock); + + rb_erase(&vma->vm_node, &vm->vmas); + if (vm->usm.last_fault_vma == vma) + vm->usm.last_fault_vma = NULL; +} + +static void async_op_work_func(struct work_struct *w); +static void vm_destroy_work_func(struct work_struct *w); + +struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) +{ + struct xe_vm *vm; + int err, i = 0, number_gts = 0; + struct xe_gt *gt; + u8 id; + + vm = kzalloc(sizeof(*vm), GFP_KERNEL); + if (!vm) + return ERR_PTR(-ENOMEM); + + vm->xe = xe; + kref_init(&vm->refcount); + dma_resv_init(&vm->resv); + + vm->size = 1ull << xe_pt_shift(xe->info.vm_max_level + 1); + + vm->vmas = RB_ROOT; + vm->flags = flags; + + init_rwsem(&vm->lock); + + INIT_LIST_HEAD(&vm->rebind_list); + + INIT_LIST_HEAD(&vm->userptr.repin_list); + INIT_LIST_HEAD(&vm->userptr.invalidated); + init_rwsem(&vm->userptr.notifier_lock); + spin_lock_init(&vm->userptr.invalidated_lock); + + INIT_LIST_HEAD(&vm->notifier.rebind_list); + spin_lock_init(&vm->notifier.list_lock); + + INIT_LIST_HEAD(&vm->async_ops.pending); + INIT_WORK(&vm->async_ops.work, async_op_work_func); + spin_lock_init(&vm->async_ops.lock); + + INIT_WORK(&vm->destroy_work, vm_destroy_work_func); + + INIT_LIST_HEAD(&vm->preempt.engines); + vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */ + + INIT_LIST_HEAD(&vm->extobj.list); + + if (!(flags & XE_VM_FLAG_MIGRATION)) { + /* We need to immeditatelly exit from any D3 state */ + xe_pm_runtime_get(xe); + xe_device_mem_access_get(xe); + } + + err = dma_resv_lock_interruptible(&vm->resv, NULL); + if (err) + goto err_put; + + if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) + vm->flags |= XE_VM_FLAGS_64K; + + for_each_gt(gt, xe, id) { + if (xe_gt_is_media_type(gt)) + continue; + + if (flags & XE_VM_FLAG_MIGRATION && + gt->info.id != XE_VM_FLAG_GT_ID(flags)) + continue; + + vm->pt_root[id] = xe_pt_create(vm, gt, xe->info.vm_max_level); + if (IS_ERR(vm->pt_root[id])) { + err = PTR_ERR(vm->pt_root[id]); + vm->pt_root[id] = NULL; + goto err_destroy_root; + } + } + + if (flags & XE_VM_FLAG_SCRATCH_PAGE) { + for_each_gt(gt, xe, id) { + if (!vm->pt_root[id]) + continue; + + err = xe_pt_create_scratch(xe, gt, vm); + if (err) + goto err_scratch_pt; + } + } + + if (flags & DRM_XE_VM_CREATE_COMPUTE_MODE) { + INIT_WORK(&vm->preempt.rebind_work, preempt_rebind_work_func); + vm->flags |= XE_VM_FLAG_COMPUTE_MODE; + } + + if (flags & DRM_XE_VM_CREATE_ASYNC_BIND_OPS) { + vm->async_ops.fence.context = dma_fence_context_alloc(1); + vm->flags |= XE_VM_FLAG_ASYNC_BIND_OPS; + } + + /* Fill pt_root after allocating scratch tables */ + for_each_gt(gt, xe, id) { + if (!vm->pt_root[id]) + continue; + + xe_pt_populate_empty(gt, vm, vm->pt_root[id]); + } + dma_resv_unlock(&vm->resv); + + /* Kernel migration VM shouldn't have a circular loop.. */ + if (!(flags & XE_VM_FLAG_MIGRATION)) { + for_each_gt(gt, xe, id) { + struct xe_vm *migrate_vm; + struct xe_engine *eng; + + if (!vm->pt_root[id]) + continue; + + migrate_vm = xe_migrate_get_vm(gt->migrate); + eng = xe_engine_create_class(xe, gt, migrate_vm, + XE_ENGINE_CLASS_COPY, + ENGINE_FLAG_VM); + xe_vm_put(migrate_vm); + if (IS_ERR(eng)) { + xe_vm_close_and_put(vm); + return ERR_CAST(eng); + } + vm->eng[id] = eng; + number_gts++; + } + } + + if (number_gts > 1) + vm->composite_fence_ctx = dma_fence_context_alloc(1); + + mutex_lock(&xe->usm.lock); + if (flags & XE_VM_FLAG_FAULT_MODE) + xe->usm.num_vm_in_fault_mode++; + else if (!(flags & XE_VM_FLAG_MIGRATION)) + xe->usm.num_vm_in_non_fault_mode++; + mutex_unlock(&xe->usm.lock); + + trace_xe_vm_create(vm); + + return vm; + +err_scratch_pt: + for_each_gt(gt, xe, id) { + if (!vm->pt_root[id]) + continue; + + i = vm->pt_root[id]->level; + while (i) + if (vm->scratch_pt[id][--i]) + xe_pt_destroy(vm->scratch_pt[id][i], + vm->flags, NULL); + xe_bo_unpin(vm->scratch_bo[id]); + xe_bo_put(vm->scratch_bo[id]); + } +err_destroy_root: + for_each_gt(gt, xe, id) { + if (vm->pt_root[id]) + xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); + } + dma_resv_unlock(&vm->resv); +err_put: + dma_resv_fini(&vm->resv); + kfree(vm); + if (!(flags & XE_VM_FLAG_MIGRATION)) { + xe_device_mem_access_put(xe); + xe_pm_runtime_put(xe); + } + return ERR_PTR(err); +} + +static void flush_async_ops(struct xe_vm *vm) +{ + queue_work(system_unbound_wq, &vm->async_ops.work); + flush_work(&vm->async_ops.work); +} + +static void vm_error_capture(struct xe_vm *vm, int err, + u32 op, u64 addr, u64 size) +{ + struct drm_xe_vm_bind_op_error_capture capture; + u64 __user *address = + u64_to_user_ptr(vm->async_ops.error_capture.addr); + bool in_kthread = !current->mm; + + capture.error = err; + capture.op = op; + capture.addr = addr; + capture.size = size; + + if (in_kthread) { + if (!mmget_not_zero(vm->async_ops.error_capture.mm)) + goto mm_closed; + kthread_use_mm(vm->async_ops.error_capture.mm); + } + + if (copy_to_user(address, &capture, sizeof(capture))) + XE_WARN_ON("Copy to user failed"); + + if (in_kthread) { + kthread_unuse_mm(vm->async_ops.error_capture.mm); + mmput(vm->async_ops.error_capture.mm); + } + +mm_closed: + wake_up_all(&vm->async_ops.error_capture.wq); +} + +void xe_vm_close_and_put(struct xe_vm *vm) +{ + struct rb_root contested = RB_ROOT; + struct ww_acquire_ctx ww; + struct xe_device *xe = vm->xe; + struct xe_gt *gt; + u8 id; + + XE_BUG_ON(vm->preempt.num_engines); + + vm->size = 0; + smp_mb(); + flush_async_ops(vm); + if (xe_vm_in_compute_mode(vm)) + flush_work(&vm->preempt.rebind_work); + + for_each_gt(gt, xe, id) { + if (vm->eng[id]) { + xe_engine_kill(vm->eng[id]); + xe_engine_put(vm->eng[id]); + vm->eng[id] = NULL; + } + } + + down_write(&vm->lock); + xe_vm_lock(vm, &ww, 0, false); + while (vm->vmas.rb_node) { + struct xe_vma *vma = to_xe_vma(vm->vmas.rb_node); + + if (xe_vma_is_userptr(vma)) { + down_read(&vm->userptr.notifier_lock); + vma->destroyed = true; + up_read(&vm->userptr.notifier_lock); + } + + rb_erase(&vma->vm_node, &vm->vmas); + + /* easy case, remove from VMA? */ + if (xe_vma_is_userptr(vma) || vma->bo->vm) { + xe_vma_destroy(vma, NULL); + continue; + } + + rb_add(&vma->vm_node, &contested, xe_vma_less_cb); + } + + /* + * All vm operations will add shared fences to resv. + * The only exception is eviction for a shared object, + * but even so, the unbind when evicted would still + * install a fence to resv. Hence it's safe to + * destroy the pagetables immediately. + */ + for_each_gt(gt, xe, id) { + if (vm->scratch_bo[id]) { + u32 i; + + xe_bo_unpin(vm->scratch_bo[id]); + xe_bo_put(vm->scratch_bo[id]); + for (i = 0; i < vm->pt_root[id]->level; i++) + xe_pt_destroy(vm->scratch_pt[id][i], vm->flags, + NULL); + } + } + xe_vm_unlock(vm, &ww); + + if (contested.rb_node) { + + /* + * VM is now dead, cannot re-add nodes to vm->vmas if it's NULL + * Since we hold a refcount to the bo, we can remove and free + * the members safely without locking. + */ + while (contested.rb_node) { + struct xe_vma *vma = to_xe_vma(contested.rb_node); + + rb_erase(&vma->vm_node, &contested); + xe_vma_destroy_unlocked(vma); + } + } + + if (vm->async_ops.error_capture.addr) + wake_up_all(&vm->async_ops.error_capture.wq); + + XE_WARN_ON(!list_empty(&vm->extobj.list)); + up_write(&vm->lock); + + xe_vm_put(vm); +} + +static void vm_destroy_work_func(struct work_struct *w) +{ + struct xe_vm *vm = + container_of(w, struct xe_vm, destroy_work); + struct ww_acquire_ctx ww; + struct xe_device *xe = vm->xe; + struct xe_gt *gt; + u8 id; + void *lookup; + + /* xe_vm_close_and_put was not called? */ + XE_WARN_ON(vm->size); + + if (!(vm->flags & XE_VM_FLAG_MIGRATION)) { + xe_device_mem_access_put(xe); + xe_pm_runtime_put(xe); + + mutex_lock(&xe->usm.lock); + lookup = xa_erase(&xe->usm.asid_to_vm, vm->usm.asid); + XE_WARN_ON(lookup != vm); + mutex_unlock(&xe->usm.lock); + } + + /* + * XXX: We delay destroying the PT root until the VM if freed as PT root + * is needed for xe_vm_lock to work. If we remove that dependency this + * can be moved to xe_vm_close_and_put. + */ + xe_vm_lock(vm, &ww, 0, false); + for_each_gt(gt, xe, id) { + if (vm->pt_root[id]) { + xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); + vm->pt_root[id] = NULL; + } + } + xe_vm_unlock(vm, &ww); + + mutex_lock(&xe->usm.lock); + if (vm->flags & XE_VM_FLAG_FAULT_MODE) + xe->usm.num_vm_in_fault_mode--; + else if (!(vm->flags & XE_VM_FLAG_MIGRATION)) + xe->usm.num_vm_in_non_fault_mode--; + mutex_unlock(&xe->usm.lock); + + trace_xe_vm_free(vm); + dma_fence_put(vm->rebind_fence); + dma_resv_fini(&vm->resv); + kfree(vm); + +} + +void xe_vm_free(struct kref *ref) +{ + struct xe_vm *vm = container_of(ref, struct xe_vm, refcount); + + /* To destroy the VM we need to be able to sleep */ + queue_work(system_unbound_wq, &vm->destroy_work); +} + +struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id) +{ + struct xe_vm *vm; + + mutex_lock(&xef->vm.lock); + vm = xa_load(&xef->vm.xa, id); + mutex_unlock(&xef->vm.lock); + + if (vm) + xe_vm_get(vm); + + return vm; +} + +u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_gt *full_gt) +{ + XE_BUG_ON(xe_gt_is_media_type(full_gt)); + + return gen8_pde_encode(vm->pt_root[full_gt->info.id]->bo, 0, + XE_CACHE_WB); +} + +static struct dma_fence * +xe_vm_unbind_vma(struct xe_vma *vma, struct xe_engine *e, + struct xe_sync_entry *syncs, u32 num_syncs) +{ + struct xe_gt *gt; + struct dma_fence *fence = NULL; + struct dma_fence **fences = NULL; + struct dma_fence_array *cf = NULL; + struct xe_vm *vm = vma->vm; + int cur_fence = 0, i; + int number_gts = hweight_long(vma->gt_present); + int err; + u8 id; + + trace_xe_vma_unbind(vma); + + if (number_gts > 1) { + fences = kmalloc_array(number_gts, sizeof(*fences), + GFP_KERNEL); + if (!fences) + return ERR_PTR(-ENOMEM); + } + + for_each_gt(gt, vm->xe, id) { + if (!(vma->gt_present & BIT(id))) + goto next; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + + fence = __xe_pt_unbind_vma(gt, vma, e, syncs, num_syncs); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + goto err_fences; + } + + if (fences) + fences[cur_fence++] = fence; + +next: + if (e && vm->pt_root[id] && !list_empty(&e->multi_gt_list)) + e = list_next_entry(e, multi_gt_list); + } + + if (fences) { + cf = dma_fence_array_create(number_gts, fences, + vm->composite_fence_ctx, + vm->composite_fence_seqno++, + false); + if (!cf) { + --vm->composite_fence_seqno; + err = -ENOMEM; + goto err_fences; + } + } + + for (i = 0; i < num_syncs; i++) + xe_sync_entry_signal(&syncs[i], NULL, cf ? &cf->base : fence); + + return cf ? &cf->base : !fence ? dma_fence_get_stub() : fence; + +err_fences: + if (fences) { + while (cur_fence) { + /* FIXME: Rewind the previous binds? */ + dma_fence_put(fences[--cur_fence]); + } + kfree(fences); + } + + return ERR_PTR(err); +} + +static struct dma_fence * +xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e, + struct xe_sync_entry *syncs, u32 num_syncs) +{ + struct xe_gt *gt; + struct dma_fence *fence; + struct dma_fence **fences = NULL; + struct dma_fence_array *cf = NULL; + struct xe_vm *vm = vma->vm; + int cur_fence = 0, i; + int number_gts = hweight_long(vma->gt_mask); + int err; + u8 id; + + trace_xe_vma_bind(vma); + + if (number_gts > 1) { + fences = kmalloc_array(number_gts, sizeof(*fences), + GFP_KERNEL); + if (!fences) + return ERR_PTR(-ENOMEM); + } + + for_each_gt(gt, vm->xe, id) { + if (!(vma->gt_mask & BIT(id))) + goto next; + + XE_BUG_ON(xe_gt_is_media_type(gt)); + fence = __xe_pt_bind_vma(gt, vma, e, syncs, num_syncs, + vma->gt_present & BIT(id)); + if (IS_ERR(fence)) { + err = PTR_ERR(fence); + goto err_fences; + } + + if (fences) + fences[cur_fence++] = fence; + +next: + if (e && vm->pt_root[id] && !list_empty(&e->multi_gt_list)) + e = list_next_entry(e, multi_gt_list); + } + + if (fences) { + cf = dma_fence_array_create(number_gts, fences, + vm->composite_fence_ctx, + vm->composite_fence_seqno++, + false); + if (!cf) { + --vm->composite_fence_seqno; + err = -ENOMEM; + goto err_fences; + } + } + + for (i = 0; i < num_syncs; i++) + xe_sync_entry_signal(&syncs[i], NULL, cf ? &cf->base : fence); + + return cf ? &cf->base : fence; + +err_fences: + if (fences) { + while (cur_fence) { + /* FIXME: Rewind the previous binds? */ + dma_fence_put(fences[--cur_fence]); + } + kfree(fences); + } + + return ERR_PTR(err); +} + +struct async_op_fence { + struct dma_fence fence; + struct dma_fence_cb cb; + struct xe_vm *vm; + wait_queue_head_t wq; + bool started; +}; + +static const char *async_op_fence_get_driver_name(struct dma_fence *dma_fence) +{ + return "xe"; +} + +static const char * +async_op_fence_get_timeline_name(struct dma_fence *dma_fence) +{ + return "async_op_fence"; +} + +static const struct dma_fence_ops async_op_fence_ops = { + .get_driver_name = async_op_fence_get_driver_name, + .get_timeline_name = async_op_fence_get_timeline_name, +}; + +static void async_op_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) +{ + struct async_op_fence *afence = + container_of(cb, struct async_op_fence, cb); + + dma_fence_signal(&afence->fence); + xe_vm_put(afence->vm); + dma_fence_put(&afence->fence); +} + +static void add_async_op_fence_cb(struct xe_vm *vm, + struct dma_fence *fence, + struct async_op_fence *afence) +{ + int ret; + + if (!xe_vm_no_dma_fences(vm)) { + afence->started = true; + smp_wmb(); + wake_up_all(&afence->wq); + } + + afence->vm = xe_vm_get(vm); + dma_fence_get(&afence->fence); + ret = dma_fence_add_callback(fence, &afence->cb, async_op_fence_cb); + if (ret == -ENOENT) + dma_fence_signal(&afence->fence); + if (ret) { + xe_vm_put(vm); + dma_fence_put(&afence->fence); + } + XE_WARN_ON(ret && ret != -ENOENT); +} + +int xe_vm_async_fence_wait_start(struct dma_fence *fence) +{ + if (fence->ops == &async_op_fence_ops) { + struct async_op_fence *afence = + container_of(fence, struct async_op_fence, fence); + + XE_BUG_ON(xe_vm_no_dma_fences(afence->vm)); + + smp_rmb(); + return wait_event_interruptible(afence->wq, afence->started); + } + + return 0; +} + +static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, + struct xe_engine *e, struct xe_sync_entry *syncs, + u32 num_syncs, struct async_op_fence *afence) +{ + struct dma_fence *fence; + + xe_vm_assert_held(vm); + + fence = xe_vm_bind_vma(vma, e, syncs, num_syncs); + if (IS_ERR(fence)) + return PTR_ERR(fence); + if (afence) + add_async_op_fence_cb(vm, fence, afence); + + dma_fence_put(fence); + return 0; +} + +static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_engine *e, + struct xe_bo *bo, struct xe_sync_entry *syncs, + u32 num_syncs, struct async_op_fence *afence) +{ + int err; + + xe_vm_assert_held(vm); + xe_bo_assert_held(bo); + + if (bo) { + err = xe_bo_validate(bo, vm, true); + if (err) + return err; + } + + return __xe_vm_bind(vm, vma, e, syncs, num_syncs, afence); +} + +static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, + struct xe_engine *e, struct xe_sync_entry *syncs, + u32 num_syncs, struct async_op_fence *afence) +{ + struct dma_fence *fence; + + xe_vm_assert_held(vm); + xe_bo_assert_held(vma->bo); + + fence = xe_vm_unbind_vma(vma, e, syncs, num_syncs); + if (IS_ERR(fence)) + return PTR_ERR(fence); + if (afence) + add_async_op_fence_cb(vm, fence, afence); + + xe_vma_destroy(vma, fence); + dma_fence_put(fence); + + return 0; +} + +static int vm_set_error_capture_address(struct xe_device *xe, struct xe_vm *vm, + u64 value) +{ + if (XE_IOCTL_ERR(xe, !value)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, !(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) + return -ENOTSUPP; + + if (XE_IOCTL_ERR(xe, vm->async_ops.error_capture.addr)) + return -ENOTSUPP; + + vm->async_ops.error_capture.mm = current->mm; + vm->async_ops.error_capture.addr = value; + init_waitqueue_head(&vm->async_ops.error_capture.wq); + + return 0; +} + +typedef int (*xe_vm_set_property_fn)(struct xe_device *xe, struct xe_vm *vm, + u64 value); + +static const xe_vm_set_property_fn vm_set_property_funcs[] = { + [XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS] = + vm_set_error_capture_address, +}; + +static int vm_user_ext_set_property(struct xe_device *xe, struct xe_vm *vm, + u64 extension) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_ext_vm_set_property ext; + int err; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_ERR(xe, err)) + return -EFAULT; + + if (XE_IOCTL_ERR(xe, ext.property >= + ARRAY_SIZE(vm_set_property_funcs))) + return -EINVAL; + + return vm_set_property_funcs[ext.property](xe, vm, ext.value); +} + +typedef int (*xe_vm_user_extension_fn)(struct xe_device *xe, struct xe_vm *vm, + u64 extension); + +static const xe_vm_set_property_fn vm_user_extension_funcs[] = { + [XE_VM_EXTENSION_SET_PROPERTY] = vm_user_ext_set_property, +}; + +#define MAX_USER_EXTENSIONS 16 +static int vm_user_extensions(struct xe_device *xe, struct xe_vm *vm, + u64 extensions, int ext_number) +{ + u64 __user *address = u64_to_user_ptr(extensions); + struct xe_user_extension ext; + int err; + + if (XE_IOCTL_ERR(xe, ext_number >= MAX_USER_EXTENSIONS)) + return -E2BIG; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_ERR(xe, err)) + return -EFAULT; + + if (XE_IOCTL_ERR(xe, ext.name >= + ARRAY_SIZE(vm_user_extension_funcs))) + return -EINVAL; + + err = vm_user_extension_funcs[ext.name](xe, vm, extensions); + if (XE_IOCTL_ERR(xe, err)) + return err; + + if (ext.next_extension) + return vm_user_extensions(xe, vm, ext.next_extension, + ++ext_number); + + return 0; +} + +#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_SCRATCH_PAGE | \ + DRM_XE_VM_CREATE_COMPUTE_MODE | \ + DRM_XE_VM_CREATE_ASYNC_BIND_OPS | \ + DRM_XE_VM_CREATE_FAULT_MODE) + +int xe_vm_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_vm_create *args = data; + struct xe_vm *vm; + u32 id, asid; + int err; + u32 flags = 0; + + if (XE_IOCTL_ERR(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->flags & DRM_XE_VM_CREATE_SCRATCH_PAGE && + args->flags & DRM_XE_VM_CREATE_FAULT_MODE)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE && + args->flags & DRM_XE_VM_CREATE_FAULT_MODE)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->flags & DRM_XE_VM_CREATE_FAULT_MODE && + xe_device_in_non_fault_mode(xe))) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, !(args->flags & DRM_XE_VM_CREATE_FAULT_MODE) && + xe_device_in_fault_mode(xe))) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->flags & DRM_XE_VM_CREATE_FAULT_MODE && + !xe->info.supports_usm)) + return -EINVAL; + + if (args->flags & DRM_XE_VM_CREATE_SCRATCH_PAGE) + flags |= XE_VM_FLAG_SCRATCH_PAGE; + if (args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE) + flags |= XE_VM_FLAG_COMPUTE_MODE; + if (args->flags & DRM_XE_VM_CREATE_ASYNC_BIND_OPS) + flags |= XE_VM_FLAG_ASYNC_BIND_OPS; + if (args->flags & DRM_XE_VM_CREATE_FAULT_MODE) + flags |= XE_VM_FLAG_FAULT_MODE; + + vm = xe_vm_create(xe, flags); + if (IS_ERR(vm)) + return PTR_ERR(vm); + + if (args->extensions) { + err = vm_user_extensions(xe, vm, args->extensions, 0); + if (XE_IOCTL_ERR(xe, err)) { + xe_vm_close_and_put(vm); + return err; + } + } + + mutex_lock(&xef->vm.lock); + err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL); + mutex_unlock(&xef->vm.lock); + if (err) { + xe_vm_close_and_put(vm); + return err; + } + + mutex_lock(&xe->usm.lock); + err = xa_alloc_cyclic(&xe->usm.asid_to_vm, &asid, vm, + XA_LIMIT(0, XE_MAX_ASID - 1), + &xe->usm.next_asid, GFP_KERNEL); + mutex_unlock(&xe->usm.lock); + if (err) { + xe_vm_close_and_put(vm); + return err; + } + vm->usm.asid = asid; + + args->vm_id = id; + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_MEM) + /* Warning: Security issue - never enable by default */ + args->reserved[0] = xe_bo_main_addr(vm->pt_root[0]->bo, GEN8_PAGE_SIZE); +#endif + + return 0; +} + +int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_vm_destroy *args = data; + struct xe_vm *vm; + + if (XE_IOCTL_ERR(xe, args->pad)) + return -EINVAL; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_ERR(xe, !vm)) + return -ENOENT; + xe_vm_put(vm); + + /* FIXME: Extend this check to non-compute mode VMs */ + if (XE_IOCTL_ERR(xe, vm->preempt.num_engines)) + return -EBUSY; + + mutex_lock(&xef->vm.lock); + xa_erase(&xef->vm.xa, args->vm_id); + mutex_unlock(&xef->vm.lock); + + xe_vm_close_and_put(vm); + + return 0; +} + +static const u32 region_to_mem_type[] = { + XE_PL_TT, + XE_PL_VRAM0, + XE_PL_VRAM1, +}; + +static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, + struct xe_engine *e, u32 region, + struct xe_sync_entry *syncs, u32 num_syncs, + struct async_op_fence *afence) +{ + int err; + + XE_BUG_ON(region > ARRAY_SIZE(region_to_mem_type)); + + if (!xe_vma_is_userptr(vma)) { + err = xe_bo_migrate(vma->bo, region_to_mem_type[region]); + if (err) + return err; + } + + if (vma->gt_mask != (vma->gt_present & ~vma->usm.gt_invalidated)) { + return xe_vm_bind(vm, vma, e, vma->bo, syncs, num_syncs, + afence); + } else { + int i; + + /* Nothing to do, signal fences now */ + for (i = 0; i < num_syncs; i++) + xe_sync_entry_signal(&syncs[i], NULL, + dma_fence_get_stub()); + if (afence) + dma_fence_signal(&afence->fence); + return 0; + } +} + +#define VM_BIND_OP(op) (op & 0xffff) + +static int __vm_bind_ioctl(struct xe_vm *vm, struct xe_vma *vma, + struct xe_engine *e, struct xe_bo *bo, u32 op, + u32 region, struct xe_sync_entry *syncs, + u32 num_syncs, struct async_op_fence *afence) +{ + switch (VM_BIND_OP(op)) { + case XE_VM_BIND_OP_MAP: + return xe_vm_bind(vm, vma, e, bo, syncs, num_syncs, afence); + case XE_VM_BIND_OP_UNMAP: + case XE_VM_BIND_OP_UNMAP_ALL: + return xe_vm_unbind(vm, vma, e, syncs, num_syncs, afence); + case XE_VM_BIND_OP_MAP_USERPTR: + return xe_vm_bind(vm, vma, e, NULL, syncs, num_syncs, afence); + case XE_VM_BIND_OP_PREFETCH: + return xe_vm_prefetch(vm, vma, e, region, syncs, num_syncs, + afence); + break; + default: + XE_BUG_ON("NOT POSSIBLE"); + return -EINVAL; + } +} + +struct ttm_buffer_object *xe_vm_ttm_bo(struct xe_vm *vm) +{ + int idx = vm->flags & XE_VM_FLAG_MIGRATION ? + XE_VM_FLAG_GT_ID(vm->flags) : 0; + + /* Safe to use index 0 as all BO in the VM share a single dma-resv lock */ + return &vm->pt_root[idx]->bo->ttm; +} + +static void xe_vm_tv_populate(struct xe_vm *vm, struct ttm_validate_buffer *tv) +{ + tv->num_shared = 1; + tv->bo = xe_vm_ttm_bo(vm); +} + +static bool is_map_op(u32 op) +{ + return VM_BIND_OP(op) == XE_VM_BIND_OP_MAP || + VM_BIND_OP(op) == XE_VM_BIND_OP_MAP_USERPTR; +} + +static bool is_unmap_op(u32 op) +{ + return VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP || + VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL; +} + +static int vm_bind_ioctl(struct xe_vm *vm, struct xe_vma *vma, + struct xe_engine *e, struct xe_bo *bo, + struct drm_xe_vm_bind_op *bind_op, + struct xe_sync_entry *syncs, u32 num_syncs, + struct async_op_fence *afence) +{ + LIST_HEAD(objs); + LIST_HEAD(dups); + struct ttm_validate_buffer tv_bo, tv_vm; + struct ww_acquire_ctx ww; + struct xe_bo *vbo; + int err, i; + + lockdep_assert_held(&vm->lock); + XE_BUG_ON(!list_empty(&vma->unbind_link)); + + /* Binds deferred to faults, signal fences now */ + if (xe_vm_in_fault_mode(vm) && is_map_op(bind_op->op) && + !(bind_op->op & XE_VM_BIND_FLAG_IMMEDIATE)) { + for (i = 0; i < num_syncs; i++) + xe_sync_entry_signal(&syncs[i], NULL, + dma_fence_get_stub()); + if (afence) + dma_fence_signal(&afence->fence); + return 0; + } + + xe_vm_tv_populate(vm, &tv_vm); + list_add_tail(&tv_vm.head, &objs); + vbo = vma->bo; + if (vbo) { + /* + * An unbind can drop the last reference to the BO and + * the BO is needed for ttm_eu_backoff_reservation so + * take a reference here. + */ + xe_bo_get(vbo); + + tv_bo.bo = &vbo->ttm; + tv_bo.num_shared = 1; + list_add(&tv_bo.head, &objs); + } + +again: + err = ttm_eu_reserve_buffers(&ww, &objs, true, &dups); + if (!err) { + err = __vm_bind_ioctl(vm, vma, e, bo, + bind_op->op, bind_op->region, syncs, + num_syncs, afence); + ttm_eu_backoff_reservation(&ww, &objs); + if (err == -EAGAIN && xe_vma_is_userptr(vma)) { + lockdep_assert_held_write(&vm->lock); + err = xe_vma_userptr_pin_pages(vma); + if (!err) + goto again; + } + } + xe_bo_put(vbo); + + return err; +} + +struct async_op { + struct xe_vma *vma; + struct xe_engine *engine; + struct xe_bo *bo; + struct drm_xe_vm_bind_op bind_op; + struct xe_sync_entry *syncs; + u32 num_syncs; + struct list_head link; + struct async_op_fence *fence; +}; + +static void async_op_cleanup(struct xe_vm *vm, struct async_op *op) +{ + while (op->num_syncs--) + xe_sync_entry_cleanup(&op->syncs[op->num_syncs]); + kfree(op->syncs); + xe_bo_put(op->bo); + if (op->engine) + xe_engine_put(op->engine); + xe_vm_put(vm); + if (op->fence) + dma_fence_put(&op->fence->fence); + kfree(op); +} + +static struct async_op *next_async_op(struct xe_vm *vm) +{ + return list_first_entry_or_null(&vm->async_ops.pending, + struct async_op, link); +} + +static void vm_set_async_error(struct xe_vm *vm, int err) +{ + lockdep_assert_held(&vm->lock); + vm->async_ops.error = err; +} + +static void async_op_work_func(struct work_struct *w) +{ + struct xe_vm *vm = container_of(w, struct xe_vm, async_ops.work); + + for (;;) { + struct async_op *op; + int err; + + if (vm->async_ops.error && !xe_vm_is_closed(vm)) + break; + + spin_lock_irq(&vm->async_ops.lock); + op = next_async_op(vm); + if (op) + list_del_init(&op->link); + spin_unlock_irq(&vm->async_ops.lock); + + if (!op) + break; + + if (!xe_vm_is_closed(vm)) { + bool first, last; + + down_write(&vm->lock); +again: + first = op->vma->first_munmap_rebind; + last = op->vma->last_munmap_rebind; +#ifdef TEST_VM_ASYNC_OPS_ERROR +#define FORCE_ASYNC_OP_ERROR BIT(31) + if (!(op->bind_op.op & FORCE_ASYNC_OP_ERROR)) { + err = vm_bind_ioctl(vm, op->vma, op->engine, + op->bo, &op->bind_op, + op->syncs, op->num_syncs, + op->fence); + } else { + err = -ENOMEM; + op->bind_op.op &= ~FORCE_ASYNC_OP_ERROR; + } +#else + err = vm_bind_ioctl(vm, op->vma, op->engine, op->bo, + &op->bind_op, op->syncs, + op->num_syncs, op->fence); +#endif + /* + * In order for the fencing to work (stall behind + * existing jobs / prevent new jobs from running) all + * the dma-resv slots need to be programmed in a batch + * relative to execs / the rebind worker. The vm->lock + * ensure this. + */ + if (!err && ((first && VM_BIND_OP(op->bind_op.op) == + XE_VM_BIND_OP_UNMAP) || + vm->async_ops.munmap_rebind_inflight)) { + if (last) { + op->vma->last_munmap_rebind = false; + vm->async_ops.munmap_rebind_inflight = + false; + } else { + vm->async_ops.munmap_rebind_inflight = + true; + + async_op_cleanup(vm, op); + + spin_lock_irq(&vm->async_ops.lock); + op = next_async_op(vm); + XE_BUG_ON(!op); + list_del_init(&op->link); + spin_unlock_irq(&vm->async_ops.lock); + + goto again; + } + } + if (err) { + trace_xe_vma_fail(op->vma); + drm_warn(&vm->xe->drm, "Async VM op(%d) failed with %d", + VM_BIND_OP(op->bind_op.op), + err); + + spin_lock_irq(&vm->async_ops.lock); + list_add(&op->link, &vm->async_ops.pending); + spin_unlock_irq(&vm->async_ops.lock); + + vm_set_async_error(vm, err); + up_write(&vm->lock); + + if (vm->async_ops.error_capture.addr) + vm_error_capture(vm, err, + op->bind_op.op, + op->bind_op.addr, + op->bind_op.range); + break; + } + up_write(&vm->lock); + } else { + trace_xe_vma_flush(op->vma); + + if (is_unmap_op(op->bind_op.op)) { + down_write(&vm->lock); + xe_vma_destroy_unlocked(op->vma); + up_write(&vm->lock); + } + + if (op->fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, + &op->fence->fence.flags)) { + if (!xe_vm_no_dma_fences(vm)) { + op->fence->started = true; + smp_wmb(); + wake_up_all(&op->fence->wq); + } + dma_fence_signal(&op->fence->fence); + } + } + + async_op_cleanup(vm, op); + } +} + +static int __vm_bind_ioctl_async(struct xe_vm *vm, struct xe_vma *vma, + struct xe_engine *e, struct xe_bo *bo, + struct drm_xe_vm_bind_op *bind_op, + struct xe_sync_entry *syncs, u32 num_syncs) +{ + struct async_op *op; + bool installed = false; + u64 seqno; + int i; + + lockdep_assert_held(&vm->lock); + + op = kmalloc(sizeof(*op), GFP_KERNEL); + if (!op) { + return -ENOMEM; + } + + if (num_syncs) { + op->fence = kmalloc(sizeof(*op->fence), GFP_KERNEL); + if (!op->fence) { + kfree(op); + return -ENOMEM; + } + + seqno = e ? ++e->bind.fence_seqno : ++vm->async_ops.fence.seqno; + dma_fence_init(&op->fence->fence, &async_op_fence_ops, + &vm->async_ops.lock, e ? e->bind.fence_ctx : + vm->async_ops.fence.context, seqno); + + if (!xe_vm_no_dma_fences(vm)) { + op->fence->vm = vm; + op->fence->started = false; + init_waitqueue_head(&op->fence->wq); + } + } else { + op->fence = NULL; + } + op->vma = vma; + op->engine = e; + op->bo = bo; + op->bind_op = *bind_op; + op->syncs = syncs; + op->num_syncs = num_syncs; + INIT_LIST_HEAD(&op->link); + + for (i = 0; i < num_syncs; i++) + installed |= xe_sync_entry_signal(&syncs[i], NULL, + &op->fence->fence); + + if (!installed && op->fence) + dma_fence_signal(&op->fence->fence); + + spin_lock_irq(&vm->async_ops.lock); + list_add_tail(&op->link, &vm->async_ops.pending); + spin_unlock_irq(&vm->async_ops.lock); + + if (!vm->async_ops.error) + queue_work(system_unbound_wq, &vm->async_ops.work); + + return 0; +} + +static int vm_bind_ioctl_async(struct xe_vm *vm, struct xe_vma *vma, + struct xe_engine *e, struct xe_bo *bo, + struct drm_xe_vm_bind_op *bind_op, + struct xe_sync_entry *syncs, u32 num_syncs) +{ + struct xe_vma *__vma, *next; + struct list_head rebind_list; + struct xe_sync_entry *in_syncs = NULL, *out_syncs = NULL; + u32 num_in_syncs = 0, num_out_syncs = 0; + bool first = true, last; + int err; + int i; + + lockdep_assert_held(&vm->lock); + + /* Not a linked list of unbinds + rebinds, easy */ + if (list_empty(&vma->unbind_link)) + return __vm_bind_ioctl_async(vm, vma, e, bo, bind_op, + syncs, num_syncs); + + /* + * Linked list of unbinds + rebinds, decompose syncs into 'in / out' + * passing the 'in' to the first operation and 'out' to the last. Also + * the reference counting is a little tricky, increment the VM / bind + * engine ref count on all but the last operation and increment the BOs + * ref count on each rebind. + */ + + XE_BUG_ON(VM_BIND_OP(bind_op->op) != XE_VM_BIND_OP_UNMAP && + VM_BIND_OP(bind_op->op) != XE_VM_BIND_OP_UNMAP_ALL && + VM_BIND_OP(bind_op->op) != XE_VM_BIND_OP_PREFETCH); + + /* Decompose syncs */ + if (num_syncs) { + in_syncs = kmalloc(sizeof(*in_syncs) * num_syncs, GFP_KERNEL); + out_syncs = kmalloc(sizeof(*out_syncs) * num_syncs, GFP_KERNEL); + if (!in_syncs || !out_syncs) { + err = -ENOMEM; + goto out_error; + } + + for (i = 0; i < num_syncs; ++i) { + bool signal = syncs[i].flags & DRM_XE_SYNC_SIGNAL; + + if (signal) + out_syncs[num_out_syncs++] = syncs[i]; + else + in_syncs[num_in_syncs++] = syncs[i]; + } + } + + /* Do unbinds + move rebinds to new list */ + INIT_LIST_HEAD(&rebind_list); + list_for_each_entry_safe(__vma, next, &vma->unbind_link, unbind_link) { + if (__vma->destroyed || + VM_BIND_OP(bind_op->op) == XE_VM_BIND_OP_PREFETCH) { + list_del_init(&__vma->unbind_link); + xe_bo_get(bo); + err = __vm_bind_ioctl_async(xe_vm_get(vm), __vma, + e ? xe_engine_get(e) : NULL, + bo, bind_op, first ? + in_syncs : NULL, + first ? num_in_syncs : 0); + if (err) { + xe_bo_put(bo); + xe_vm_put(vm); + if (e) + xe_engine_put(e); + goto out_error; + } + in_syncs = NULL; + first = false; + } else { + list_move_tail(&__vma->unbind_link, &rebind_list); + } + } + last = list_empty(&rebind_list); + if (!last) { + xe_vm_get(vm); + if (e) + xe_engine_get(e); + } + err = __vm_bind_ioctl_async(vm, vma, e, + bo, bind_op, + first ? in_syncs : + last ? out_syncs : NULL, + first ? num_in_syncs : + last ? num_out_syncs : 0); + if (err) { + if (!last) { + xe_vm_put(vm); + if (e) + xe_engine_put(e); + } + goto out_error; + } + in_syncs = NULL; + + /* Do rebinds */ + list_for_each_entry_safe(__vma, next, &rebind_list, unbind_link) { + list_del_init(&__vma->unbind_link); + last = list_empty(&rebind_list); + + if (xe_vma_is_userptr(__vma)) { + bind_op->op = XE_VM_BIND_FLAG_ASYNC | + XE_VM_BIND_OP_MAP_USERPTR; + } else { + bind_op->op = XE_VM_BIND_FLAG_ASYNC | + XE_VM_BIND_OP_MAP; + xe_bo_get(__vma->bo); + } + + if (!last) { + xe_vm_get(vm); + if (e) + xe_engine_get(e); + } + + err = __vm_bind_ioctl_async(vm, __vma, e, + __vma->bo, bind_op, last ? + out_syncs : NULL, + last ? num_out_syncs : 0); + if (err) { + if (!last) { + xe_vm_put(vm); + if (e) + xe_engine_put(e); + } + goto out_error; + } + } + + kfree(syncs); + return 0; + +out_error: + kfree(in_syncs); + kfree(out_syncs); + kfree(syncs); + + return err; +} + +static int __vm_bind_ioctl_lookup_vma(struct xe_vm *vm, struct xe_bo *bo, + u64 addr, u64 range, u32 op) +{ + struct xe_device *xe = vm->xe; + struct xe_vma *vma, lookup; + bool async = !!(op & XE_VM_BIND_FLAG_ASYNC); + + lockdep_assert_held(&vm->lock); + + lookup.start = addr; + lookup.end = addr + range - 1; + + switch (VM_BIND_OP(op)) { + case XE_VM_BIND_OP_MAP: + case XE_VM_BIND_OP_MAP_USERPTR: + vma = xe_vm_find_overlapping_vma(vm, &lookup); + if (XE_IOCTL_ERR(xe, vma)) + return -EBUSY; + break; + case XE_VM_BIND_OP_UNMAP: + case XE_VM_BIND_OP_PREFETCH: + vma = xe_vm_find_overlapping_vma(vm, &lookup); + if (XE_IOCTL_ERR(xe, !vma) || + XE_IOCTL_ERR(xe, (vma->start != addr || + vma->end != addr + range - 1) && !async)) + return -EINVAL; + break; + case XE_VM_BIND_OP_UNMAP_ALL: + break; + default: + XE_BUG_ON("NOT POSSIBLE"); + return -EINVAL; + } + + return 0; +} + +static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma) +{ + down_read(&vm->userptr.notifier_lock); + vma->destroyed = true; + up_read(&vm->userptr.notifier_lock); + xe_vm_remove_vma(vm, vma); +} + +static int prep_replacement_vma(struct xe_vm *vm, struct xe_vma *vma) +{ + int err; + + if (vma->bo && !vma->bo->vm) { + vm_insert_extobj(vm, vma); + err = add_preempt_fences(vm, vma->bo); + if (err) + return err; + } + + return 0; +} + +/* + * Find all overlapping VMAs in lookup range and add to a list in the returned + * VMA, all of VMAs found will be unbound. Also possibly add 2 new VMAs that + * need to be bound if first / last VMAs are not fully unbound. This is akin to + * how munmap works. + */ +static struct xe_vma *vm_unbind_lookup_vmas(struct xe_vm *vm, + struct xe_vma *lookup) +{ + struct xe_vma *vma = xe_vm_find_overlapping_vma(vm, lookup); + struct rb_node *node; + struct xe_vma *first = vma, *last = vma, *new_first = NULL, + *new_last = NULL, *__vma, *next; + int err = 0; + bool first_munmap_rebind = false; + + lockdep_assert_held(&vm->lock); + XE_BUG_ON(!vma); + + node = &vma->vm_node; + while ((node = rb_next(node))) { + if (!xe_vma_cmp_vma_cb(lookup, node)) { + __vma = to_xe_vma(node); + list_add_tail(&__vma->unbind_link, &vma->unbind_link); + last = __vma; + } else { + break; + } + } + + node = &vma->vm_node; + while ((node = rb_prev(node))) { + if (!xe_vma_cmp_vma_cb(lookup, node)) { + __vma = to_xe_vma(node); + list_add(&__vma->unbind_link, &vma->unbind_link); + first = __vma; + } else { + break; + } + } + + if (first->start != lookup->start) { + struct ww_acquire_ctx ww; + + if (first->bo) + err = xe_bo_lock(first->bo, &ww, 0, true); + if (err) + goto unwind; + new_first = xe_vma_create(first->vm, first->bo, + first->bo ? first->bo_offset : + first->userptr.ptr, + first->start, + lookup->start - 1, + (first->pte_flags & PTE_READ_ONLY), + first->gt_mask); + if (first->bo) + xe_bo_unlock(first->bo, &ww); + if (!new_first) { + err = -ENOMEM; + goto unwind; + } + if (!first->bo) { + err = xe_vma_userptr_pin_pages(new_first); + if (err) + goto unwind; + } + err = prep_replacement_vma(vm, new_first); + if (err) + goto unwind; + } + + if (last->end != lookup->end) { + struct ww_acquire_ctx ww; + u64 chunk = lookup->end + 1 - last->start; + + if (last->bo) + err = xe_bo_lock(last->bo, &ww, 0, true); + if (err) + goto unwind; + new_last = xe_vma_create(last->vm, last->bo, + last->bo ? last->bo_offset + chunk : + last->userptr.ptr + chunk, + last->start + chunk, + last->end, + (last->pte_flags & PTE_READ_ONLY), + last->gt_mask); + if (last->bo) + xe_bo_unlock(last->bo, &ww); + if (!new_last) { + err = -ENOMEM; + goto unwind; + } + if (!last->bo) { + err = xe_vma_userptr_pin_pages(new_last); + if (err) + goto unwind; + } + err = prep_replacement_vma(vm, new_last); + if (err) + goto unwind; + } + + prep_vma_destroy(vm, vma); + if (list_empty(&vma->unbind_link) && (new_first || new_last)) + vma->first_munmap_rebind = true; + list_for_each_entry(__vma, &vma->unbind_link, unbind_link) { + if ((new_first || new_last) && !first_munmap_rebind) { + __vma->first_munmap_rebind = true; + first_munmap_rebind = true; + } + prep_vma_destroy(vm, __vma); + } + if (new_first) { + xe_vm_insert_vma(vm, new_first); + list_add_tail(&new_first->unbind_link, &vma->unbind_link); + if (!new_last) + new_first->last_munmap_rebind = true; + } + if (new_last) { + xe_vm_insert_vma(vm, new_last); + list_add_tail(&new_last->unbind_link, &vma->unbind_link); + new_last->last_munmap_rebind = true; + } + + return vma; + +unwind: + list_for_each_entry_safe(__vma, next, &vma->unbind_link, unbind_link) + list_del_init(&__vma->unbind_link); + if (new_last) { + prep_vma_destroy(vm, new_last); + xe_vma_destroy_unlocked(new_last); + } + if (new_first) { + prep_vma_destroy(vm, new_first); + xe_vma_destroy_unlocked(new_first); + } + + return ERR_PTR(err); +} + +/* + * Similar to vm_unbind_lookup_vmas, find all VMAs in lookup range to prefetch + */ +static struct xe_vma *vm_prefetch_lookup_vmas(struct xe_vm *vm, + struct xe_vma *lookup, + u32 region) +{ + struct xe_vma *vma = xe_vm_find_overlapping_vma(vm, lookup), *__vma, + *next; + struct rb_node *node; + + if (!xe_vma_is_userptr(vma)) { + if (!xe_bo_can_migrate(vma->bo, region_to_mem_type[region])) + return ERR_PTR(-EINVAL); + } + + node = &vma->vm_node; + while ((node = rb_next(node))) { + if (!xe_vma_cmp_vma_cb(lookup, node)) { + __vma = to_xe_vma(node); + if (!xe_vma_is_userptr(__vma)) { + if (!xe_bo_can_migrate(__vma->bo, region_to_mem_type[region])) + goto flush_list; + } + list_add_tail(&__vma->unbind_link, &vma->unbind_link); + } else { + break; + } + } + + node = &vma->vm_node; + while ((node = rb_prev(node))) { + if (!xe_vma_cmp_vma_cb(lookup, node)) { + __vma = to_xe_vma(node); + if (!xe_vma_is_userptr(__vma)) { + if (!xe_bo_can_migrate(__vma->bo, region_to_mem_type[region])) + goto flush_list; + } + list_add(&__vma->unbind_link, &vma->unbind_link); + } else { + break; + } + } + + return vma; + +flush_list: + list_for_each_entry_safe(__vma, next, &vma->unbind_link, + unbind_link) + list_del_init(&__vma->unbind_link); + + return ERR_PTR(-EINVAL); +} + +static struct xe_vma *vm_unbind_all_lookup_vmas(struct xe_vm *vm, + struct xe_bo *bo) +{ + struct xe_vma *first = NULL, *vma; + + lockdep_assert_held(&vm->lock); + xe_bo_assert_held(bo); + + list_for_each_entry(vma, &bo->vmas, bo_link) { + if (vma->vm != vm) + continue; + + prep_vma_destroy(vm, vma); + if (!first) + first = vma; + else + list_add_tail(&vma->unbind_link, &first->unbind_link); + } + + return first; +} + +static struct xe_vma *vm_bind_ioctl_lookup_vma(struct xe_vm *vm, + struct xe_bo *bo, + u64 bo_offset_or_userptr, + u64 addr, u64 range, u32 op, + u64 gt_mask, u32 region) +{ + struct ww_acquire_ctx ww; + struct xe_vma *vma, lookup; + int err; + + lockdep_assert_held(&vm->lock); + + lookup.start = addr; + lookup.end = addr + range - 1; + + switch (VM_BIND_OP(op)) { + case XE_VM_BIND_OP_MAP: + XE_BUG_ON(!bo); + + err = xe_bo_lock(bo, &ww, 0, true); + if (err) + return ERR_PTR(err); + vma = xe_vma_create(vm, bo, bo_offset_or_userptr, addr, + addr + range - 1, + op & XE_VM_BIND_FLAG_READONLY, + gt_mask); + xe_bo_unlock(bo, &ww); + if (!vma) + return ERR_PTR(-ENOMEM); + + xe_vm_insert_vma(vm, vma); + if (!bo->vm) { + vm_insert_extobj(vm, vma); + err = add_preempt_fences(vm, bo); + if (err) { + prep_vma_destroy(vm, vma); + xe_vma_destroy_unlocked(vma); + + return ERR_PTR(err); + } + } + break; + case XE_VM_BIND_OP_UNMAP: + vma = vm_unbind_lookup_vmas(vm, &lookup); + break; + case XE_VM_BIND_OP_PREFETCH: + vma = vm_prefetch_lookup_vmas(vm, &lookup, region); + break; + case XE_VM_BIND_OP_UNMAP_ALL: + XE_BUG_ON(!bo); + + err = xe_bo_lock(bo, &ww, 0, true); + if (err) + return ERR_PTR(err); + vma = vm_unbind_all_lookup_vmas(vm, bo); + if (!vma) + vma = ERR_PTR(-EINVAL); + xe_bo_unlock(bo, &ww); + break; + case XE_VM_BIND_OP_MAP_USERPTR: + XE_BUG_ON(bo); + + vma = xe_vma_create(vm, NULL, bo_offset_or_userptr, addr, + addr + range - 1, + op & XE_VM_BIND_FLAG_READONLY, + gt_mask); + if (!vma) + return ERR_PTR(-ENOMEM); + + err = xe_vma_userptr_pin_pages(vma); + if (err) { + xe_vma_destroy(vma, NULL); + + return ERR_PTR(err); + } else { + xe_vm_insert_vma(vm, vma); + } + break; + default: + XE_BUG_ON("NOT POSSIBLE"); + vma = ERR_PTR(-EINVAL); + } + + return vma; +} + +#ifdef TEST_VM_ASYNC_OPS_ERROR +#define SUPPORTED_FLAGS \ + (FORCE_ASYNC_OP_ERROR | XE_VM_BIND_FLAG_ASYNC | \ + XE_VM_BIND_FLAG_READONLY | XE_VM_BIND_FLAG_IMMEDIATE | 0xffff) +#else +#define SUPPORTED_FLAGS \ + (XE_VM_BIND_FLAG_ASYNC | XE_VM_BIND_FLAG_READONLY | \ + XE_VM_BIND_FLAG_IMMEDIATE | 0xffff) +#endif +#define XE_64K_PAGE_MASK 0xffffull + +#define MAX_BINDS 512 /* FIXME: Picking random upper limit */ + +static int vm_bind_ioctl_check_args(struct xe_device *xe, + struct drm_xe_vm_bind *args, + struct drm_xe_vm_bind_op **bind_ops, + bool *async) +{ + int err; + int i; + + if (XE_IOCTL_ERR(xe, args->extensions) || + XE_IOCTL_ERR(xe, !args->num_binds) || + XE_IOCTL_ERR(xe, args->num_binds > MAX_BINDS)) + return -EINVAL; + + if (args->num_binds > 1) { + u64 __user *bind_user = + u64_to_user_ptr(args->vector_of_binds); + + *bind_ops = kmalloc(sizeof(struct drm_xe_vm_bind_op) * + args->num_binds, GFP_KERNEL); + if (!*bind_ops) + return -ENOMEM; + + err = __copy_from_user(*bind_ops, bind_user, + sizeof(struct drm_xe_vm_bind_op) * + args->num_binds); + if (XE_IOCTL_ERR(xe, err)) { + err = -EFAULT; + goto free_bind_ops; + } + } else { + *bind_ops = &args->bind; + } + + for (i = 0; i < args->num_binds; ++i) { + u64 range = (*bind_ops)[i].range; + u64 addr = (*bind_ops)[i].addr; + u32 op = (*bind_ops)[i].op; + u32 obj = (*bind_ops)[i].obj; + u64 obj_offset = (*bind_ops)[i].obj_offset; + u32 region = (*bind_ops)[i].region; + + if (i == 0) { + *async = !!(op & XE_VM_BIND_FLAG_ASYNC); + } else if (XE_IOCTL_ERR(xe, !*async) || + XE_IOCTL_ERR(xe, !(op & XE_VM_BIND_FLAG_ASYNC)) || + XE_IOCTL_ERR(xe, VM_BIND_OP(op) == + XE_VM_BIND_OP_RESTART)) { + err = -EINVAL; + goto free_bind_ops; + } + + if (XE_IOCTL_ERR(xe, !*async && + VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL)) { + err = -EINVAL; + goto free_bind_ops; + } + + if (XE_IOCTL_ERR(xe, !*async && + VM_BIND_OP(op) == XE_VM_BIND_OP_PREFETCH)) { + err = -EINVAL; + goto free_bind_ops; + } + + if (XE_IOCTL_ERR(xe, VM_BIND_OP(op) > + XE_VM_BIND_OP_PREFETCH) || + XE_IOCTL_ERR(xe, op & ~SUPPORTED_FLAGS) || + XE_IOCTL_ERR(xe, !obj && + VM_BIND_OP(op) == XE_VM_BIND_OP_MAP) || + XE_IOCTL_ERR(xe, !obj && + VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) || + XE_IOCTL_ERR(xe, addr && + VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) || + XE_IOCTL_ERR(xe, range && + VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) || + XE_IOCTL_ERR(xe, obj && + VM_BIND_OP(op) == XE_VM_BIND_OP_MAP_USERPTR) || + XE_IOCTL_ERR(xe, obj && + VM_BIND_OP(op) == XE_VM_BIND_OP_PREFETCH) || + XE_IOCTL_ERR(xe, region && + VM_BIND_OP(op) != XE_VM_BIND_OP_PREFETCH) || + XE_IOCTL_ERR(xe, !(BIT(region) & + xe->info.mem_region_mask)) || + XE_IOCTL_ERR(xe, obj && + VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP)) { + err = -EINVAL; + goto free_bind_ops; + } + + if (XE_IOCTL_ERR(xe, obj_offset & ~PAGE_MASK) || + XE_IOCTL_ERR(xe, addr & ~PAGE_MASK) || + XE_IOCTL_ERR(xe, range & ~PAGE_MASK) || + XE_IOCTL_ERR(xe, !range && VM_BIND_OP(op) != + XE_VM_BIND_OP_RESTART && + VM_BIND_OP(op) != XE_VM_BIND_OP_UNMAP_ALL)) { + err = -EINVAL; + goto free_bind_ops; + } + } + + return 0; + +free_bind_ops: + if (args->num_binds > 1) + kfree(*bind_ops); + return err; +} + +int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_vm_bind *args = data; + struct drm_xe_sync __user *syncs_user; + struct xe_bo **bos = NULL; + struct xe_vma **vmas = NULL; + struct xe_vm *vm; + struct xe_engine *e = NULL; + u32 num_syncs; + struct xe_sync_entry *syncs = NULL; + struct drm_xe_vm_bind_op *bind_ops; + bool async; + int err; + int i, j = 0; + + err = vm_bind_ioctl_check_args(xe, args, &bind_ops, &async); + if (err) + return err; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_ERR(xe, !vm)) { + err = -EINVAL; + goto free_objs; + } + + if (XE_IOCTL_ERR(xe, xe_vm_is_closed(vm))) { + DRM_ERROR("VM closed while we began looking up?\n"); + err = -ENOENT; + goto put_vm; + } + + if (args->engine_id) { + e = xe_engine_lookup(xef, args->engine_id); + if (XE_IOCTL_ERR(xe, !e)) { + err = -ENOENT; + goto put_vm; + } + if (XE_IOCTL_ERR(xe, !(e->flags & ENGINE_FLAG_VM))) { + err = -EINVAL; + goto put_engine; + } + } + + if (VM_BIND_OP(bind_ops[0].op) == XE_VM_BIND_OP_RESTART) { + if (XE_IOCTL_ERR(xe, !(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) + err = -ENOTSUPP; + if (XE_IOCTL_ERR(xe, !err && args->num_syncs)) + err = EINVAL; + if (XE_IOCTL_ERR(xe, !err && !vm->async_ops.error)) + err = -EPROTO; + + if (!err) { + down_write(&vm->lock); + trace_xe_vm_restart(vm); + vm_set_async_error(vm, 0); + up_write(&vm->lock); + + queue_work(system_unbound_wq, &vm->async_ops.work); + + /* Rebinds may have been blocked, give worker a kick */ + if (xe_vm_in_compute_mode(vm)) + queue_work(vm->xe->ordered_wq, + &vm->preempt.rebind_work); + } + + goto put_engine; + } + + if (XE_IOCTL_ERR(xe, !vm->async_ops.error && + async != !!(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) { + err = -ENOTSUPP; + goto put_engine; + } + + for (i = 0; i < args->num_binds; ++i) { + u64 range = bind_ops[i].range; + u64 addr = bind_ops[i].addr; + + if (XE_IOCTL_ERR(xe, range > vm->size) || + XE_IOCTL_ERR(xe, addr > vm->size - range)) { + err = -EINVAL; + goto put_engine; + } + + if (bind_ops[i].gt_mask) { + u64 valid_gts = BIT(xe->info.tile_count) - 1; + + if (XE_IOCTL_ERR(xe, bind_ops[i].gt_mask & + ~valid_gts)) { + err = -EINVAL; + goto put_engine; + } + } + } + + bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL); + if (!bos) { + err = -ENOMEM; + goto put_engine; + } + + vmas = kzalloc(sizeof(*vmas) * args->num_binds, GFP_KERNEL); + if (!vmas) { + err = -ENOMEM; + goto put_engine; + } + + for (i = 0; i < args->num_binds; ++i) { + struct drm_gem_object *gem_obj; + u64 range = bind_ops[i].range; + u64 addr = bind_ops[i].addr; + u32 obj = bind_ops[i].obj; + u64 obj_offset = bind_ops[i].obj_offset; + + if (!obj) + continue; + + gem_obj = drm_gem_object_lookup(file, obj); + if (XE_IOCTL_ERR(xe, !gem_obj)) { + err = -ENOENT; + goto put_obj; + } + bos[i] = gem_to_xe_bo(gem_obj); + + if (XE_IOCTL_ERR(xe, range > bos[i]->size) || + XE_IOCTL_ERR(xe, obj_offset > + bos[i]->size - range)) { + err = -EINVAL; + goto put_obj; + } + + if (bos[i]->flags & XE_BO_INTERNAL_64K) { + if (XE_IOCTL_ERR(xe, obj_offset & + XE_64K_PAGE_MASK) || + XE_IOCTL_ERR(xe, addr & XE_64K_PAGE_MASK) || + XE_IOCTL_ERR(xe, range & XE_64K_PAGE_MASK)) { + err = -EINVAL; + goto put_obj; + } + } + } + + if (args->num_syncs) { + syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); + if (!syncs) { + err = -ENOMEM; + goto put_obj; + } + } + + syncs_user = u64_to_user_ptr(args->syncs); + for (num_syncs = 0; num_syncs < args->num_syncs; num_syncs++) { + err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs], + &syncs_user[num_syncs], false, + xe_vm_no_dma_fences(vm)); + if (err) + goto free_syncs; + } + + err = down_write_killable(&vm->lock); + if (err) + goto free_syncs; + + /* Do some error checking first to make the unwind easier */ + for (i = 0; i < args->num_binds; ++i) { + u64 range = bind_ops[i].range; + u64 addr = bind_ops[i].addr; + u32 op = bind_ops[i].op; + + err = __vm_bind_ioctl_lookup_vma(vm, bos[i], addr, range, op); + if (err) + goto release_vm_lock; + } + + for (i = 0; i < args->num_binds; ++i) { + u64 range = bind_ops[i].range; + u64 addr = bind_ops[i].addr; + u32 op = bind_ops[i].op; + u64 obj_offset = bind_ops[i].obj_offset; + u64 gt_mask = bind_ops[i].gt_mask; + u32 region = bind_ops[i].region; + + vmas[i] = vm_bind_ioctl_lookup_vma(vm, bos[i], obj_offset, + addr, range, op, gt_mask, + region); + if (IS_ERR(vmas[i])) { + err = PTR_ERR(vmas[i]); + vmas[i] = NULL; + goto destroy_vmas; + } + } + + for (j = 0; j < args->num_binds; ++j) { + struct xe_sync_entry *__syncs; + u32 __num_syncs = 0; + bool first_or_last = j == 0 || j == args->num_binds - 1; + + if (args->num_binds == 1) { + __num_syncs = num_syncs; + __syncs = syncs; + } else if (first_or_last && num_syncs) { + bool first = j == 0; + + __syncs = kmalloc(sizeof(*__syncs) * num_syncs, + GFP_KERNEL); + if (!__syncs) { + err = ENOMEM; + break; + } + + /* in-syncs on first bind, out-syncs on last bind */ + for (i = 0; i < num_syncs; ++i) { + bool signal = syncs[i].flags & + DRM_XE_SYNC_SIGNAL; + + if ((first && !signal) || (!first && signal)) + __syncs[__num_syncs++] = syncs[i]; + } + } else { + __num_syncs = 0; + __syncs = NULL; + } + + if (async) { + bool last = j == args->num_binds - 1; + + /* + * Each pass of async worker drops the ref, take a ref + * here, 1 set of refs taken above + */ + if (!last) { + if (e) + xe_engine_get(e); + xe_vm_get(vm); + } + + err = vm_bind_ioctl_async(vm, vmas[j], e, bos[j], + bind_ops + j, __syncs, + __num_syncs); + if (err && !last) { + if (e) + xe_engine_put(e); + xe_vm_put(vm); + } + if (err) + break; + } else { + XE_BUG_ON(j != 0); /* Not supported */ + err = vm_bind_ioctl(vm, vmas[j], e, bos[j], + bind_ops + j, __syncs, + __num_syncs, NULL); + break; /* Needed so cleanup loops work */ + } + } + + /* Most of cleanup owned by the async bind worker */ + if (async && !err) { + up_write(&vm->lock); + if (args->num_binds > 1) + kfree(syncs); + goto free_objs; + } + +destroy_vmas: + for (i = j; err && i < args->num_binds; ++i) { + u32 op = bind_ops[i].op; + struct xe_vma *vma, *next; + + if (!vmas[i]) + break; + + list_for_each_entry_safe(vma, next, &vma->unbind_link, + unbind_link) { + list_del_init(&vma->unbind_link); + if (!vma->destroyed) { + prep_vma_destroy(vm, vma); + xe_vma_destroy_unlocked(vma); + } + } + + switch (VM_BIND_OP(op)) { + case XE_VM_BIND_OP_MAP: + prep_vma_destroy(vm, vmas[i]); + xe_vma_destroy_unlocked(vmas[i]); + break; + case XE_VM_BIND_OP_MAP_USERPTR: + prep_vma_destroy(vm, vmas[i]); + xe_vma_destroy_unlocked(vmas[i]); + break; + } + } +release_vm_lock: + up_write(&vm->lock); +free_syncs: + while (num_syncs--) { + if (async && j && + !(syncs[num_syncs].flags & DRM_XE_SYNC_SIGNAL)) + continue; /* Still in async worker */ + xe_sync_entry_cleanup(&syncs[num_syncs]); + } + + kfree(syncs); +put_obj: + for (i = j; i < args->num_binds; ++i) + xe_bo_put(bos[i]); +put_engine: + if (e) + xe_engine_put(e); +put_vm: + xe_vm_put(vm); +free_objs: + kfree(bos); + kfree(vmas); + if (args->num_binds > 1) + kfree(bind_ops); + return err; +} + +/* + * XXX: Using the TTM wrappers for now, likely can call into dma-resv code + * directly to optimize. Also this likely should be an inline function. + */ +int xe_vm_lock(struct xe_vm *vm, struct ww_acquire_ctx *ww, + int num_resv, bool intr) +{ + struct ttm_validate_buffer tv_vm; + LIST_HEAD(objs); + LIST_HEAD(dups); + + XE_BUG_ON(!ww); + + tv_vm.num_shared = num_resv; + tv_vm.bo = xe_vm_ttm_bo(vm);; + list_add_tail(&tv_vm.head, &objs); + + return ttm_eu_reserve_buffers(ww, &objs, intr, &dups); +} + +void xe_vm_unlock(struct xe_vm *vm, struct ww_acquire_ctx *ww) +{ + dma_resv_unlock(&vm->resv); + ww_acquire_fini(ww); +} + +/** + * xe_vm_invalidate_vma - invalidate GPU mappings for VMA without a lock + * @vma: VMA to invalidate + * + * Walks a list of page tables leaves which it memset the entries owned by this + * VMA to zero, invalidates the TLBs, and block until TLBs invalidation is + * complete. + * + * Returns 0 for success, negative error code otherwise. + */ +int xe_vm_invalidate_vma(struct xe_vma *vma) +{ + struct xe_device *xe = vma->vm->xe; + struct xe_gt *gt; + u32 gt_needs_invalidate = 0; + int seqno[XE_MAX_GT]; + u8 id; + int ret; + + XE_BUG_ON(!xe_vm_in_fault_mode(vma->vm)); + trace_xe_vma_usm_invalidate(vma); + + /* Check that we don't race with page-table updates */ + if (IS_ENABLED(CONFIG_PROVE_LOCKING)) { + if (xe_vma_is_userptr(vma)) { + WARN_ON_ONCE(!mmu_interval_check_retry + (&vma->userptr.notifier, + vma->userptr.notifier_seq)); + WARN_ON_ONCE(!dma_resv_test_signaled(&vma->vm->resv, + DMA_RESV_USAGE_BOOKKEEP)); + + } else { + xe_bo_assert_held(vma->bo); + } + } + + for_each_gt(gt, xe, id) { + if (xe_pt_zap_ptes(gt, vma)) { + gt_needs_invalidate |= BIT(id); + xe_device_wmb(xe); + seqno[id] = xe_gt_tlb_invalidation(gt); + if (seqno[id] < 0) + return seqno[id]; + } + } + + for_each_gt(gt, xe, id) { + if (gt_needs_invalidate & BIT(id)) { + ret = xe_gt_tlb_invalidation_wait(gt, seqno[id]); + if (ret < 0) + return ret; + } + } + + vma->usm.gt_invalidated = vma->gt_mask; + + return 0; +} + +#if IS_ENABLED(CONFIG_DRM_XE_SIMPLE_ERROR_CAPTURE) +int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id) +{ + struct rb_node *node; + bool is_lmem; + uint64_t addr; + + if (!down_read_trylock(&vm->lock)) { + drm_printf(p, " Failed to acquire VM lock to dump capture"); + return 0; + } + if (vm->pt_root[gt_id]) { + addr = xe_bo_addr(vm->pt_root[gt_id]->bo, 0, GEN8_PAGE_SIZE, &is_lmem); + drm_printf(p, " VM root: A:0x%llx %s\n", addr, is_lmem ? "LMEM" : "SYS"); + } + + for (node = rb_first(&vm->vmas); node; node = rb_next(node)) { + struct xe_vma *vma = to_xe_vma(node); + bool is_userptr = xe_vma_is_userptr(vma); + + if (is_userptr) { + struct xe_res_cursor cur; + + xe_res_first_sg(vma->userptr.sg, 0, GEN8_PAGE_SIZE, &cur); + addr = xe_res_dma(&cur); + } else { + addr = xe_bo_addr(vma->bo, 0, GEN8_PAGE_SIZE, &is_lmem); + } + drm_printf(p, " [%016llx-%016llx] S:0x%016llx A:%016llx %s\n", + vma->start, vma->end, vma->end - vma->start + 1ull, + addr, is_userptr ? "USR" : is_lmem ? "VRAM" : "SYS"); + } + up_read(&vm->lock); + + return 0; +} +#else +int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id) +{ + return 0; +} +#endif diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h new file mode 100644 index 000000000000..3468ed9d0528 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -0,0 +1,141 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_VM_H_ +#define _XE_VM_H_ + +#include "xe_macros.h" +#include "xe_map.h" +#include "xe_vm_types.h" + +struct drm_device; +struct drm_printer; +struct drm_file; + +struct ttm_buffer_object; +struct ttm_validate_buffer; + +struct xe_engine; +struct xe_file; +struct xe_sync_entry; + +struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags); +void xe_vm_free(struct kref *ref); + +struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id); +int xe_vma_cmp_vma_cb(const void *key, const struct rb_node *node); + +static inline struct xe_vm *xe_vm_get(struct xe_vm *vm) +{ + kref_get(&vm->refcount); + return vm; +} + +static inline void xe_vm_put(struct xe_vm *vm) +{ + kref_put(&vm->refcount, xe_vm_free); +} + +int xe_vm_lock(struct xe_vm *vm, struct ww_acquire_ctx *ww, + int num_resv, bool intr); + +void xe_vm_unlock(struct xe_vm *vm, struct ww_acquire_ctx *ww); + +static inline bool xe_vm_is_closed(struct xe_vm *vm) +{ + /* Only guaranteed not to change when vm->resv is held */ + return !vm->size; +} + +struct xe_vma * +xe_vm_find_overlapping_vma(struct xe_vm *vm, const struct xe_vma *vma); + +#define xe_vm_assert_held(vm) dma_resv_assert_held(&(vm)->resv) + +u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_gt *full_gt); + +int xe_vm_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_vm_bind_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +void xe_vm_close_and_put(struct xe_vm *vm); + +static inline bool xe_vm_in_compute_mode(struct xe_vm *vm) +{ + return vm->flags & XE_VM_FLAG_COMPUTE_MODE; +} + +static inline bool xe_vm_in_fault_mode(struct xe_vm *vm) +{ + return vm->flags & XE_VM_FLAG_FAULT_MODE; +} + +static inline bool xe_vm_no_dma_fences(struct xe_vm *vm) +{ + return xe_vm_in_compute_mode(vm) || xe_vm_in_fault_mode(vm); +} + +int xe_vm_add_compute_engine(struct xe_vm *vm, struct xe_engine *e); + +int xe_vm_userptr_pin(struct xe_vm *vm); + +int __xe_vm_userptr_needs_repin(struct xe_vm *vm); + +int xe_vm_userptr_check_repin(struct xe_vm *vm); + +struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker); + +int xe_vm_invalidate_vma(struct xe_vma *vma); + +int xe_vm_async_fence_wait_start(struct dma_fence *fence); + +extern struct ttm_device_funcs xe_ttm_funcs; + +struct ttm_buffer_object *xe_vm_ttm_bo(struct xe_vm *vm); + +static inline bool xe_vma_is_userptr(struct xe_vma *vma) +{ + return !vma->bo; +} + +int xe_vma_userptr_pin_pages(struct xe_vma *vma); + +int xe_vma_userptr_check_repin(struct xe_vma *vma); + +/* + * XE_ONSTACK_TV is used to size the tv_onstack array that is input + * to xe_vm_lock_dma_resv() and xe_vm_unlock_dma_resv(). + */ +#define XE_ONSTACK_TV 20 +int xe_vm_lock_dma_resv(struct xe_vm *vm, struct ww_acquire_ctx *ww, + struct ttm_validate_buffer *tv_onstack, + struct ttm_validate_buffer **tv, + struct list_head *objs, + bool intr, + unsigned int num_shared); + +void xe_vm_unlock_dma_resv(struct xe_vm *vm, + struct ttm_validate_buffer *tv_onstack, + struct ttm_validate_buffer *tv, + struct ww_acquire_ctx *ww, + struct list_head *objs); + +void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence, + enum dma_resv_usage usage); + +int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id); + +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_VM) +#define vm_dbg drm_dbg +#else +__printf(2, 3) +static inline void vm_dbg(const struct drm_device *dev, + const char *format, ...) +{ /* noop */ } +#endif +#endif diff --git a/drivers/gpu/drm/xe/xe_vm_doc.h b/drivers/gpu/drm/xe/xe_vm_doc.h new file mode 100644 index 000000000000..5b6216964c45 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_doc.h @@ -0,0 +1,555 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_VM_DOC_H_ +#define _XE_VM_DOC_H_ + +/** + * DOC: XE VM (user address space) + * + * VM creation + * =========== + * + * Allocate a physical page for root of the page table structure, create default + * bind engine, and return a handle to the user. + * + * Scratch page + * ------------ + * + * If the VM is created with the flag, DRM_XE_VM_CREATE_SCRATCH_PAGE, set the + * entire page table structure defaults pointing to blank page allocated by the + * VM. Invalid memory access rather than fault just read / write to this page. + * + * VM bind (create GPU mapping for a BO or userptr) + * ================================================ + * + * Creates GPU mapings for a BO or userptr within a VM. VM binds uses the same + * in / out fence interface (struct drm_xe_sync) as execs which allows users to + * think of binds and execs as more or less the same operation. + * + * Operations + * ---------- + * + * XE_VM_BIND_OP_MAP - Create mapping for a BO + * XE_VM_BIND_OP_UNMAP - Destroy mapping for a BO / userptr + * XE_VM_BIND_OP_MAP_USERPTR - Create mapping for userptr + * + * Implementation details + * ~~~~~~~~~~~~~~~~~~~~~~ + * + * All bind operations are implemented via a hybrid approach of using the CPU + * and GPU to modify page tables. If a new physical page is allocated in the + * page table structure we populate that page via the CPU and insert that new + * page into the existing page table structure via a GPU job. Also any existing + * pages in the page table structure that need to be modified also are updated + * via the GPU job. As the root physical page is prealloced on VM creation our + * GPU job will always have at least 1 update. The in / out fences are passed to + * this job so again this is conceptually the same as an exec. + * + * Very simple example of few binds on an empty VM with 48 bits of address space + * and the resulting operations: + * + * .. code-block:: + * + * bind BO0 0x0-0x1000 + * alloc page level 3a, program PTE[0] to BO0 phys address (CPU) + * alloc page level 2, program PDE[0] page level 3a phys address (CPU) + * alloc page level 1, program PDE[0] page level 2 phys address (CPU) + * update root PDE[0] to page level 1 phys address (GPU) + * + * bind BO1 0x201000-0x202000 + * alloc page level 3b, program PTE[1] to BO1 phys address (CPU) + * update page level 2 PDE[1] to page level 3b phys address (GPU) + * + * bind BO2 0x1ff000-0x201000 + * update page level 3a PTE[511] to BO2 phys addres (GPU) + * update page level 3b PTE[0] to BO2 phys addres + 0x1000 (GPU) + * + * GPU bypass + * ~~~~~~~~~~ + * + * In the above example the steps using the GPU can be converted to CPU if the + * bind can be done immediately (all in-fences satisfied, VM dma-resv kernel + * slot is idle). + * + * Address space + * ------------- + * + * Depending on platform either 48 or 57 bits of address space is supported. + * + * Page sizes + * ---------- + * + * The minimum page size is either 4k or 64k depending on platform and memory + * placement (sysmem vs. VRAM). We enforce that binds must be aligned to the + * minimum page size. + * + * Larger pages (2M or 1GB) can be used for BOs in VRAM, the BO physical address + * is aligned to the larger pages size, and VA is aligned to the larger page + * size. Larger pages for userptrs / BOs in sysmem should be possible but is not + * yet implemented. + * + * Sync error handling mode + * ------------------------ + * + * In both modes during the bind IOCTL the user input is validated. In sync + * error handling mode the newly bound BO is validated (potentially moved back + * to a region of memory where is can be used), page tables are updated by the + * CPU and the job to do the GPU binds is created in the IOCTL itself. This step + * can fail due to memory pressure. The user can recover by freeing memory and + * trying this operation again. + * + * Async error handling mode + * ------------------------- + * + * In async error handling the step of validating the BO, updating page tables, + * and generating a job are deferred to an async worker. As this step can now + * fail after the IOCTL has reported success we need an error handling flow for + * which the user can recover from. + * + * The solution is for a user to register a user address with the VM which the + * VM uses to report errors to. The ufence wait interface can be used to wait on + * a VM going into an error state. Once an error is reported the VM's async + * worker is paused. While the VM's async worker is paused sync, + * XE_VM_BIND_OP_UNMAP operations are allowed (this can free memory). Once the + * uses believe the error state is fixed, the async worker can be resumed via + * XE_VM_BIND_OP_RESTART operation. When VM async bind work is restarted, the + * first operation processed is the operation that caused the original error. + * + * Bind queues / engines + * --------------------- + * + * Think of the case where we have two bind operations A + B and are submitted + * in that order. A has in fences while B has none. If using a single bind + * queue, B is now blocked on A's in fences even though it is ready to run. This + * example is a real use case for VK sparse binding. We work around this + * limitation by implementing bind engines. + * + * In the bind IOCTL the user can optionally pass in an engine ID which must map + * to an engine which is of the special class DRM_XE_ENGINE_CLASS_VM_BIND. + * Underneath this is a really virtual engine that can run on any of the copy + * hardware engines. The job(s) created each IOCTL are inserted into this + * engine's ring. In the example above if A and B have different bind engines B + * is free to pass A. If the engine ID field is omitted, the default bind queue + * for the VM is used. + * + * TODO: Explain race in issue 41 and how we solve it + * + * Array of bind operations + * ------------------------ + * + * The uAPI allows multiple binds operations to be passed in via a user array, + * of struct drm_xe_vm_bind_op, in a single VM bind IOCTL. This interface + * matches the VK sparse binding API. The implementation is rather simple, parse + * the array into a list of operations, pass the in fences to the first operation, + * and pass the out fences to the last operation. The ordered nature of a bind + * engine makes this possible. + * + * Munmap semantics for unbinds + * ---------------------------- + * + * Munmap allows things like: + * + * .. code-block:: + * + * 0x0000-0x2000 and 0x3000-0x5000 have mappings + * Munmap 0x1000-0x4000, results in mappings 0x0000-0x1000 and 0x4000-0x5000 + * + * To support this semantic in the above example we decompose the above example + * into 4 operations: + * + * .. code-block:: + * + * unbind 0x0000-0x2000 + * unbind 0x3000-0x5000 + * rebind 0x0000-0x1000 + * rebind 0x4000-0x5000 + * + * Why not just do a partial unbind of 0x1000-0x2000 and 0x3000-0x4000? This + * falls apart when using large pages at the edges and the unbind forces us to + * use a smaller page size. For simplity we always issue a set of unbinds + * unmapping anything in the range and at most 2 rebinds on the edges. + * + * Similar to an array of binds, in fences are passed to the first operation and + * out fences are signaled on the last operation. + * + * In this example there is a window of time where 0x0000-0x1000 and + * 0x4000-0x5000 are invalid but the user didn't ask for these addresses to be + * removed from the mapping. To work around this we treat any munmap style + * unbinds which require a rebind as a kernel operations (BO eviction or userptr + * invalidation). The first operation waits on the VM's + * DMA_RESV_USAGE_PREEMPT_FENCE slots (waits for all pending jobs on VM to + * complete / triggers preempt fences) and the last operation is installed in + * the VM's DMA_RESV_USAGE_KERNEL slot (blocks future jobs / resume compute mode + * VM). The caveat is all dma-resv slots must be updated atomically with respect + * to execs and compute mode rebind worker. To accomplish this, hold the + * vm->lock in write mode from the first operation until the last. + * + * Deferred binds in fault mode + * ---------------------------- + * + * In a VM is in fault mode (TODO: link to fault mode), new bind operations that + * create mappings are by default are deferred to the page fault handler (first + * use). This behavior can be overriden by setting the flag + * XE_VM_BIND_FLAG_IMMEDIATE which indicates to creating the mapping + * immediately. + * + * User pointer + * ============ + * + * User pointers are user allocated memory (malloc'd, mmap'd, etc..) for which the + * user wants to create a GPU mapping. Typically in other DRM drivers a dummy BO + * was created and then a binding was created. We bypass creating a dummy BO in + * XE and simply create a binding directly from the userptr. + * + * Invalidation + * ------------ + * + * Since this a core kernel managed memory the kernel can move this memory + * whenever it wants. We register an invalidation MMU notifier to alert XE when + * a user poiter is about to move. The invalidation notifier needs to block + * until all pending users (jobs or compute mode engines) of the userptr are + * idle to ensure no faults. This done by waiting on all of VM's dma-resv slots. + * + * Rebinds + * ------- + * + * Either the next exec (non-compute) or rebind worker (compute mode) will + * rebind the userptr. The invalidation MMU notifier kicks the rebind worker + * after the VM dma-resv wait if the VM is in compute mode. + * + * Compute mode + * ============ + * + * A VM in compute mode enables long running workloads and ultra low latency + * submission (ULLS). ULLS is implemented via a continuously running batch + + * semaphores. This enables to the user to insert jump to new batch commands + * into the continuously running batch. In both cases these batches exceed the + * time a dma fence is allowed to exist for before signaling, as such dma fences + * are not used when a VM is in compute mode. User fences (TODO: link user fence + * doc) are used instead to signal operation's completion. + * + * Preempt fences + * -------------- + * + * If the kernel decides to move memory around (either userptr invalidate, BO + * eviction, or mumap style unbind which results in a rebind) and a batch is + * running on an engine, that batch can fault or cause a memory corruption as + * page tables for the moved memory are no longer valid. To work around this we + * introduce the concept of preempt fences. When sw signaling is enabled on a + * preempt fence it tells the submission backend to kick that engine off the + * hardware and the preempt fence signals when the engine is off the hardware. + * Once all preempt fences are signaled for a VM the kernel can safely move the + * memory and kick the rebind worker which resumes all the engines execution. + * + * A preempt fence, for every engine using the VM, is installed the VM's + * dma-resv DMA_RESV_USAGE_PREEMPT_FENCE slot. The same preempt fence, for every + * engine using the VM, is also installed into the same dma-resv slot of every + * external BO mapped in the VM. + * + * Rebind worker + * ------------- + * + * The rebind worker is very similar to an exec. It is resposible for rebinding + * evicted BOs or userptrs, waiting on those operations, installing new preempt + * fences, and finally resuming executing of engines in the VM. + * + * Flow + * ~~~~ + * + * .. code-block:: + * + * <----------------------------------------------------------------------| + * Check if VM is closed, if so bail out | + * Lock VM global lock in read mode | + * Pin userptrs (also finds userptr invalidated since last rebind worker) | + * Lock VM dma-resv and external BOs dma-resv | + * Validate BOs that have been evicted | + * Wait on and allocate new preempt fences for every engine using the VM | + * Rebind invalidated userptrs + evicted BOs | + * Wait on last rebind fence | + * Wait VM's DMA_RESV_USAGE_KERNEL dma-resv slot | + * Install preeempt fences and issue resume for every engine using the VM | + * Check if any userptrs invalidated since pin | + * Squash resume for all engines | + * Unlock all | + * Wait all VM's dma-resv slots | + * Retry ---------------------------------------------------------- + * Release all engines waiting to resume + * Unlock all + * + * Timeslicing + * ----------- + * + * In order to prevent an engine from continuously being kicked off the hardware + * and making no forward progress an engine has a period of time it allowed to + * run after resume before it can be kicked off again. This effectively gives + * each engine a timeslice. + * + * Handling multiple GTs + * ===================== + * + * If a GT has slower access to some regions and the page table structure are in + * the slow region, the performance on that GT could adversely be affected. To + * work around this we allow a VM page tables to be shadowed in multiple GTs. + * When VM is created, a default bind engine and PT table structure are created + * on each GT. + * + * Binds can optionally pass in a mask of GTs where a mapping should be created, + * if this mask is zero then default to all the GTs where the VM has page + * tables. + * + * The implementation for this breaks down into a bunch for_each_gt loops in + * various places plus exporting a composite fence for multi-GT binds to the + * user. + * + * Fault mode (unified shared memory) + * ================================== + * + * A VM in fault mode can be enabled on devices that support page faults. If + * page faults are enabled, using dma fences can potentially induce a deadlock: + * A pending page fault can hold up the GPU work which holds up the dma fence + * signaling, and memory allocation is usually required to resolve a page + * fault, but memory allocation is not allowed to gate dma fence signaling. As + * such, dma fences are not allowed when VM is in fault mode. Because dma-fences + * are not allowed, long running workloads and ULLS are enabled on a faulting + * VM. + * + * Defered VM binds + * ---------------- + * + * By default, on a faulting VM binds just allocate the VMA and the actual + * updating of the page tables is defered to the page fault handler. This + * behavior can be overridden by setting the flag XE_VM_BIND_FLAG_IMMEDIATE in + * the VM bind which will then do the bind immediately. + * + * Page fault handler + * ------------------ + * + * Page faults are received in the G2H worker under the CT lock which is in the + * path of dma fences (no memory allocations are allowed, faults require memory + * allocations) thus we cannot process faults under the CT lock. Another issue + * is faults issue TLB invalidations which require G2H credits and we cannot + * allocate G2H credits in the G2H handlers without deadlocking. Lastly, we do + * not want the CT lock to be an outer lock of the VM global lock (VM global + * lock required to fault processing). + * + * To work around the above issue with processing faults in the G2H worker, we + * sink faults to a buffer which is large enough to sink all possible faults on + * the GT (1 per hardware engine) and kick a worker to process the faults. Since + * the page faults G2H are already received in a worker, kicking another worker + * adds more latency to a critical performance path. We add a fast path in the + * G2H irq handler which looks at first G2H and if it is a page fault we sink + * the fault to the buffer and kick the worker to process the fault. TLB + * invalidation responses are also in the critical path so these can also be + * processed in this fast path. + * + * Multiple buffers and workers are used and hashed over based on the ASID so + * faults from different VMs can be processed in parallel. + * + * The page fault handler itself is rather simple, flow is below. + * + * .. code-block:: + * + * Lookup VM from ASID in page fault G2H + * Lock VM global lock in read mode + * Lookup VMA from address in page fault G2H + * Check if VMA is valid, if not bail + * Check if VMA's BO has backing store, if not allocate + * <----------------------------------------------------------------------| + * If userptr, pin pages | + * Lock VM & BO dma-resv locks | + * If atomic fault, migrate to VRAM, else validate BO location | + * Issue rebind | + * Wait on rebind to complete | + * Check if userptr invalidated since pin | + * Drop VM & BO dma-resv locks | + * Retry ---------------------------------------------------------- + * Unlock all + * Issue blocking TLB invalidation | + * Send page fault response to GuC + * + * Access counters + * --------------- + * + * Access counters can be configured to trigger a G2H indicating the device is + * accessing VMAs in system memory frequently as hint to migrate those VMAs to + * VRAM. + * + * Same as the page fault handler, access counters G2H cannot be processed the + * G2H worker under the CT lock. Again we use a buffer to sink access counter + * G2H. Unlike page faults there is no upper bound so if the buffer is full we + * simply drop the G2H. Access counters are a best case optimization and it is + * safe to drop these unlike page faults. + * + * The access counter handler itself is rather simple flow is below. + * + * .. code-block:: + * + * Lookup VM from ASID in access counter G2H + * Lock VM global lock in read mode + * Lookup VMA from address in access counter G2H + * If userptr, bail nothing to do + * Lock VM & BO dma-resv locks + * Issue migration to VRAM + * Unlock all + * + * Notice no rebind is issued in the access counter handler as the rebind will + * be issued on next page fault. + * + * Cavets with eviction / user pointer invalidation + * ------------------------------------------------ + * + * In the case of eviction and user pointer invalidation on a faulting VM, there + * is no need to issue a rebind rather we just need to blow away the page tables + * for the VMAs and the page fault handler will rebind the VMAs when they fault. + * The cavet is to update / read the page table structure the VM global lock is + * neeeed. In both the case of eviction and user pointer invalidation locks are + * held which make acquiring the VM global lock impossible. To work around this + * every VMA maintains a list of leaf page table entries which should be written + * to zero to blow away the VMA's page tables. After writing zero to these + * entries a blocking TLB invalidate is issued. At this point it is safe for the + * kernel to move the VMA's memory around. This is a necessary lockless + * algorithm and is safe as leafs cannot be changed while either an eviction or + * userptr invalidation is occurring. + * + * Locking + * ======= + * + * VM locking protects all of the core data paths (bind operations, execs, + * evictions, and compute mode rebind worker) in XE. + * + * Locks + * ----- + * + * VM global lock (vm->lock) - rw semaphore lock. Outer most lock which protects + * the list of userptrs mapped in the VM, the list of engines using this VM, and + * the array of external BOs mapped in the VM. When adding or removing any of the + * aforemented state from the VM should acquire this lock in write mode. The VM + * bind path also acquires this lock in write while while the exec / compute + * mode rebind worker acquire this lock in read mode. + * + * VM dma-resv lock (vm->ttm.base.resv->lock) - WW lock. Protects VM dma-resv + * slots which is shared with any private BO in the VM. Expected to be acquired + * during VM binds, execs, and compute mode rebind worker. This lock is also + * held when private BOs are being evicted. + * + * external BO dma-resv lock (bo->ttm.base.resv->lock) - WW lock. Protects + * external BO dma-resv slots. Expected to be acquired during VM binds (in + * addition to the VM dma-resv lock). All external BO dma-locks within a VM are + * expected to be acquired (in addition to the VM dma-resv lock) during execs + * and the compute mode rebind worker. This lock is also held when an external + * BO is being evicted. + * + * Putting it all together + * ----------------------- + * + * 1. An exec and bind operation with the same VM can't be executing at the same + * time (vm->lock). + * + * 2. A compute mode rebind worker and bind operation with the same VM can't be + * executing at the same time (vm->lock). + * + * 3. We can't add / remove userptrs or external BOs to a VM while an exec with + * the same VM is executing (vm->lock). + * + * 4. We can't add / remove userptrs, external BOs, or engines to a VM while a + * compute mode rebind worker with the same VM is executing (vm->lock). + * + * 5. Evictions within a VM can't be happen while an exec with the same VM is + * executing (dma-resv locks). + * + * 6. Evictions within a VM can't be happen while a compute mode rebind worker + * with the same VM is executing (dma-resv locks). + * + * dma-resv usage + * ============== + * + * As previously stated to enforce the ordering of kernel ops (eviction, userptr + * invalidation, munmap style unbinds which result in a rebind), rebinds during + * execs, execs, and resumes in the rebind worker we use both the VMs and + * external BOs dma-resv slots. Let try to make this as clear as possible. + * + * Slot installation + * ----------------- + * + * 1. Jobs from kernel ops install themselves into the DMA_RESV_USAGE_KERNEL + * slot of either an external BO or VM (depends on if kernel op is operating on + * an external or private BO) + * + * 2. In non-compute mode, jobs from execs install themselves into the + * DMA_RESV_USAGE_BOOKKEEP slot of the VM + * + * 3. In non-compute mode, jobs from execs install themselves into the + * DMA_RESV_USAGE_WRITE slot of all external BOs in the VM + * + * 4. Jobs from binds install themselves into the DMA_RESV_USAGE_BOOKKEEP slot + * of the VM + * + * 5. Jobs from binds install themselves into the DMA_RESV_USAGE_BOOKKEEP slot + * of the external BO (if the bind is to an external BO, this is addition to #4) + * + * 6. Every engine using a compute mode VM has a preempt fence in installed into + * the DMA_RESV_USAGE_PREEMPT_FENCE slot of the VM + * + * 7. Every engine using a compute mode VM has a preempt fence in installed into + * the DMA_RESV_USAGE_PREEMPT_FENCE slot of all the external BOs in the VM + * + * Slot waiting + * ------------ + * + * 1. The exection of all jobs from kernel ops shall wait on all slots + * (DMA_RESV_USAGE_PREEMPT_FENCE) of either an external BO or VM (depends on if + * kernel op is operating on external or private BO) + * + * 2. In non-compute mode, the exection of all jobs from rebinds in execs shall + * wait on the DMA_RESV_USAGE_KERNEL slot of either an external BO or VM + * (depends on if the rebind is operatiing on an external or private BO) + * + * 3. In non-compute mode, the exection of all jobs from execs shall wait on the + * last rebind job + * + * 4. In compute mode, the exection of all jobs from rebinds in the rebind + * worker shall wait on the DMA_RESV_USAGE_KERNEL slot of either an external BO + * or VM (depends on if rebind is operating on external or private BO) + * + * 5. In compute mode, resumes in rebind worker shall wait on last rebind fence + * + * 6. In compute mode, resumes in rebind worker shall wait on the + * DMA_RESV_USAGE_KERNEL slot of the VM + * + * Putting it all together + * ----------------------- + * + * 1. New jobs from kernel ops are blocked behind any existing jobs from + * non-compute mode execs + * + * 2. New jobs from non-compute mode execs are blocked behind any existing jobs + * from kernel ops and rebinds + * + * 3. New jobs from kernel ops are blocked behind all preempt fences signaling in + * compute mode + * + * 4. Compute mode engine resumes are blocked behind any existing jobs from + * kernel ops and rebinds + * + * Future work + * =========== + * + * Support large pages for sysmem and userptr. + * + * Update page faults to handle BOs are page level grainularity (e.g. part of BO + * could be in system memory while another part could be in VRAM). + * + * Page fault handler likely we be optimized a bit more (e.g. Rebinds always + * wait on the dma-resv kernel slots of VM or BO, technically we only have to + * wait the BO moving. If using a job to do the rebind, we could not block in + * the page fault handler rather attach a callback to fence of the rebind job to + * signal page fault complete. Our handling of short circuting for atomic faults + * for bound VMAs could be better. etc...). We can tune all of this once we have + * benchmarks / performance number from workloads up and running. + */ + +#endif diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c new file mode 100644 index 000000000000..4498aa2fbd47 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_madvise.c @@ -0,0 +1,347 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2021 Intel Corporation + */ + +#include +#include +#include + +#include "xe_bo.h" +#include "xe_vm.h" +#include "xe_vm_madvise.h" + +static int madvise_preferred_mem_class(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + u64 value) +{ + int i, err; + + if (XE_IOCTL_ERR(xe, value > XE_MEM_REGION_CLASS_VRAM)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, value == XE_MEM_REGION_CLASS_VRAM && + !xe->info.is_dgfx)) + return -EINVAL; + + for (i = 0; i < num_vmas; ++i) { + struct xe_bo *bo; + struct ww_acquire_ctx ww; + + bo = vmas[i]->bo; + + err = xe_bo_lock(bo, &ww, 0, true); + if (err) + return err; + bo->props.preferred_mem_class = value; + xe_bo_placement_for_flags(xe, bo, bo->flags); + xe_bo_unlock(bo, &ww); + } + + return 0; +} + +static int madvise_preferred_gt(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, u64 value) +{ + int i, err; + + if (XE_IOCTL_ERR(xe, value > xe->info.tile_count)) + return -EINVAL; + + for (i = 0; i < num_vmas; ++i) { + struct xe_bo *bo; + struct ww_acquire_ctx ww; + + bo = vmas[i]->bo; + + err = xe_bo_lock(bo, &ww, 0, true); + if (err) + return err; + bo->props.preferred_gt = value; + xe_bo_placement_for_flags(xe, bo, bo->flags); + xe_bo_unlock(bo, &ww); + } + + return 0; +} + +static int madvise_preferred_mem_class_gt(struct xe_device *xe, + struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, + u64 value) +{ + int i, err; + u32 gt_id = upper_32_bits(value); + u32 mem_class = lower_32_bits(value); + + if (XE_IOCTL_ERR(xe, mem_class > XE_MEM_REGION_CLASS_VRAM)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, mem_class == XE_MEM_REGION_CLASS_VRAM && + !xe->info.is_dgfx)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, gt_id > xe->info.tile_count)) + return -EINVAL; + + for (i = 0; i < num_vmas; ++i) { + struct xe_bo *bo; + struct ww_acquire_ctx ww; + + bo = vmas[i]->bo; + + err = xe_bo_lock(bo, &ww, 0, true); + if (err) + return err; + bo->props.preferred_mem_class = mem_class; + bo->props.preferred_gt = gt_id; + xe_bo_placement_for_flags(xe, bo, bo->flags); + xe_bo_unlock(bo, &ww); + } + + return 0; +} + +static int madvise_cpu_atomic(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, u64 value) +{ + int i, err; + + for (i = 0; i < num_vmas; ++i) { + struct xe_bo *bo; + struct ww_acquire_ctx ww; + + bo = vmas[i]->bo; + if (XE_IOCTL_ERR(xe, !(bo->flags & XE_BO_CREATE_SYSTEM_BIT))) + return -EINVAL; + + err = xe_bo_lock(bo, &ww, 0, true); + if (err) + return err; + bo->props.cpu_atomic = !!value; + + /* + * All future CPU accesses must be from system memory only, we + * just invalidate the CPU page tables which will trigger a + * migration on next access. + */ + if (bo->props.cpu_atomic) + ttm_bo_unmap_virtual(&bo->ttm); + xe_bo_unlock(bo, &ww); + } + + return 0; +} + +static int madvise_device_atomic(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, u64 value) +{ + int i, err; + + for (i = 0; i < num_vmas; ++i) { + struct xe_bo *bo; + struct ww_acquire_ctx ww; + + bo = vmas[i]->bo; + if (XE_IOCTL_ERR(xe, !(bo->flags & XE_BO_CREATE_VRAM0_BIT) && + !(bo->flags & XE_BO_CREATE_VRAM1_BIT))) + return -EINVAL; + + err = xe_bo_lock(bo, &ww, 0, true); + if (err) + return err; + bo->props.device_atomic = !!value; + xe_bo_unlock(bo, &ww); + } + + return 0; +} + +static int madvise_priority(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, u64 value) +{ + int i, err; + + if (XE_IOCTL_ERR(xe, value > DRM_XE_VMA_PRIORITY_HIGH)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, value == DRM_XE_VMA_PRIORITY_HIGH && + !capable(CAP_SYS_NICE))) + return -EPERM; + + for (i = 0; i < num_vmas; ++i) { + struct xe_bo *bo; + struct ww_acquire_ctx ww; + + bo = vmas[i]->bo; + + err = xe_bo_lock(bo, &ww, 0, true); + if (err) + return err; + bo->ttm.priority = value; + ttm_bo_move_to_lru_tail(&bo->ttm); + xe_bo_unlock(bo, &ww); + } + + return 0; +} + +static int madvise_pin(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, u64 value) +{ + XE_WARN_ON("NIY"); + return 0; +} + +typedef int (*madvise_func)(struct xe_device *xe, struct xe_vm *vm, + struct xe_vma **vmas, int num_vmas, u64 value); + +static const madvise_func madvise_funcs[] = { + [DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS] = madvise_preferred_mem_class, + [DRM_XE_VM_MADVISE_PREFERRED_GT] = madvise_preferred_gt, + [DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS_GT] = + madvise_preferred_mem_class_gt, + [DRM_XE_VM_MADVISE_CPU_ATOMIC] = madvise_cpu_atomic, + [DRM_XE_VM_MADVISE_DEVICE_ATOMIC] = madvise_device_atomic, + [DRM_XE_VM_MADVISE_PRIORITY] = madvise_priority, + [DRM_XE_VM_MADVISE_PIN] = madvise_pin, +}; + +static struct xe_vma *node_to_vma(const struct rb_node *node) +{ + BUILD_BUG_ON(offsetof(struct xe_vma, vm_node) != 0); + return (struct xe_vma *)node; +} + +static struct xe_vma ** +get_vmas(struct xe_vm *vm, int *num_vmas, u64 addr, u64 range) +{ + struct xe_vma **vmas; + struct xe_vma *vma, *__vma, lookup; + int max_vmas = 8; + struct rb_node *node; + + lockdep_assert_held(&vm->lock); + + vmas = kmalloc(max_vmas * sizeof(*vmas), GFP_KERNEL); + if (!vmas) + return NULL; + + lookup.start = addr; + lookup.end = addr + range - 1; + + vma = xe_vm_find_overlapping_vma(vm, &lookup); + if (!vma) + return vmas; + + if (!xe_vma_is_userptr(vma)) { + vmas[*num_vmas] = vma; + *num_vmas += 1; + } + + node = &vma->vm_node; + while ((node = rb_next(node))) { + if (!xe_vma_cmp_vma_cb(&lookup, node)) { + __vma = node_to_vma(node); + if (xe_vma_is_userptr(__vma)) + continue; + + if (*num_vmas == max_vmas) { + struct xe_vma **__vmas = + krealloc(vmas, max_vmas * sizeof(*vmas), + GFP_KERNEL); + + if (!__vmas) + return NULL; + vmas = __vmas; + } + vmas[*num_vmas] = __vma; + *num_vmas += 1; + } else { + break; + } + } + + node = &vma->vm_node; + while ((node = rb_prev(node))) { + if (!xe_vma_cmp_vma_cb(&lookup, node)) { + __vma = node_to_vma(node); + if (xe_vma_is_userptr(__vma)) + continue; + + if (*num_vmas == max_vmas) { + struct xe_vma **__vmas = + krealloc(vmas, max_vmas * sizeof(*vmas), + GFP_KERNEL); + + if (!__vmas) + return NULL; + vmas = __vmas; + } + vmas[*num_vmas] = __vma; + *num_vmas += 1; + } else { + break; + } + } + + return vmas; +} + +int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_vm_madvise *args = data; + struct xe_vm *vm; + struct xe_vma **vmas = NULL; + int num_vmas = 0, err = 0, idx; + + if (XE_IOCTL_ERR(xe, args->extensions)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->property > ARRAY_SIZE(madvise_funcs))) + return -EINVAL; + + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_ERR(xe, !vm)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, xe_vm_is_closed(vm))) { + err = -ENOENT; + goto put_vm; + } + + if (XE_IOCTL_ERR(xe, !xe_vm_in_fault_mode(vm))) { + err = -EINVAL; + goto put_vm; + } + + down_read(&vm->lock); + + vmas = get_vmas(vm, &num_vmas, args->addr, args->range); + if (XE_IOCTL_ERR(xe, err)) + goto unlock_vm; + + if (XE_IOCTL_ERR(xe, !vmas)) { + err = -ENOMEM; + goto unlock_vm; + } + + if (XE_IOCTL_ERR(xe, !num_vmas)) { + err = -EINVAL; + goto unlock_vm; + } + + idx = array_index_nospec(args->property, ARRAY_SIZE(madvise_funcs)); + err = madvise_funcs[idx](xe, vm, vmas, num_vmas, args->value); + +unlock_vm: + up_read(&vm->lock); +put_vm: + xe_vm_put(vm); + kfree(vmas); + return err; +} diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.h b/drivers/gpu/drm/xe/xe_vm_madvise.h new file mode 100644 index 000000000000..eecd33acd248 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_madvise.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2021 Intel Corporation + */ + +#ifndef _XE_VM_MADVISE_H_ +#define _XE_VM_MADVISE_H_ + +struct drm_device; +struct drm_file; + +int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +#endif diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h new file mode 100644 index 000000000000..2a3b911ab358 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -0,0 +1,337 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_VM_TYPES_H_ +#define _XE_VM_TYPES_H_ + +#include +#include +#include +#include + +#include "xe_device_types.h" +#include "xe_pt_types.h" + +struct xe_bo; +struct xe_vm; + +struct xe_vma { + struct rb_node vm_node; + /** @vm: VM which this VMA belongs to */ + struct xe_vm *vm; + + /** + * @start: start address of this VMA within its address domain, end - + * start + 1 == VMA size + */ + u64 start; + /** @end: end address of this VMA within its address domain */ + u64 end; + /** @pte_flags: pte flags for this VMA */ + u32 pte_flags; + + /** @bo: BO if not a userptr, must be NULL is userptr */ + struct xe_bo *bo; + /** @bo_offset: offset into BO if not a userptr, unused for userptr */ + u64 bo_offset; + + /** @gt_mask: GT mask of where to create binding for this VMA */ + u64 gt_mask; + + /** + * @gt_present: GT mask of binding are present for this VMA. + * protected by vm->lock, vm->resv and for userptrs, + * vm->userptr.notifier_lock for writing. Needs either for reading, + * but if reading is done under the vm->lock only, it needs to be held + * in write mode. + */ + u64 gt_present; + + /** + * @destroyed: VMA is destroyed, in the sense that it shouldn't be + * subject to rebind anymore. This field must be written under + * the vm lock in write mode and the userptr.notifier_lock in + * either mode. Read under the vm lock or the userptr.notifier_lock in + * write mode. + */ + bool destroyed; + + /** + * @first_munmap_rebind: VMA is first in a sequence of ops that triggers + * a rebind (munmap style VM unbinds). This indicates the operation + * using this VMA must wait on all dma-resv slots (wait for pending jobs + * / trigger preempt fences). + */ + bool first_munmap_rebind; + + /** + * @last_munmap_rebind: VMA is first in a sequence of ops that triggers + * a rebind (munmap style VM unbinds). This indicates the operation + * using this VMA must install itself into kernel dma-resv slot (blocks + * future jobs) and kick the rebind work in compute mode. + */ + bool last_munmap_rebind; + + /** @use_atomic_access_pte_bit: Set atomic access bit in PTE */ + bool use_atomic_access_pte_bit; + + union { + /** @bo_link: link into BO if not a userptr */ + struct list_head bo_link; + /** @userptr_link: link into VM repin list if userptr */ + struct list_head userptr_link; + }; + + /** + * @rebind_link: link into VM if this VMA needs rebinding, and + * if it's a bo (not userptr) needs validation after a possible + * eviction. Protected by the vm's resv lock. + */ + struct list_head rebind_link; + + /** + * @unbind_link: link or list head if an unbind of multiple VMAs, in + * single unbind op, is being done. + */ + struct list_head unbind_link; + + /** @destroy_cb: callback to destroy VMA when unbind job is done */ + struct dma_fence_cb destroy_cb; + + /** @destroy_work: worker to destroy this BO */ + struct work_struct destroy_work; + + /** @userptr: user pointer state */ + struct { + /** @ptr: user pointer */ + uintptr_t ptr; + /** @invalidate_link: Link for the vm::userptr.invalidated list */ + struct list_head invalidate_link; + /** + * @notifier: MMU notifier for user pointer (invalidation call back) + */ + struct mmu_interval_notifier notifier; + /** @sgt: storage for a scatter gather table */ + struct sg_table sgt; + /** @sg: allocated scatter gather table */ + struct sg_table *sg; + /** @notifier_seq: notifier sequence number */ + unsigned long notifier_seq; + /** + * @initial_bind: user pointer has been bound at least once. + * write: vm->userptr.notifier_lock in read mode and vm->resv held. + * read: vm->userptr.notifier_lock in write mode or vm->resv held. + */ + bool initial_bind; +#if IS_ENABLED(CONFIG_DRM_XE_USERPTR_INVAL_INJECT) + u32 divisor; +#endif + } userptr; + + /** @usm: unified shared memory state */ + struct { + /** @gt_invalidated: VMA has been invalidated */ + u64 gt_invalidated; + } usm; + + struct { + struct list_head rebind_link; + } notifier; + + struct { + /** + * @extobj.link: Link into vm's external object list. + * protected by the vm lock. + */ + struct list_head link; + } extobj; +}; + +struct xe_device; + +#define xe_vm_assert_held(vm) dma_resv_assert_held(&(vm)->resv) + +struct xe_vm { + struct xe_device *xe; + + struct kref refcount; + + /* engine used for (un)binding vma's */ + struct xe_engine *eng[XE_MAX_GT]; + + /** Protects @rebind_list and the page-table structures */ + struct dma_resv resv; + + u64 size; + struct rb_root vmas; + + struct xe_pt *pt_root[XE_MAX_GT]; + struct xe_bo *scratch_bo[XE_MAX_GT]; + struct xe_pt *scratch_pt[XE_MAX_GT][XE_VM_MAX_LEVEL]; + + /** @flags: flags for this VM, statically setup a creation time */ +#define XE_VM_FLAGS_64K BIT(0) +#define XE_VM_FLAG_COMPUTE_MODE BIT(1) +#define XE_VM_FLAG_ASYNC_BIND_OPS BIT(2) +#define XE_VM_FLAG_MIGRATION BIT(3) +#define XE_VM_FLAG_SCRATCH_PAGE BIT(4) +#define XE_VM_FLAG_FAULT_MODE BIT(5) +#define XE_VM_FLAG_GT_ID(flags) (((flags) >> 6) & 0x3) +#define XE_VM_FLAG_SET_GT_ID(gt) ((gt)->info.id << 6) + unsigned long flags; + + /** @composite_fence_ctx: context composite fence */ + u64 composite_fence_ctx; + /** @composite_fence_seqno: seqno for composite fence */ + u32 composite_fence_seqno; + + /** + * @lock: outer most lock, protects objects of anything attached to this + * VM + */ + struct rw_semaphore lock; + + /** + * @rebind_list: list of VMAs that need rebinding, and if they are + * bos (not userptr), need validation after a possible eviction. The + * list is protected by @resv. + */ + struct list_head rebind_list; + + /** @rebind_fence: rebind fence from execbuf */ + struct dma_fence *rebind_fence; + + /** + * @destroy_work: worker to destroy VM, needed as a dma_fence signaling + * from an irq context can be last put and the destroy needs to be able + * to sleep. + */ + struct work_struct destroy_work; + + /** @extobj: bookkeeping for external objects. Protected by the vm lock */ + struct { + /** @enties: number of external BOs attached this VM */ + u32 entries; + /** @list: list of vmas with external bos attached */ + struct list_head list; + } extobj; + + /** @async_ops: async VM operations (bind / unbinds) */ + struct { + /** @list: list of pending async VM ops */ + struct list_head pending; + /** @work: worker to execute async VM ops */ + struct work_struct work; + /** @lock: protects list of pending async VM ops and fences */ + spinlock_t lock; + /** @error_capture: error capture state */ + struct { + /** @mm: user MM */ + struct mm_struct *mm; + /** + * @addr: user pointer to copy error capture state too + */ + u64 addr; + /** @wq: user fence wait queue for VM errors */ + wait_queue_head_t wq; + } error_capture; + /** @fence: fence state */ + struct { + /** @context: context of async fence */ + u64 context; + /** @seqno: seqno of async fence */ + u32 seqno; + } fence; + /** @error: error state for async VM ops */ + int error; + /** + * @munmap_rebind_inflight: an munmap style VM bind is in the + * middle of a set of ops which requires a rebind at the end. + */ + bool munmap_rebind_inflight; + } async_ops; + + /** @userptr: user pointer state */ + struct { + /** + * @userptr.repin_list: list of VMAs which are user pointers, + * and needs repinning. Protected by @lock. + */ + struct list_head repin_list; + /** + * @notifier_lock: protects notifier in write mode and + * submission in read mode. + */ + struct rw_semaphore notifier_lock; + /** + * @userptr.invalidated_lock: Protects the + * @userptr.invalidated list. + */ + spinlock_t invalidated_lock; + /** + * @userptr.invalidated: List of invalidated userptrs, not yet + * picked + * up for revalidation. Protected from access with the + * @invalidated_lock. Removing items from the list + * additionally requires @lock in write mode, and adding + * items to the list requires the @userptr.notifer_lock in + * write mode. + */ + struct list_head invalidated; + } userptr; + + /** @preempt: preempt state */ + struct { + /** + * @min_run_period_ms: The minimum run period before preempting + * an engine again + */ + s64 min_run_period_ms; + /** @engines: list of engines attached to this VM */ + struct list_head engines; + /** @num_engines: number user engines attached to this VM */ + int num_engines; + /** + * @rebind_work: worker to rebind invalidated userptrs / evicted + * BOs + */ + struct work_struct rebind_work; + } preempt; + + /** @um: unified memory state */ + struct { + /** @asid: address space ID, unique to each VM */ + u32 asid; + /** + * @last_fault_vma: Last fault VMA, used for fast lookup when we + * get a flood of faults to the same VMA + */ + struct xe_vma *last_fault_vma; + } usm; + + /** + * @notifier: Lists and locks for temporary usage within notifiers where + * we either can't grab the vm lock or the vm resv. + */ + struct { + /** @notifier.list_lock: lock protecting @rebind_list */ + spinlock_t list_lock; + /** + * @notifier.rebind_list: list of vmas that we want to put on the + * main @rebind_list. This list is protected for writing by both + * notifier.list_lock, and the resv of the bo the vma points to, + * and for reading by the notifier.list_lock only. + */ + struct list_head rebind_list; + } notifier; + + /** @error_capture: allow to track errors */ + struct { + /** @capture_once: capture only one error per VM */ + bool capture_once; + } error_capture; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_wa.c b/drivers/gpu/drm/xe/xe_wa.c new file mode 100644 index 000000000000..b56141ba7145 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wa.c @@ -0,0 +1,326 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_wa.h" + +#include + +#include "xe_device_types.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_hw_engine_types.h" +#include "xe_mmio.h" +#include "xe_platform_types.h" +#include "xe_rtp.h" +#include "xe_step.h" + +#include "gt/intel_engine_regs.h" +#include "gt/intel_gt_regs.h" +#include "i915_reg.h" + +/** + * DOC: Hardware workarounds + * + * Hardware workarounds are register programming documented to be executed in + * the driver that fall outside of the normal programming sequences for a + * platform. There are some basic categories of workarounds, depending on + * how/when they are applied: + * + * - LRC workarounds: workarounds that touch registers that are + * saved/restored to/from the HW context image. The list is emitted (via Load + * Register Immediate commands) once when initializing the device and saved in + * the default context. That default context is then used on every context + * creation to have a "primed golden context", i.e. a context image that + * already contains the changes needed to all the registers. + * + * TODO: Although these workarounds are maintained here, they are not + * currently being applied. + * + * - Engine workarounds: the list of these WAs is applied whenever the specific + * engine is reset. It's also possible that a set of engine classes share a + * common power domain and they are reset together. This happens on some + * platforms with render and compute engines. In this case (at least) one of + * them need to keeep the workaround programming: the approach taken in the + * driver is to tie those workarounds to the first compute/render engine that + * is registered. When executing with GuC submission, engine resets are + * outside of kernel driver control, hence the list of registers involved in + * written once, on engine initialization, and then passed to GuC, that + * saves/restores their values before/after the reset takes place. See + * ``drivers/gpu/drm/xe/xe_guc_ads.c`` for reference. + * + * - GT workarounds: the list of these WAs is applied whenever these registers + * revert to their default values: on GPU reset, suspend/resume [1]_, etc. + * + * - Register whitelist: some workarounds need to be implemented in userspace, + * but need to touch privileged registers. The whitelist in the kernel + * instructs the hardware to allow the access to happen. From the kernel side, + * this is just a special case of a MMIO workaround (as we write the list of + * these to/be-whitelisted registers to some special HW registers). + * + * - Workaround batchbuffers: buffers that get executed automatically by the + * hardware on every HW context restore. These buffers are created and + * programmed in the default context so the hardware always go through those + * programming sequences when switching contexts. The support for workaround + * batchbuffers is enabled these hardware mechanisms: + * + * #. INDIRECT_CTX: A batchbuffer and an offset are provided in the default + * context, pointing the hardware to jump to that location when that offset + * is reached in the context restore. Workaround batchbuffer in the driver + * currently uses this mechanism for all platforms. + * + * #. BB_PER_CTX_PTR: A batchbuffer is provided in the default context, + * pointing the hardware to a buffer to continue executing after the + * engine registers are restored in a context restore sequence. This is + * currently not used in the driver. + * + * - Other: There are WAs that, due to their nature, cannot be applied from a + * central place. Those are peppered around the rest of the code, as needed. + * Workarounds related to the display IP are the main example. + * + * .. [1] Technically, some registers are powercontext saved & restored, so they + * survive a suspend/resume. In practice, writing them again is not too + * costly and simplifies things, so it's the approach taken in the driver. + * + * .. note:: + * Hardware workarounds in xe work the same way as in i915, with the + * difference of how they are maintained in the code. In xe it uses the + * xe_rtp infrastructure so the workarounds can be kept in tables, following + * a more declarative approach rather than procedural. + */ + +#undef _MMIO +#undef MCR_REG +#define _MMIO(x) _XE_RTP_REG(x) +#define MCR_REG(x) _XE_RTP_MCR_REG(x) + +static bool match_14011060649(const struct xe_gt *gt, + const struct xe_hw_engine *hwe) +{ + return hwe->instance % 2 == 0; +} + +static const struct xe_rtp_entry gt_was[] = { + { XE_RTP_NAME("14011060649"), + XE_RTP_RULES(MEDIA_VERSION_RANGE(1200, 1255), + ENGINE_CLASS(VIDEO_DECODE), + FUNC(match_14011060649)), + XE_RTP_SET(VDBOX_CGCTL3F10(0), IECPUNIT_CLKGATE_DIS, + XE_RTP_FLAG(FOREACH_ENGINE)) + }, + { XE_RTP_NAME("16010515920"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10), + STEP(A0, B0), + ENGINE_CLASS(VIDEO_DECODE)), + XE_RTP_SET(VDBOX_CGCTL3F18(0), ALNUNIT_CLKGATE_DIS, + XE_RTP_FLAG(FOREACH_ENGINE)) + }, + { XE_RTP_NAME("22010523718"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10)), + XE_RTP_SET(UNSLICE_UNIT_LEVEL_CLKGATE, CG3DDISCFEG_CLKGATE_DIS) + }, + { XE_RTP_NAME("14011006942"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10)), + XE_RTP_SET(GEN11_SUBSLICE_UNIT_LEVEL_CLKGATE, DSS_ROUTER_CLKGATE_DIS) + }, + { XE_RTP_NAME("14010948348"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10), STEP(A0, B0)), + XE_RTP_SET(UNSLCGCTL9430, MSQDUNIT_CLKGATE_DIS) + }, + { XE_RTP_NAME("14011037102"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10), STEP(A0, B0)), + XE_RTP_SET(UNSLCGCTL9444, LTCDD_CLKGATE_DIS) + }, + { XE_RTP_NAME("14011371254"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10), STEP(A0, B0)), + XE_RTP_SET(GEN11_SLICE_UNIT_LEVEL_CLKGATE, NODEDSS_CLKGATE_DIS) + }, + { XE_RTP_NAME("14011431319/0"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10), STEP(A0, B0)), + XE_RTP_SET(UNSLCGCTL9440, + GAMTLBOACS_CLKGATE_DIS | + GAMTLBVDBOX7_CLKGATE_DIS | GAMTLBVDBOX6_CLKGATE_DIS | + GAMTLBVDBOX5_CLKGATE_DIS | GAMTLBVDBOX4_CLKGATE_DIS | + GAMTLBVDBOX3_CLKGATE_DIS | GAMTLBVDBOX2_CLKGATE_DIS | + GAMTLBVDBOX1_CLKGATE_DIS | GAMTLBVDBOX0_CLKGATE_DIS | + GAMTLBKCR_CLKGATE_DIS | GAMTLBGUC_CLKGATE_DIS | + GAMTLBBLT_CLKGATE_DIS) + }, + { XE_RTP_NAME("14011431319/1"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10), STEP(A0, B0)), + XE_RTP_SET(UNSLCGCTL9444, + GAMTLBGFXA0_CLKGATE_DIS | GAMTLBGFXA1_CLKGATE_DIS | + GAMTLBCOMPA0_CLKGATE_DIS | GAMTLBCOMPA1_CLKGATE_DIS | + GAMTLBCOMPB0_CLKGATE_DIS | GAMTLBCOMPB1_CLKGATE_DIS | + GAMTLBCOMPC0_CLKGATE_DIS | GAMTLBCOMPC1_CLKGATE_DIS | + GAMTLBCOMPD0_CLKGATE_DIS | GAMTLBCOMPD1_CLKGATE_DIS | + GAMTLBMERT_CLKGATE_DIS | + GAMTLBVEBOX3_CLKGATE_DIS | GAMTLBVEBOX2_CLKGATE_DIS | + GAMTLBVEBOX1_CLKGATE_DIS | GAMTLBVEBOX0_CLKGATE_DIS) + }, + { XE_RTP_NAME("14010569222"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10), STEP(A0, B0)), + XE_RTP_SET(UNSLICE_UNIT_LEVEL_CLKGATE, GAMEDIA_CLKGATE_DIS) + }, + { XE_RTP_NAME("14011028019"), + XE_RTP_RULES(SUBPLATFORM(DG2, G10), STEP(A0, B0)), + XE_RTP_SET(SSMCGCTL9530, RTFUNIT_CLKGATE_DIS) + }, + { XE_RTP_NAME("14014830051"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_CLR(SARB_CHICKEN1, COMP_CKN_IN) + }, + { XE_RTP_NAME("14015795083"), + XE_RTP_RULES(PLATFORM(DG2)), + XE_RTP_CLR(GEN7_MISCCPCTL, GEN12_DOP_CLOCK_GATE_RENDER_ENABLE) + }, + { XE_RTP_NAME("14011059788"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)), + XE_RTP_SET(GEN10_DFR_RATIO_EN_AND_CHICKEN, DFR_DISABLE) + }, + { XE_RTP_NAME("1409420604"), + XE_RTP_RULES(PLATFORM(DG1)), + XE_RTP_SET(SUBSLICE_UNIT_LEVEL_CLKGATE2, CPSSUNIT_CLKGATE_DIS) + }, + { XE_RTP_NAME("1408615072"), + XE_RTP_RULES(PLATFORM(DG1)), + XE_RTP_SET(UNSLICE_UNIT_LEVEL_CLKGATE2, VSUNIT_CLKGATE_DIS_TGL) + }, + {} +}; + +static const struct xe_rtp_entry engine_was[] = { + { XE_RTP_NAME("14015227452"), + XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)), + XE_RTP_SET(GEN9_ROW_CHICKEN4, XEHP_DIS_BBL_SYSPIPE, + XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("1606931601"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_RTP_SET(GEN7_ROW_CHICKEN2, GEN12_DISABLE_EARLY_READ, + XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("22010931296, 18011464164, 14010919138"), + XE_RTP_RULES(GRAPHICS_VERSION(1200), ENGINE_CLASS(RENDER)), + XE_RTP_SET(GEN7_FF_THREAD_MODE, GEN12_FF_TESSELATION_DOP_GATE_DISABLE) + }, + { XE_RTP_NAME("14010826681, 1606700617, 22010271021"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_RTP_SET(GEN9_CS_DEBUG_MODE1, FF_DOP_CLOCK_GATE_DISABLE, + XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("18019627453"), + XE_RTP_RULES(PLATFORM(DG2), ENGINE_CLASS(RENDER)), + XE_RTP_SET(GEN9_CS_DEBUG_MODE1, FF_DOP_CLOCK_GATE_DISABLE, + XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("1409804808"), + XE_RTP_RULES(GRAPHICS_VERSION(1200), + ENGINE_CLASS(RENDER), + IS_INTEGRATED), + XE_RTP_SET(GEN7_ROW_CHICKEN2, GEN12_PUSH_CONST_DEREF_HOLD_DIS, + XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("14010229206, 1409085225"), + XE_RTP_RULES(GRAPHICS_VERSION(1200), + ENGINE_CLASS(RENDER), + IS_INTEGRATED), + XE_RTP_SET(GEN9_ROW_CHICKEN4, GEN12_DISABLE_TDL_PUSH, + XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("1607297627, 1607030317, 1607186500"), + XE_RTP_RULES(PLATFORM(TIGERLAKE), ENGINE_CLASS(RENDER)), + XE_RTP_SET(RING_PSMI_CTL(RENDER_RING_BASE), + GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE | + GEN8_RC_SEMA_IDLE_MSG_DISABLE, XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("1607297627, 1607030317, 1607186500"), + XE_RTP_RULES(PLATFORM(ROCKETLAKE), ENGINE_CLASS(RENDER)), + XE_RTP_SET(RING_PSMI_CTL(RENDER_RING_BASE), + GEN12_WAIT_FOR_EVENT_POWER_DOWN_DISABLE | + GEN8_RC_SEMA_IDLE_MSG_DISABLE, XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("1406941453"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210), ENGINE_CLASS(RENDER)), + XE_RTP_SET(GEN10_SAMPLER_MODE, ENABLE_SMALLPL, XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("FtrPerCtxtPreemptionGranularityControl"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1250), ENGINE_CLASS(RENDER)), + XE_RTP_SET(GEN7_FF_SLICE_CS_CHICKEN1, GEN9_FFSC_PERCTX_PREEMPT_CTRL, + XE_RTP_FLAG(MASKED_REG)) + }, + {} +}; + +static const struct xe_rtp_entry lrc_was[] = { + { XE_RTP_NAME("1409342910, 14010698770, 14010443199, 1408979724, 1409178076, 1409207793, 1409217633, 1409252684, 1409347922, 1409142259"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)), + XE_RTP_SET(GEN11_COMMON_SLICE_CHICKEN3, + GEN12_DISABLE_CPS_AWARE_COLOR_PIPE, + XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("WaDisableGPGPUMidThreadPreemption"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)), + XE_RTP_FIELD_SET(GEN8_CS_CHICKEN1, GEN9_PREEMPT_GPGPU_LEVEL_MASK, + GEN9_PREEMPT_GPGPU_THREAD_GROUP_LEVEL, + XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("16011163337"), + XE_RTP_RULES(GRAPHICS_VERSION_RANGE(1200, 1210)), + /* read verification is ignored due to 1608008084. */ + XE_RTP_FIELD_SET_NO_READ_MASK(GEN12_FF_MODE2, FF_MODE2_GS_TIMER_MASK, + FF_MODE2_GS_TIMER_224) + }, + { XE_RTP_NAME("1409044764"), + XE_RTP_RULES(PLATFORM(DG1)), + XE_RTP_CLR(GEN11_COMMON_SLICE_CHICKEN3, + DG1_FLOAT_POINT_BLEND_OPT_STRICT_MODE_EN, + XE_RTP_FLAG(MASKED_REG)) + }, + { XE_RTP_NAME("22010493298"), + XE_RTP_RULES(PLATFORM(DG1)), + XE_RTP_SET(HIZ_CHICKEN, + DG1_HZ_READ_SUPPRESSION_OPTIMIZATION_DISABLE, + XE_RTP_FLAG(MASKED_REG)) + }, + {} +}; + +/** + * xe_wa_process_gt - process GT workaround table + * @gt: GT instance to process workarounds for + * + * Process GT workaround table for this platform, saving in @gt all the + * workarounds that need to be applied at the GT level. + */ +void xe_wa_process_gt(struct xe_gt *gt) +{ + xe_rtp_process(gt_was, >->reg_sr, gt, NULL); +} + +/** + * xe_wa_process_engine - process engine workaround table + * @hwe: engine instance to process workarounds for + * + * Process engine workaround table for this platform, saving in @hwe all the + * workarounds that need to be applied at the engine level that match this + * engine. + */ +void xe_wa_process_engine(struct xe_hw_engine *hwe) +{ + xe_rtp_process(engine_was, &hwe->reg_sr, hwe->gt, hwe); +} + +/** + * xe_wa_process_lrc - process context workaround table + * @hwe: engine instance to process workarounds for + * + * Process context workaround table for this platform, saving in @hwe all the + * workarounds that need to be applied on context restore. These are workarounds + * touching registers that are part of the HW context image. + */ +void xe_wa_process_lrc(struct xe_hw_engine *hwe) +{ + xe_rtp_process(lrc_was, &hwe->reg_lrc, hwe->gt, hwe); +} diff --git a/drivers/gpu/drm/xe/xe_wa.h b/drivers/gpu/drm/xe/xe_wa.h new file mode 100644 index 000000000000..cd2307d58795 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wa.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_WA_ +#define _XE_WA_ + +struct xe_gt; +struct xe_hw_engine; + +void xe_wa_process_gt(struct xe_gt *gt); +void xe_wa_process_engine(struct xe_hw_engine *hwe); +void xe_wa_process_lrc(struct xe_hw_engine *hwe); + +void xe_reg_whitelist_process_engine(struct xe_hw_engine *hwe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.c b/drivers/gpu/drm/xe/xe_wait_user_fence.c new file mode 100644 index 000000000000..8a8d814a0e7a --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include +#include +#include + +#include "xe_device.h" +#include "xe_gt.h" +#include "xe_macros.h" +#include "xe_vm.h" + +static int do_compare(u64 addr, u64 value, u64 mask, u16 op) +{ + u64 rvalue; + int err; + bool passed; + + err = copy_from_user(&rvalue, u64_to_user_ptr(addr), sizeof(rvalue)); + if (err) + return -EFAULT; + + switch (op) { + case DRM_XE_UFENCE_WAIT_EQ: + passed = (rvalue & mask) == (value & mask); + break; + case DRM_XE_UFENCE_WAIT_NEQ: + passed = (rvalue & mask) != (value & mask); + break; + case DRM_XE_UFENCE_WAIT_GT: + passed = (rvalue & mask) > (value & mask); + break; + case DRM_XE_UFENCE_WAIT_GTE: + passed = (rvalue & mask) >= (value & mask); + break; + case DRM_XE_UFENCE_WAIT_LT: + passed = (rvalue & mask) < (value & mask); + break; + case DRM_XE_UFENCE_WAIT_LTE: + passed = (rvalue & mask) <= (value & mask); + break; + default: + XE_BUG_ON("Not possible"); + } + + return passed ? 0 : 1; +} + +static const enum xe_engine_class user_to_xe_engine_class[] = { + [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, + [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, + [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, + [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, + [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, +}; + +int check_hw_engines(struct xe_device *xe, + struct drm_xe_engine_class_instance *eci, + int num_engines) +{ + int i; + + for (i = 0; i < num_engines; ++i) { + enum xe_engine_class user_class = + user_to_xe_engine_class[eci[i].engine_class]; + + if (eci[i].gt_id >= xe->info.tile_count) + return -EINVAL; + + if (!xe_gt_hw_engine(xe_device_get_gt(xe, eci[i].gt_id), + user_class, eci[i].engine_instance, true)) + return -EINVAL; + } + + return 0; +} + +#define VALID_FLAGS (DRM_XE_UFENCE_WAIT_SOFT_OP | \ + DRM_XE_UFENCE_WAIT_ABSTIME | \ + DRM_XE_UFENCE_WAIT_VM_ERROR) +#define MAX_OP DRM_XE_UFENCE_WAIT_LTE + +int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + DEFINE_WAIT_FUNC(w_wait, woken_wake_function); + struct drm_xe_wait_user_fence *args = data; + struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; + struct drm_xe_engine_class_instance __user *user_eci = + u64_to_user_ptr(args->instances); + struct xe_vm *vm = NULL; + u64 addr = args->addr; + int err; + bool no_engines = args->flags & DRM_XE_UFENCE_WAIT_SOFT_OP || + args->flags & DRM_XE_UFENCE_WAIT_VM_ERROR; + unsigned long timeout = args->timeout; + + if (XE_IOCTL_ERR(xe, args->extensions)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->flags & ~VALID_FLAGS)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, args->op > MAX_OP)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, no_engines && + (args->num_engines || args->instances))) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, !no_engines && !args->num_engines)) + return -EINVAL; + + if (XE_IOCTL_ERR(xe, !(args->flags & DRM_XE_UFENCE_WAIT_VM_ERROR) && + addr & 0x7)) + return -EINVAL; + + if (!no_engines) { + err = copy_from_user(eci, user_eci, + sizeof(struct drm_xe_engine_class_instance) * + args->num_engines); + if (XE_IOCTL_ERR(xe, err)) + return -EFAULT; + + if (XE_IOCTL_ERR(xe, check_hw_engines(xe, eci, + args->num_engines))) + return -EINVAL; + } + + if (args->flags & DRM_XE_UFENCE_WAIT_VM_ERROR) { + if (XE_IOCTL_ERR(xe, args->vm_id >> 32)) + return -EINVAL; + + vm = xe_vm_lookup(to_xe_file(file), args->vm_id); + if (XE_IOCTL_ERR(xe, !vm)) + return -ENOENT; + + if (XE_IOCTL_ERR(xe, !vm->async_ops.error_capture.addr)) { + xe_vm_put(vm); + return -ENOTSUPP; + } + + addr = vm->async_ops.error_capture.addr; + } + + if (XE_IOCTL_ERR(xe, timeout > MAX_SCHEDULE_TIMEOUT)) + return -EINVAL; + + /* + * FIXME: Very simple implementation at the moment, single wait queue + * for everything. Could be optimized to have a wait queue for every + * hardware engine. Open coding as 'do_compare' can sleep which doesn't + * work with the wait_event_* macros. + */ + if (vm) + add_wait_queue(&vm->async_ops.error_capture.wq, &w_wait); + else + add_wait_queue(&xe->ufence_wq, &w_wait); + for (;;) { + if (vm && xe_vm_is_closed(vm)) { + err = -ENODEV; + break; + } + err = do_compare(addr, args->value, args->mask, args->op); + if (err <= 0) + break; + + if (signal_pending(current)) { + err = -ERESTARTSYS; + break; + } + + if (!timeout) { + err = -ETIME; + break; + } + + timeout = wait_woken(&w_wait, TASK_INTERRUPTIBLE, timeout); + } + if (vm) { + remove_wait_queue(&vm->async_ops.error_capture.wq, &w_wait); + xe_vm_put(vm); + } else { + remove_wait_queue(&xe->ufence_wq, &w_wait); + } + if (XE_IOCTL_ERR(xe, err < 0)) + return err; + else if (XE_IOCTL_ERR(xe, !timeout)) + return -ETIME; + + /* + * Again very simple, return the time in jiffies that has past, may need + * a more precision + */ + if (args->flags & DRM_XE_UFENCE_WAIT_ABSTIME) + args->timeout = args->timeout - timeout; + + return 0; +} diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.h b/drivers/gpu/drm/xe/xe_wait_user_fence.h new file mode 100644 index 000000000000..0e268978f9e6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_WAIT_USER_FENCE_H_ +#define _XE_WAIT_USER_FENCE_H_ + +struct drm_device; +struct drm_file; + +int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); + +#endif diff --git a/drivers/gpu/drm/xe/xe_wopcm.c b/drivers/gpu/drm/xe/xe_wopcm.c new file mode 100644 index 000000000000..e4a8d4a1899e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wopcm.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2022 Intel Corporation + */ + +#include "xe_device.h" +#include "xe_force_wake.h" +#include "xe_gt.h" +#include "xe_guc_reg.h" +#include "xe_mmio.h" +#include "xe_uc_fw.h" +#include "xe_wopcm.h" + +#include "i915_utils.h" + +/** + * DOC: Write Once Protected Content Memory (WOPCM) Layout + * + * The layout of the WOPCM will be fixed after writing to GuC WOPCM size and + * offset registers whose values are calculated and determined by HuC/GuC + * firmware size and set of hardware requirements/restrictions as shown below: + * + * :: + * + * +=========> +====================+ <== WOPCM Top + * ^ | HW contexts RSVD | + * | +===> +====================+ <== GuC WOPCM Top + * | ^ | | + * | | | | + * | | | | + * | GuC | | + * | WOPCM | | + * | Size +--------------------+ + * WOPCM | | GuC FW RSVD | + * | | +--------------------+ + * | | | GuC Stack RSVD | + * | | +------------------- + + * | v | GuC WOPCM RSVD | + * | +===> +====================+ <== GuC WOPCM base + * | | WOPCM RSVD | + * | +------------------- + <== HuC Firmware Top + * v | HuC FW | + * +=========> +====================+ <== WOPCM Base + * + * GuC accessible WOPCM starts at GuC WOPCM base and ends at GuC WOPCM top. + * The top part of the WOPCM is reserved for hardware contexts (e.g. RC6 + * context). + */ + +/* Default WOPCM size is 2MB from Gen11, 1MB on previous platforms */ +#define DGFX_WOPCM_SIZE SZ_4M /* FIXME: Larger size require + for 2 tile PVC, do a proper + probe sooner or later */ +#define MTL_WOPCM_SIZE SZ_4M /* FIXME: Larger size require + for MTL, do a proper probe + sooner or later */ +#define GEN11_WOPCM_SIZE SZ_2M +/* 16KB WOPCM (RSVD WOPCM) is reserved from HuC firmware top. */ +#define WOPCM_RESERVED_SIZE SZ_16K + +/* 16KB reserved at the beginning of GuC WOPCM. */ +#define GUC_WOPCM_RESERVED SZ_16K +/* 8KB from GUC_WOPCM_RESERVED is reserved for GuC stack. */ +#define GUC_WOPCM_STACK_RESERVED SZ_8K + +/* GuC WOPCM Offset value needs to be aligned to 16KB. */ +#define GUC_WOPCM_OFFSET_ALIGNMENT (1UL << GUC_WOPCM_OFFSET_SHIFT) + +/* 36KB WOPCM reserved at the end of WOPCM on GEN11. */ +#define GEN11_WOPCM_HW_CTX_RESERVED (SZ_32K + SZ_4K) + +static inline struct xe_gt *wopcm_to_gt(struct xe_wopcm *wopcm) +{ + return container_of(wopcm, struct xe_gt, uc.wopcm); +} + +static inline struct xe_device *wopcm_to_xe(struct xe_wopcm *wopcm) +{ + return gt_to_xe(wopcm_to_gt(wopcm)); +} + +static u32 context_reserved_size(void) +{ + return GEN11_WOPCM_HW_CTX_RESERVED; +} + +static bool __check_layout(struct xe_device *xe, u32 wopcm_size, + u32 guc_wopcm_base, u32 guc_wopcm_size, + u32 guc_fw_size, u32 huc_fw_size) +{ + const u32 ctx_rsvd = context_reserved_size(); + u32 size; + + size = wopcm_size - ctx_rsvd; + if (unlikely(range_overflows(guc_wopcm_base, guc_wopcm_size, size))) { + drm_err(&xe->drm, + "WOPCM: invalid GuC region layout: %uK + %uK > %uK\n", + guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K, + size / SZ_1K); + return false; + } + + size = guc_fw_size + GUC_WOPCM_RESERVED + GUC_WOPCM_STACK_RESERVED; + if (unlikely(guc_wopcm_size < size)) { + drm_err(&xe->drm, "WOPCM: no space for %s: %uK < %uK\n", + xe_uc_fw_type_repr(XE_UC_FW_TYPE_GUC), + guc_wopcm_size / SZ_1K, size / SZ_1K); + return false; + } + + size = huc_fw_size + WOPCM_RESERVED_SIZE; + if (unlikely(guc_wopcm_base < size)) { + drm_err(&xe->drm, "WOPCM: no space for %s: %uK < %uK\n", + xe_uc_fw_type_repr(XE_UC_FW_TYPE_HUC), + guc_wopcm_base / SZ_1K, size / SZ_1K); + return false; + } + + return true; +} + +static bool __wopcm_regs_locked(struct xe_gt *gt, + u32 *guc_wopcm_base, u32 *guc_wopcm_size) +{ + u32 reg_base = xe_mmio_read32(gt, DMA_GUC_WOPCM_OFFSET.reg); + u32 reg_size = xe_mmio_read32(gt, GUC_WOPCM_SIZE.reg); + + if (!(reg_size & GUC_WOPCM_SIZE_LOCKED) || + !(reg_base & GUC_WOPCM_OFFSET_VALID)) + return false; + + *guc_wopcm_base = reg_base & GUC_WOPCM_OFFSET_MASK; + *guc_wopcm_size = reg_size & GUC_WOPCM_SIZE_MASK; + return true; +} + +static int __wopcm_init_regs(struct xe_device *xe, struct xe_gt *gt, + struct xe_wopcm *wopcm) +{ + u32 base = wopcm->guc.base; + u32 size = wopcm->guc.size; + u32 huc_agent = xe_uc_fw_is_disabled(>->uc.huc.fw) ? 0 : + HUC_LOADING_AGENT_GUC; + u32 mask; + int err; + + XE_BUG_ON(!(base & GUC_WOPCM_OFFSET_MASK)); + XE_BUG_ON(base & ~GUC_WOPCM_OFFSET_MASK); + XE_BUG_ON(!(size & GUC_WOPCM_SIZE_MASK)); + XE_BUG_ON(size & ~GUC_WOPCM_SIZE_MASK); + + mask = GUC_WOPCM_SIZE_MASK | GUC_WOPCM_SIZE_LOCKED; + err = xe_mmio_write32_and_verify(gt, GUC_WOPCM_SIZE.reg, size, mask, + size | GUC_WOPCM_SIZE_LOCKED); + if (err) + goto err_out; + + mask = GUC_WOPCM_OFFSET_MASK | GUC_WOPCM_OFFSET_VALID | huc_agent; + err = xe_mmio_write32_and_verify(gt, DMA_GUC_WOPCM_OFFSET.reg, + base | huc_agent, mask, + base | huc_agent | + GUC_WOPCM_OFFSET_VALID); + if (err) + goto err_out; + + return 0; + +err_out: + drm_notice(&xe->drm, "Failed to init uC WOPCM registers!\n"); + drm_notice(&xe->drm, "%s(%#x)=%#x\n", "DMA_GUC_WOPCM_OFFSET", + DMA_GUC_WOPCM_OFFSET.reg, + xe_mmio_read32(gt, DMA_GUC_WOPCM_OFFSET.reg)); + drm_notice(&xe->drm, "%s(%#x)=%#x\n", "GUC_WOPCM_SIZE", + GUC_WOPCM_SIZE.reg, + xe_mmio_read32(gt, GUC_WOPCM_SIZE.reg)); + + return err; +} + +u32 xe_wopcm_size(struct xe_device *xe) +{ + return IS_DGFX(xe) ? DGFX_WOPCM_SIZE : + xe->info.platform == XE_METEORLAKE ? MTL_WOPCM_SIZE : + GEN11_WOPCM_SIZE; +} + +/** + * xe_wopcm_init() - Initialize the WOPCM structure. + * @wopcm: pointer to xe_wopcm. + * + * This function will partition WOPCM space based on GuC and HuC firmware sizes + * and will allocate max remaining for use by GuC. This function will also + * enforce platform dependent hardware restrictions on GuC WOPCM offset and + * size. It will fail the WOPCM init if any of these checks fail, so that the + * following WOPCM registers setup and GuC firmware uploading would be aborted. + */ +int xe_wopcm_init(struct xe_wopcm *wopcm) +{ + struct xe_device *xe = wopcm_to_xe(wopcm); + struct xe_gt *gt = wopcm_to_gt(wopcm); + u32 guc_fw_size = xe_uc_fw_get_upload_size(>->uc.guc.fw); + u32 huc_fw_size = xe_uc_fw_get_upload_size(>->uc.huc.fw); + u32 ctx_rsvd = context_reserved_size(); + u32 guc_wopcm_base; + u32 guc_wopcm_size; + bool locked; + int ret = 0; + + if (!guc_fw_size) + return -EINVAL; + + wopcm->size = xe_wopcm_size(xe); + drm_dbg(&xe->drm, "WOPCM: %uK\n", wopcm->size / SZ_1K); + + xe_force_wake_assert_held(gt_to_fw(gt), XE_FW_GT); + XE_BUG_ON(guc_fw_size >= wopcm->size); + XE_BUG_ON(huc_fw_size >= wopcm->size); + XE_BUG_ON(ctx_rsvd + WOPCM_RESERVED_SIZE >= wopcm->size); + + locked = __wopcm_regs_locked(gt, &guc_wopcm_base, &guc_wopcm_size); + if (locked) { + drm_dbg(&xe->drm, "GuC WOPCM is already locked [%uK, %uK)\n", + guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K); + goto check; + } + + /* + * Aligned value of guc_wopcm_base will determine available WOPCM space + * for HuC firmware and mandatory reserved area. + */ + guc_wopcm_base = huc_fw_size + WOPCM_RESERVED_SIZE; + guc_wopcm_base = ALIGN(guc_wopcm_base, GUC_WOPCM_OFFSET_ALIGNMENT); + + /* + * Need to clamp guc_wopcm_base now to make sure the following math is + * correct. Formal check of whole WOPCM layout will be done below. + */ + guc_wopcm_base = min(guc_wopcm_base, wopcm->size - ctx_rsvd); + + /* Aligned remainings of usable WOPCM space can be assigned to GuC. */ + guc_wopcm_size = wopcm->size - ctx_rsvd - guc_wopcm_base; + guc_wopcm_size &= GUC_WOPCM_SIZE_MASK; + + drm_dbg(&xe->drm, "Calculated GuC WOPCM [%uK, %uK)\n", + guc_wopcm_base / SZ_1K, guc_wopcm_size / SZ_1K); + +check: + if (__check_layout(xe, wopcm->size, guc_wopcm_base, guc_wopcm_size, + guc_fw_size, huc_fw_size)) { + wopcm->guc.base = guc_wopcm_base; + wopcm->guc.size = guc_wopcm_size; + XE_BUG_ON(!wopcm->guc.base); + XE_BUG_ON(!wopcm->guc.size); + } else { + drm_notice(&xe->drm, "Unsuccessful WOPCM partitioning\n"); + return -E2BIG; + } + + if (!locked) + ret = __wopcm_init_regs(xe, gt, wopcm); + + return ret; +} diff --git a/drivers/gpu/drm/xe/xe_wopcm.h b/drivers/gpu/drm/xe/xe_wopcm.h new file mode 100644 index 000000000000..0197a282460b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wopcm.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_WOPCM_H_ +#define _XE_WOPCM_H_ + +#include "xe_wopcm_types.h" + +struct xe_device; + +int xe_wopcm_init(struct xe_wopcm *wopcm); +u32 xe_wopcm_size(struct xe_device *xe); + +#endif diff --git a/drivers/gpu/drm/xe/xe_wopcm_types.h b/drivers/gpu/drm/xe/xe_wopcm_types.h new file mode 100644 index 000000000000..486d850c4084 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_wopcm_types.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_WOPCM_TYPES_H_ +#define _XE_WOPCM_TYPES_H_ + +#include + +/** + * struct xe_wopcm - Overall WOPCM info and WOPCM regions. + */ +struct xe_wopcm { + /** @size: Size of overall WOPCM */ + u32 size; + /** @guc: GuC WOPCM Region info */ + struct { + /** @base: GuC WOPCM base which is offset from WOPCM base */ + u32 base; + /** @size: Size of the GuC WOPCM region */ + u32 size; + } guc; +}; + +#endif diff --git a/include/drm/xe_pciids.h b/include/drm/xe_pciids.h new file mode 100644 index 000000000000..e539594ed939 --- /dev/null +++ b/include/drm/xe_pciids.h @@ -0,0 +1,195 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_PCIIDS_H_ +#define _XE_PCIIDS_H_ + +/* + * Lists below can be turned into initializers for a struct pci_device_id + * by defining INTEL_VGA_DEVICE: + * + * #define INTEL_VGA_DEVICE(id, info) { \ + * 0x8086, id, \ + * ~0, ~0, \ + * 0x030000, 0xff0000, \ + * (unsigned long) info } + * + * And then calling like: + * + * XE_TGL_12_GT1_IDS(INTEL_VGA_DEVICE, ## __VA_ARGS__) + * + * To turn them into something else, just provide a different macro passed as + * first argument. + */ + +/* TGL */ +#define XE_TGL_GT1_IDS(MACRO__, ...) \ + MACRO__(0x9A60, ## __VA_ARGS__), \ + MACRO__(0x9A68, ## __VA_ARGS__), \ + MACRO__(0x9A70, ## __VA_ARGS__) + +#define XE_TGL_GT2_IDS(MACRO__, ...) \ + MACRO__(0x9A40, ## __VA_ARGS__), \ + MACRO__(0x9A49, ## __VA_ARGS__), \ + MACRO__(0x9A59, ## __VA_ARGS__), \ + MACRO__(0x9A78, ## __VA_ARGS__), \ + MACRO__(0x9AC0, ## __VA_ARGS__), \ + MACRO__(0x9AC9, ## __VA_ARGS__), \ + MACRO__(0x9AD9, ## __VA_ARGS__), \ + MACRO__(0x9AF8, ## __VA_ARGS__) + +#define XE_TGL_IDS(MACRO__, ...) \ + XE_TGL_GT1_IDS(MACRO__, ...), \ + XE_TGL_GT2_IDS(MACRO__, ...) + +/* RKL */ +#define XE_RKL_IDS(MACRO__, ...) \ + MACRO__(0x4C80, ## __VA_ARGS__), \ + MACRO__(0x4C8A, ## __VA_ARGS__), \ + MACRO__(0x4C8B, ## __VA_ARGS__), \ + MACRO__(0x4C8C, ## __VA_ARGS__), \ + MACRO__(0x4C90, ## __VA_ARGS__), \ + MACRO__(0x4C9A, ## __VA_ARGS__) + +/* DG1 */ +#define XE_DG1_IDS(MACRO__, ...) \ + MACRO__(0x4905, ## __VA_ARGS__), \ + MACRO__(0x4906, ## __VA_ARGS__), \ + MACRO__(0x4907, ## __VA_ARGS__), \ + MACRO__(0x4908, ## __VA_ARGS__), \ + MACRO__(0x4909, ## __VA_ARGS__) + +/* ADL-S */ +#define XE_ADLS_IDS(MACRO__, ...) \ + MACRO__(0x4680, ## __VA_ARGS__), \ + MACRO__(0x4682, ## __VA_ARGS__), \ + MACRO__(0x4688, ## __VA_ARGS__), \ + MACRO__(0x468A, ## __VA_ARGS__), \ + MACRO__(0x4690, ## __VA_ARGS__), \ + MACRO__(0x4692, ## __VA_ARGS__), \ + MACRO__(0x4693, ## __VA_ARGS__) + +/* ADL-P */ +#define XE_ADLP_IDS(MACRO__, ...) \ + MACRO__(0x46A0, ## __VA_ARGS__), \ + MACRO__(0x46A1, ## __VA_ARGS__), \ + MACRO__(0x46A2, ## __VA_ARGS__), \ + MACRO__(0x46A3, ## __VA_ARGS__), \ + MACRO__(0x46A6, ## __VA_ARGS__), \ + MACRO__(0x46A8, ## __VA_ARGS__), \ + MACRO__(0x46AA, ## __VA_ARGS__), \ + MACRO__(0x462A, ## __VA_ARGS__), \ + MACRO__(0x4626, ## __VA_ARGS__), \ + MACRO__(0x4628, ## __VA_ARGS__), \ + MACRO__(0x46B0, ## __VA_ARGS__), \ + MACRO__(0x46B1, ## __VA_ARGS__), \ + MACRO__(0x46B2, ## __VA_ARGS__), \ + MACRO__(0x46B3, ## __VA_ARGS__), \ + MACRO__(0x46C0, ## __VA_ARGS__), \ + MACRO__(0x46C1, ## __VA_ARGS__), \ + MACRO__(0x46C2, ## __VA_ARGS__), \ + MACRO__(0x46C3, ## __VA_ARGS__) + +/* ADL-N */ +#define XE_ADLN_IDS(MACRO__, ...) \ + MACRO__(0x46D0, ## __VA_ARGS__), \ + MACRO__(0x46D1, ## __VA_ARGS__), \ + MACRO__(0x46D2, ## __VA_ARGS__) + +/* RPL-S */ +#define XE_RPLS_IDS(MACRO__, ...) \ + MACRO__(0xA780, ## __VA_ARGS__), \ + MACRO__(0xA781, ## __VA_ARGS__), \ + MACRO__(0xA782, ## __VA_ARGS__), \ + MACRO__(0xA783, ## __VA_ARGS__), \ + MACRO__(0xA788, ## __VA_ARGS__), \ + MACRO__(0xA789, ## __VA_ARGS__), \ + MACRO__(0xA78A, ## __VA_ARGS__), \ + MACRO__(0xA78B, ## __VA_ARGS__) + +/* RPL-U */ +#define XE_RPLU_IDS(MACRO__, ...) \ + MACRO__(0xA721, ## __VA_ARGS__), \ + MACRO__(0xA7A1, ## __VA_ARGS__), \ + MACRO__(0xA7A9, ## __VA_ARGS__) + +/* RPL-P */ +#define XE_RPLP_IDS(MACRO__, ...) \ + MACRO__(0xA720, ## __VA_ARGS__), \ + MACRO__(0xA7A0, ## __VA_ARGS__), \ + MACRO__(0xA7A8, ## __VA_ARGS__) + +/* DG2 */ +#define XE_DG2_G10_IDS(MACRO__, ...) \ + MACRO__(0x5690, ## __VA_ARGS__), \ + MACRO__(0x5691, ## __VA_ARGS__), \ + MACRO__(0x5692, ## __VA_ARGS__), \ + MACRO__(0x56A0, ## __VA_ARGS__), \ + MACRO__(0x56A1, ## __VA_ARGS__), \ + MACRO__(0x56A2, ## __VA_ARGS__) + +#define XE_DG2_G11_IDS(MACRO__, ...) \ + MACRO__(0x5693, ## __VA_ARGS__), \ + MACRO__(0x5694, ## __VA_ARGS__), \ + MACRO__(0x5695, ## __VA_ARGS__), \ + MACRO__(0x5698, ## __VA_ARGS__), \ + MACRO__(0x56A5, ## __VA_ARGS__), \ + MACRO__(0x56A6, ## __VA_ARGS__), \ + MACRO__(0x56B0, ## __VA_ARGS__), \ + MACRO__(0x56B1, ## __VA_ARGS__) + +#define XE_DG2_G12_IDS(MACRO__, ...) \ + MACRO__(0x5696, ## __VA_ARGS__), \ + MACRO__(0x5697, ## __VA_ARGS__), \ + MACRO__(0x56A3, ## __VA_ARGS__), \ + MACRO__(0x56A4, ## __VA_ARGS__), \ + MACRO__(0x56B2, ## __VA_ARGS__), \ + MACRO__(0x56B3, ## __VA_ARGS__) + +#define XE_DG2_IDS(MACRO__, ...) \ + XE_DG2_G10_IDS(MACRO__, ## __VA_ARGS__),\ + XE_DG2_G11_IDS(MACRO__, ## __VA_ARGS__),\ + XE_DG2_G12_IDS(MACRO__, ## __VA_ARGS__) + +#define XE_ATS_M150_IDS(MACRO__, ...) \ + MACRO__(0x56C0, ## __VA_ARGS__) + +#define XE_ATS_M75_IDS(MACRO__, ...) \ + MACRO__(0x56C1, ## __VA_ARGS__) + +#define XE_ATS_M_IDS(MACRO__, ...) \ + XE_ATS_M150_IDS(MACRO__, ## __VA_ARGS__),\ + XE_ATS_M75_IDS(MACRO__, ## __VA_ARGS__) + +/* MTL */ +#define XE_MTL_M_IDS(MACRO__, ...) \ + MACRO__(0x7D40, ## __VA_ARGS__), \ + MACRO__(0x7D43, ## __VA_ARGS__), \ + MACRO__(0x7DC0, ## __VA_ARGS__) + +#define XE_MTL_P_IDS(MACRO__, ...) \ + MACRO__(0x7D45, ## __VA_ARGS__), \ + MACRO__(0x7D47, ## __VA_ARGS__), \ + MACRO__(0x7D50, ## __VA_ARGS__), \ + MACRO__(0x7D55, ## __VA_ARGS__), \ + MACRO__(0x7DC5, ## __VA_ARGS__), \ + MACRO__(0x7DD0, ## __VA_ARGS__), \ + MACRO__(0x7DD5, ## __VA_ARGS__) + +#define XE_MTL_S_IDS(MACRO__, ...) \ + MACRO__(0x7D60, ## __VA_ARGS__), \ + MACRO__(0x7DE0, ## __VA_ARGS__) + +#define XE_ARL_IDS(MACRO__, ...) \ + MACRO__(0x7D66, ## __VA_ARGS__), \ + MACRO__(0x7D76, ## __VA_ARGS__) + +#define XE_MTL_IDS(MACRO__, ...) \ + XE_MTL_M_IDS(MACRO__, ## __VA_ARGS__), \ + XE_MTL_P_IDS(MACRO__, ## __VA_ARGS__), \ + XE_MTL_S_IDS(MACRO__, ## __VA_ARGS__), \ + XE_ARL_IDS(MACRO__, ## __VA_ARGS__) + +#endif diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h new file mode 100644 index 000000000000..f64b1c785fad --- /dev/null +++ b/include/uapi/drm/xe_drm.h @@ -0,0 +1,787 @@ +/* + * Copyright 2021 Intel Corporation. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the + * "Software"), to deal in the Software without restriction, including + * without limitation the rights to use, copy, modify, merge, publish, + * distribute, sub license, and/or sell copies of the Software, and to + * permit persons to whom the Software is furnished to do so, subject to + * the following conditions: + * + * The above copyright notice and this permission notice (including the + * next paragraph) shall be included in all copies or substantial portions + * of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS + * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. + * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR + * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, + * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE + * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + * + */ + +#ifndef _UAPI_XE_DRM_H_ +#define _UAPI_XE_DRM_H_ + +#include "drm.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +/* Please note that modifications to all structs defined here are + * subject to backwards-compatibility constraints. + */ + +/** + * struct i915_user_extension - Base class for defining a chain of extensions + * + * Many interfaces need to grow over time. In most cases we can simply + * extend the struct and have userspace pass in more data. Another option, + * as demonstrated by Vulkan's approach to providing extensions for forward + * and backward compatibility, is to use a list of optional structs to + * provide those extra details. + * + * The key advantage to using an extension chain is that it allows us to + * redefine the interface more easily than an ever growing struct of + * increasing complexity, and for large parts of that interface to be + * entirely optional. The downside is more pointer chasing; chasing across + * the __user boundary with pointers encapsulated inside u64. + * + * Example chaining: + * + * .. code-block:: C + * + * struct i915_user_extension ext3 { + * .next_extension = 0, // end + * .name = ..., + * }; + * struct i915_user_extension ext2 { + * .next_extension = (uintptr_t)&ext3, + * .name = ..., + * }; + * struct i915_user_extension ext1 { + * .next_extension = (uintptr_t)&ext2, + * .name = ..., + * }; + * + * Typically the struct i915_user_extension would be embedded in some uAPI + * struct, and in this case we would feed it the head of the chain(i.e ext1), + * which would then apply all of the above extensions. + * + */ +struct xe_user_extension { + /** + * @next_extension: + * + * Pointer to the next struct i915_user_extension, or zero if the end. + */ + __u64 next_extension; + /** + * @name: Name of the extension. + * + * Note that the name here is just some integer. + * + * Also note that the name space for this is not global for the whole + * driver, but rather its scope/meaning is limited to the specific piece + * of uAPI which has embedded the struct i915_user_extension. + */ + __u32 name; + /** + * @flags: MBZ + * + * All undefined bits must be zero. + */ + __u32 pad; +}; + +/* + * i915 specific ioctls. + * + * The device specific ioctl range is [DRM_COMMAND_BASE, DRM_COMMAND_END) ie + * [0x40, 0xa0) (a0 is excluded). The numbers below are defined as offset + * against DRM_COMMAND_BASE and should be between [0x0, 0x60). + */ +#define DRM_XE_DEVICE_QUERY 0x00 +#define DRM_XE_GEM_CREATE 0x01 +#define DRM_XE_GEM_MMAP_OFFSET 0x02 +#define DRM_XE_VM_CREATE 0x03 +#define DRM_XE_VM_DESTROY 0x04 +#define DRM_XE_VM_BIND 0x05 +#define DRM_XE_ENGINE_CREATE 0x06 +#define DRM_XE_ENGINE_DESTROY 0x07 +#define DRM_XE_EXEC 0x08 +#define DRM_XE_MMIO 0x09 +#define DRM_XE_ENGINE_SET_PROPERTY 0x0a +#define DRM_XE_WAIT_USER_FENCE 0x0b +#define DRM_XE_VM_MADVISE 0x0c + +/* Must be kept compact -- no holes */ +#define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query) +#define DRM_IOCTL_XE_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_CREATE, struct drm_xe_gem_create) +#define DRM_IOCTL_XE_GEM_MMAP_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_MMAP_OFFSET, struct drm_xe_gem_mmap_offset) +#define DRM_IOCTL_XE_VM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_CREATE, struct drm_xe_vm_create) +#define DRM_IOCTL_XE_VM_DESTROY DRM_IOW( DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) +#define DRM_IOCTL_XE_VM_BIND DRM_IOW( DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) +#define DRM_IOCTL_XE_ENGINE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_ENGINE_CREATE, struct drm_xe_engine_create) +#define DRM_IOCTL_XE_ENGINE_DESTROY DRM_IOW( DRM_COMMAND_BASE + DRM_XE_ENGINE_DESTROY, struct drm_xe_engine_destroy) +#define DRM_IOCTL_XE_EXEC DRM_IOW( DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) +#define DRM_IOCTL_XE_MMIO DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_MMIO, struct drm_xe_mmio) +#define DRM_IOCTL_XE_ENGINE_SET_PROPERTY DRM_IOW( DRM_COMMAND_BASE + DRM_XE_ENGINE_SET_PROPERTY, struct drm_xe_engine_set_property) +#define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) +#define DRM_IOCTL_XE_VM_MADVISE DRM_IOW( DRM_COMMAND_BASE + DRM_XE_VM_MADVISE, struct drm_xe_vm_madvise) + +struct drm_xe_engine_class_instance { + __u16 engine_class; + +#define DRM_XE_ENGINE_CLASS_RENDER 0 +#define DRM_XE_ENGINE_CLASS_COPY 1 +#define DRM_XE_ENGINE_CLASS_VIDEO_DECODE 2 +#define DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE 3 +#define DRM_XE_ENGINE_CLASS_COMPUTE 4 + /* + * Kernel only class (not actual hardware engine class). Used for + * creating ordered queues of VM bind operations. + */ +#define DRM_XE_ENGINE_CLASS_VM_BIND 5 + + __u16 engine_instance; + __u16 gt_id; +}; + +#define XE_MEM_REGION_CLASS_SYSMEM 0 +#define XE_MEM_REGION_CLASS_VRAM 1 + +struct drm_xe_query_mem_usage { + __u32 num_regions; + __u32 pad; + + struct drm_xe_query_mem_region { + __u16 mem_class; + __u16 instance; /* unique ID even among different classes */ + __u32 pad; + __u32 min_page_size; + __u32 max_page_size; + __u64 total_size; + __u64 used; + __u64 reserved[8]; + } regions[]; +}; + +struct drm_xe_query_config { + __u32 num_params; + __u32 pad; +#define XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 +#define XE_QUERY_CONFIG_FLAGS 1 + #define XE_QUERY_CONFIG_FLAGS_HAS_VRAM (0x1 << 0) + #define XE_QUERY_CONFIG_FLAGS_USE_GUC (0x1 << 1) +#define XE_QUERY_CONFIG_MIN_ALIGNEMENT 2 +#define XE_QUERY_CONFIG_VA_BITS 3 +#define XE_QUERY_CONFIG_GT_COUNT 4 +#define XE_QUERY_CONFIG_MEM_REGION_COUNT 5 +#define XE_QUERY_CONFIG_NUM_PARAM XE_QUERY_CONFIG_MEM_REGION_COUNT + 1 + __u64 info[]; +}; + +struct drm_xe_query_gts { + __u32 num_gt; + __u32 pad; + + /* + * TODO: Perhaps info about every mem region relative to this GT? e.g. + * bandwidth between this GT and remote region? + */ + + struct drm_xe_query_gt { +#define XE_QUERY_GT_TYPE_MAIN 0 +#define XE_QUERY_GT_TYPE_REMOTE 1 +#define XE_QUERY_GT_TYPE_MEDIA 2 + __u16 type; + __u16 instance; + __u32 clock_freq; + __u64 features; + __u64 native_mem_regions; /* bit mask of instances from drm_xe_query_mem_usage */ + __u64 slow_mem_regions; /* bit mask of instances from drm_xe_query_mem_usage */ + __u64 inaccessible_mem_regions; /* bit mask of instances from drm_xe_query_mem_usage */ + __u64 reserved[8]; + } gts[]; +}; + +struct drm_xe_query_topology_mask { + /** @gt_id: GT ID the mask is associated with */ + __u16 gt_id; + + /** @type: type of mask */ + __u16 type; +#define XE_TOPO_DSS_GEOMETRY (1 << 0) +#define XE_TOPO_DSS_COMPUTE (1 << 1) +#define XE_TOPO_EU_PER_DSS (1 << 2) + + /** @num_bytes: number of bytes in requested mask */ + __u32 num_bytes; + + /** @mask: little-endian mask of @num_bytes */ + __u8 mask[]; +}; + +struct drm_xe_device_query { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @query: The type of data to query */ + __u32 query; + +#define DRM_XE_DEVICE_QUERY_ENGINES 0 +#define DRM_XE_DEVICE_QUERY_MEM_USAGE 1 +#define DRM_XE_DEVICE_QUERY_CONFIG 2 +#define DRM_XE_DEVICE_QUERY_GTS 3 +#define DRM_XE_DEVICE_QUERY_HWCONFIG 4 +#define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5 + + /** @size: Size of the queried data */ + __u32 size; + + /** @data: Queried data is placed here */ + __u64 data; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_gem_create { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** + * @size: Requested size for the object + * + * The (page-aligned) allocated size for the object will be returned. + */ + __u64 size; + + /** + * @flags: Flags, currently a mask of memory instances of where BO can + * be placed + */ +#define XE_GEM_CREATE_FLAG_DEFER_BACKING (0x1 << 24) +#define XE_GEM_CREATE_FLAG_SCANOUT (0x1 << 25) + __u32 flags; + + /** + * @vm_id: Attached VM, if any + * + * If a VM is specified, this BO must: + * + * 1. Only ever be bound to that VM. + * + * 2. Cannot be exported as a PRIME fd. + */ + __u32 vm_id; + + /** + * @handle: Returned handle for the object. + * + * Object handles are nonzero. + */ + __u32 handle; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_gem_mmap_offset { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @handle: Handle for the object being mapped. */ + __u32 handle; + + /** @flags: Must be zero */ + __u32 flags; + + /** @offset: The fake offset to use for subsequent mmap call */ + __u64 offset; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +/** + * struct drm_xe_vm_bind_op_error_capture - format of VM bind op error capture + */ +struct drm_xe_vm_bind_op_error_capture { + /** @error: errno that occured */ + __s32 error; + /** @op: operation that encounter an error */ + __u32 op; + /** @addr: address of bind op */ + __u64 addr; + /** @size: size of bind */ + __u64 size; +}; + +/** struct drm_xe_ext_vm_set_property - VM set property extension */ +struct drm_xe_ext_vm_set_property { + /** @base: base user extension */ + struct xe_user_extension base; + + /** @property: property to set */ +#define XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS 0 + __u32 property; + + /** @value: property value */ + __u64 value; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_vm_create { + /** @extensions: Pointer to the first extension struct, if any */ +#define XE_VM_EXTENSION_SET_PROPERTY 0 + __u64 extensions; + + /** @flags: Flags */ + __u32 flags; + +#define DRM_XE_VM_CREATE_SCRATCH_PAGE (0x1 << 0) +#define DRM_XE_VM_CREATE_COMPUTE_MODE (0x1 << 1) +#define DRM_XE_VM_CREATE_ASYNC_BIND_OPS (0x1 << 2) +#define DRM_XE_VM_CREATE_FAULT_MODE (0x1 << 3) + + /** @vm_id: Returned VM ID */ + __u32 vm_id; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_vm_destroy { + /** @vm_id: VM ID */ + __u32 vm_id; + + /** @pad: MBZ */ + __u32 pad; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_vm_bind_op { + /** + * @obj: GEM object to operate on, MBZ for MAP_USERPTR, MBZ for UNMAP + */ + __u32 obj; + + union { + /** + * @obj_offset: Offset into the object, MBZ for CLEAR_RANGE, + * ignored for unbind + */ + __u64 obj_offset; + /** @userptr: user pointer to bind on */ + __u64 userptr; + }; + + /** + * @range: Number of bytes from the object to bind to addr, MBZ for UNMAP_ALL + */ + __u64 range; + + /** @addr: Address to operate on, MBZ for UNMAP_ALL */ + __u64 addr; + + /** + * @gt_mask: Mask for which GTs to create binds for, 0 == All GTs, + * only applies to creating new VMAs + */ + __u64 gt_mask; + + /** @op: Operation to perform (lower 16 bits) and flags (upper 16 bits) */ + __u32 op; + + /** @mem_region: Memory region to prefetch VMA to, instance not a mask */ + __u32 region; + +#define XE_VM_BIND_OP_MAP 0x0 +#define XE_VM_BIND_OP_UNMAP 0x1 +#define XE_VM_BIND_OP_MAP_USERPTR 0x2 +#define XE_VM_BIND_OP_RESTART 0x3 +#define XE_VM_BIND_OP_UNMAP_ALL 0x4 +#define XE_VM_BIND_OP_PREFETCH 0x5 + +#define XE_VM_BIND_FLAG_READONLY (0x1 << 16) + /* + * A bind ops completions are always async, hence the support for out + * sync. This flag indicates the allocation of the memory for new page + * tables and the job to program the pages tables is asynchronous + * relative to the IOCTL. That part of a bind operation can fail under + * memory pressure, the job in practice can't fail unless the system is + * totally shot. + * + * If this flag is clear and the IOCTL doesn't return an error, in + * practice the bind op is good and will complete. + * + * If this flag is set and doesn't return return an error, the bind op + * can still fail and recovery is needed. If configured, the bind op that + * caused the error will be captured in drm_xe_vm_bind_op_error_capture. + * Once the user sees the error (via a ufence + + * XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS), it should free memory + * via non-async unbinds, and then restart all queue'd async binds op via + * XE_VM_BIND_OP_RESTART. Or alternatively the user should destroy the + * VM. + * + * This flag is only allowed when DRM_XE_VM_CREATE_ASYNC_BIND_OPS is + * configured in the VM and must be set if the VM is configured with + * DRM_XE_VM_CREATE_ASYNC_BIND_OPS and not in an error state. + */ +#define XE_VM_BIND_FLAG_ASYNC (0x1 << 17) + /* + * Valid on a faulting VM only, do the MAP operation immediately rather + * than differing the MAP to the page fault handler. + */ +#define XE_VM_BIND_FLAG_IMMEDIATE (0x1 << 18) + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_vm_bind { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @vm_id: The ID of the VM to bind to */ + __u32 vm_id; + + /** + * @engine_id: engine_id, must be of class DRM_XE_ENGINE_CLASS_VM_BIND + * and engine must have same vm_id. If zero, the default VM bind engine + * is used. + */ + __u32 engine_id; + + /** @num_binds: number of binds in this IOCTL */ + __u32 num_binds; + + union { + /** @bind: used if num_binds == 1 */ + struct drm_xe_vm_bind_op bind; + /** + * @vector_of_binds: userptr to array of struct + * drm_xe_vm_bind_op if num_binds > 1 + */ + __u64 vector_of_binds; + }; + + /** @num_syncs: amount of syncs to wait on */ + __u32 num_syncs; + + /** @syncs: pointer to struct drm_xe_sync array */ + __u64 syncs; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +/** struct drm_xe_ext_engine_set_property - engine set property extension */ +struct drm_xe_ext_engine_set_property { + /** @base: base user extension */ + struct xe_user_extension base; + + /** @property: property to set */ + __u32 property; + + /** @value: property value */ + __u64 value; +}; + +/** + * struct drm_xe_engine_set_property - engine set property + * + * Same namespace for extensions as drm_xe_engine_create + */ +struct drm_xe_engine_set_property { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @engine_id: Engine ID */ + __u32 engine_id; + + /** @property: property to set */ +#define XE_ENGINE_PROPERTY_PRIORITY 0 +#define XE_ENGINE_PROPERTY_TIMESLICE 1 +#define XE_ENGINE_PROPERTY_PREEMPTION_TIMEOUT 2 + /* + * Long running or ULLS engine mode. DMA fences not allowed in this + * mode. Must match the value of DRM_XE_VM_CREATE_COMPUTE_MODE, serves + * as a sanity check the UMD knows what it is doing. Can only be set at + * engine create time. + */ +#define XE_ENGINE_PROPERTY_COMPUTE_MODE 3 +#define XE_ENGINE_PROPERTY_PERSISTENCE 4 +#define XE_ENGINE_PROPERTY_JOB_TIMEOUT 5 +#define XE_ENGINE_PROPERTY_ACC_TRIGGER 6 +#define XE_ENGINE_PROPERTY_ACC_NOTIFY 7 +#define XE_ENGINE_PROPERTY_ACC_GRANULARITY 8 + __u32 property; + + /** @value: property value */ + __u64 value; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_engine_create { + /** @extensions: Pointer to the first extension struct, if any */ +#define XE_ENGINE_EXTENSION_SET_PROPERTY 0 + __u64 extensions; + + /** @width: submission width (number BB per exec) for this engine */ + __u16 width; + + /** @num_placements: number of valid placements for this engine */ + __u16 num_placements; + + /** @vm_id: VM to use for this engine */ + __u32 vm_id; + + /** @flags: MBZ */ + __u32 flags; + + /** @engine_id: Returned engine ID */ + __u32 engine_id; + + /** + * @instances: user pointer to a 2-d array of struct + * drm_xe_engine_class_instance + * + * length = width (i) * num_placements (j) + * index = j + i * width + */ + __u64 instances; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_engine_destroy { + /** @vm_id: VM ID */ + __u32 engine_id; + + /** @pad: MBZ */ + __u32 pad; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_sync { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + __u32 flags; + +#define DRM_XE_SYNC_SYNCOBJ 0x0 +#define DRM_XE_SYNC_TIMELINE_SYNCOBJ 0x1 +#define DRM_XE_SYNC_DMA_BUF 0x2 +#define DRM_XE_SYNC_USER_FENCE 0x3 +#define DRM_XE_SYNC_SIGNAL 0x10 + + union { + __u32 handle; + /** + * @addr: Address of user fence. When sync passed in via exec + * IOCTL this a GPU address in the VM. When sync passed in via + * VM bind IOCTL this is a user pointer. In either case, it is + * the users responsibility that this address is present and + * mapped when the user fence is signalled. Must be qword + * aligned. + */ + __u64 addr; + }; + + __u64 timeline_value; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_exec { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @engine_id: Engine ID for the batch buffer */ + __u32 engine_id; + + /** @num_syncs: Amount of struct drm_xe_sync in array. */ + __u32 num_syncs; + + /** @syncs: Pointer to struct drm_xe_sync array. */ + __u64 syncs; + + /** + * @address: address of batch buffer if num_batch_buffer == 1 or an + * array of batch buffer addresses + */ + __u64 address; + + /** + * @num_batch_buffer: number of batch buffer in this exec, must match + * the width of the engine + */ + __u16 num_batch_buffer; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_mmio { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + __u32 addr; + + __u32 flags; + +#define DRM_XE_MMIO_8BIT 0x0 +#define DRM_XE_MMIO_16BIT 0x1 +#define DRM_XE_MMIO_32BIT 0x2 +#define DRM_XE_MMIO_64BIT 0x3 +#define DRM_XE_MMIO_BITS_MASK 0x3 +#define DRM_XE_MMIO_READ 0x4 +#define DRM_XE_MMIO_WRITE 0x8 + + __u64 value; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +/** + * struct drm_xe_wait_user_fence - wait user fence + * + * Wait on user fence, XE will wakeup on every HW engine interrupt in the + * instances list and check if user fence is complete: + * (*addr & MASK) OP (VALUE & MASK) + * + * Returns to user on user fence completion or timeout. + */ +struct drm_xe_wait_user_fence { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + union { + /** + * @addr: user pointer address to wait on, must qword aligned + */ + __u64 addr; + /** + * @vm_id: The ID of the VM which encounter an error used with + * DRM_XE_UFENCE_WAIT_VM_ERROR. Upper 32 bits must be clear. + */ + __u64 vm_id; + }; + /** @op: wait operation (type of comparison) */ +#define DRM_XE_UFENCE_WAIT_EQ 0 +#define DRM_XE_UFENCE_WAIT_NEQ 1 +#define DRM_XE_UFENCE_WAIT_GT 2 +#define DRM_XE_UFENCE_WAIT_GTE 3 +#define DRM_XE_UFENCE_WAIT_LT 4 +#define DRM_XE_UFENCE_WAIT_LTE 5 + __u16 op; + /** @flags: wait flags */ +#define DRM_XE_UFENCE_WAIT_SOFT_OP (1 << 0) /* e.g. Wait on VM bind */ +#define DRM_XE_UFENCE_WAIT_ABSTIME (1 << 1) +#define DRM_XE_UFENCE_WAIT_VM_ERROR (1 << 2) + __u16 flags; + /** @value: compare value */ + __u64 value; + /** @mask: comparison mask */ +#define DRM_XE_UFENCE_WAIT_U8 0xffu +#define DRM_XE_UFENCE_WAIT_U16 0xffffu +#define DRM_XE_UFENCE_WAIT_U32 0xffffffffu +#define DRM_XE_UFENCE_WAIT_U64 0xffffffffffffffffu + __u64 mask; + /** @timeout: how long to wait before bailing, value in jiffies */ + __s64 timeout; + /** + * @num_engines: number of engine instances to wait on, must be zero + * when DRM_XE_UFENCE_WAIT_SOFT_OP set + */ + __u64 num_engines; + /** + * @instances: user pointer to array of drm_xe_engine_class_instance to + * wait on, must be NULL when DRM_XE_UFENCE_WAIT_SOFT_OP set + */ + __u64 instances; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +struct drm_xe_vm_madvise { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @vm_id: The ID VM in which the VMA exists */ + __u32 vm_id; + + /** @range: Number of bytes in the VMA */ + __u64 range; + + /** @addr: Address of the VMA to operation on */ + __u64 addr; + + /* + * Setting the preferred location will trigger a migrate of the VMA + * backing store to new location if the backing store is already + * allocated. + */ +#define DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS 0 +#define DRM_XE_VM_MADVISE_PREFERRED_GT 1 + /* + * In this case lower 32 bits are mem class, upper 32 are GT. + * Combination provides a single IOCTL plus migrate VMA to preferred + * location. + */ +#define DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS_GT 2 + /* + * The CPU will do atomic memory operations to this VMA. Must be set on + * some devices for atomics to behave correctly. + */ +#define DRM_XE_VM_MADVISE_CPU_ATOMIC 3 + /* + * The device will do atomic memory operations to this VMA. Must be set + * on some devices for atomics to behave correctly. + */ +#define DRM_XE_VM_MADVISE_DEVICE_ATOMIC 4 + /* + * Priority WRT to eviction (moving from preferred memory location due + * to memory pressure). The lower the priority, the more likely to be + * evicted. + */ +#define DRM_XE_VM_MADVISE_PRIORITY 5 +#define DRM_XE_VMA_PRIORITY_LOW 0 +#define DRM_XE_VMA_PRIORITY_NORMAL 1 /* Default */ +#define DRM_XE_VMA_PRIORITY_HIGH 2 /* Must be elevated user */ + /* Pin the VMA in memory, must be elevated user */ +#define DRM_XE_VM_MADVISE_PIN 6 + + /** @property: property to set */ + __u32 property; + + /** @value: property value */ + __u64 value; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + +#if defined(__cplusplus) +} +#endif + +#endif /* _UAPI_XE_DRM_H_ */ -- cgit v1.2.3 From c4c5391adae2c5a328232bb4fecd9510310b2fdf Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 12 Dec 2023 20:39:25 -0800 Subject: drm/fourcc: fix spelling/typos Correct spelling mistakes that were identified by codespell. Signed-off-by: Randy Dunlap Cc: David Airlie Cc: Daniel Vetter Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Thomas Zimmermann Reviewed-by: Jani Nikula Signed-off-by: Maxime Ripard Link: https://patchwork.freedesktop.org/patch/msgid/20231213043925.13852-1-rdunlap@infradead.org --- include/uapi/drm/drm_fourcc.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/drm_fourcc.h b/include/uapi/drm/drm_fourcc.h index 3151f1fc7ebb..84d502e42961 100644 --- a/include/uapi/drm/drm_fourcc.h +++ b/include/uapi/drm/drm_fourcc.h @@ -54,7 +54,7 @@ extern "C" { * Format modifiers may change any property of the buffer, including the number * of planes and/or the required allocation size. Format modifiers are * vendor-namespaced, and as such the relationship between a fourcc code and a - * modifier is specific to the modifer being used. For example, some modifiers + * modifier is specific to the modifier being used. For example, some modifiers * may preserve meaning - such as number of planes - from the fourcc code, * whereas others may not. * @@ -79,7 +79,7 @@ extern "C" { * format. * - Higher-level programs interfacing with KMS/GBM/EGL/Vulkan/etc: these users * see modifiers as opaque tokens they can check for equality and intersect. - * These users musn't need to know to reason about the modifier value + * These users mustn't need to know to reason about the modifier value * (i.e. they are not expected to extract information out of the modifier). * * Vendors should document their modifier usage in as much detail as @@ -540,7 +540,7 @@ extern "C" { * This is a tiled layout using 4Kb tiles in row-major layout. * Within the tile pixels are laid out in 16 256 byte units / sub-tiles which * are arranged in four groups (two wide, two high) with column-major layout. - * Each group therefore consits out of four 256 byte units, which are also laid + * Each group therefore consists out of four 256 byte units, which are also laid * out as 2x2 column-major. * 256 byte units are made out of four 64 byte blocks of pixels, producing * either a square block or a 2:1 unit. @@ -1103,7 +1103,7 @@ drm_fourcc_canonicalize_nvidia_format_mod(__u64 modifier) */ /* - * The top 4 bits (out of the 56 bits alloted for specifying vendor specific + * The top 4 bits (out of the 56 bits allotted for specifying vendor specific * modifiers) denote the category for modifiers. Currently we have three * categories of modifiers ie AFBC, MISC and AFRC. We can have a maximum of * sixteen different categories. @@ -1419,7 +1419,7 @@ drm_fourcc_canonicalize_nvidia_format_mod(__u64 modifier) * Amlogic FBC Memory Saving mode * * Indicates the storage is packed when pixel size is multiple of word - * boudaries, i.e. 8bit should be stored in this mode to save allocation + * boundaries, i.e. 8bit should be stored in this mode to save allocation * memory. * * This mode reduces body layout to 3072 bytes per 64x32 superblock with -- cgit v1.2.3 From 9567dab3a8cb4dfc4b0382c2678ad01bff13a3bf Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 12 Dec 2023 20:41:07 -0800 Subject: drm/uapi: drm_mode.h: fix spellos and grammar Correct spellos reported by codespell. Fix some grammar (as 's' to a few words). Signed-off-by: Randy Dunlap Cc: David Airlie Cc: Daniel Vetter Cc: Maarten Lankhorst Cc: Maxime Ripard Cc: Thomas Zimmermann Reviewed-by: Jani Nikula Signed-off-by: Maxime Ripard Link: https://patchwork.freedesktop.org/patch/msgid/20231213044107.29214-1-rdunlap@infradead.org --- include/uapi/drm/drm_mode.h | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index 95630f170110..aee3d40c96f0 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -36,10 +36,10 @@ extern "C" { /** * DOC: overview * - * DRM exposes many UAPI and structure definition to have a consistent - * and standardized interface with user. + * DRM exposes many UAPI and structure definitions to have a consistent + * and standardized interface with users. * Userspace can refer to these structure definitions and UAPI formats - * to communicate to driver + * to communicate to drivers. */ #define DRM_CONNECTOR_NAME_LEN 32 @@ -540,7 +540,7 @@ struct drm_mode_get_connector { /* the PROP_ATOMIC flag is used to hide properties from userspace that * is not aware of atomic properties. This is mostly to work around * older userspace (DDX drivers) that read/write each prop they find, - * witout being aware that this could be triggering a lengthy modeset. + * without being aware that this could be triggering a lengthy modeset. */ #define DRM_MODE_PROP_ATOMIC 0x80000000 @@ -664,7 +664,7 @@ struct drm_mode_fb_cmd { }; #define DRM_MODE_FB_INTERLACED (1<<0) /* for interlaced framebuffers */ -#define DRM_MODE_FB_MODIFIERS (1<<1) /* enables ->modifer[] */ +#define DRM_MODE_FB_MODIFIERS (1<<1) /* enables ->modifier[] */ /** * struct drm_mode_fb_cmd2 - Frame-buffer metadata. @@ -881,8 +881,8 @@ struct hdr_metadata_infoframe { * These are coded as unsigned 16-bit values in units of * 0.00002, where 0x0000 represents zero and 0xC350 * represents 1.0000. - * @display_primaries.x: X cordinate of color primary. - * @display_primaries.y: Y cordinate of color primary. + * @display_primaries.x: X coordinate of color primary. + * @display_primaries.y: Y coordinate of color primary. */ struct { __u16 x, y; @@ -892,8 +892,8 @@ struct hdr_metadata_infoframe { * These are coded as unsigned 16-bit values in units of * 0.00002, where 0x0000 represents zero and 0xC350 * represents 1.0000. - * @white_point.x: X cordinate of whitepoint of color primary. - * @white_point.y: Y cordinate of whitepoint of color primary. + * @white_point.x: X coordinate of whitepoint of color primary. + * @white_point.y: Y coordinate of whitepoint of color primary. */ struct { __u16 x, y; -- cgit v1.2.3 From afa5cf3175a22b719a65fc0b13dbf78196a60869 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Tue, 12 Dec 2023 20:40:14 -0800 Subject: drm/i915/uapi: fix typos/spellos and punctuation Use "its" for possessive form instead of "it's". Hyphenate multi-word adjectives. Correct some spelling. End one line of code with ';' instead of ','. The before and after object files are identical. Signed-off-by: Randy Dunlap Cc: Jani Nikula Cc: Joonas Lahtinen Cc: Rodrigo Vivi Cc: Tvrtko Ursulin Cc: intel-gfx@lists.freedesktop.org Reviewed-by: Tvrtko Ursulin Signed-off-by: Tvrtko Ursulin Link: https://patchwork.freedesktop.org/patch/msgid/20231213044014.21410-1-rdunlap@infradead.org --- include/uapi/drm/i915_drm.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h index 218edb0a96f8..fd4f9574d177 100644 --- a/include/uapi/drm/i915_drm.h +++ b/include/uapi/drm/i915_drm.h @@ -693,7 +693,7 @@ typedef struct drm_i915_irq_wait { #define I915_PARAM_HAS_EXEC_FENCE 44 /* Query whether DRM_I915_GEM_EXECBUFFER2 supports the ability to capture - * user specified bufffers for post-mortem debugging of GPU hangs. See + * user-specified buffers for post-mortem debugging of GPU hangs. See * EXEC_OBJECT_CAPTURE. */ #define I915_PARAM_HAS_EXEC_CAPTURE 45 @@ -1606,7 +1606,7 @@ struct drm_i915_gem_busy { * is accurate. * * The returned dword is split into two fields to indicate both - * the engine classess on which the object is being read, and the + * the engine classes on which the object is being read, and the * engine class on which it is currently being written (if any). * * The low word (bits 0:15) indicate if the object is being written @@ -1815,7 +1815,7 @@ struct drm_i915_gem_madvise { __u32 handle; /* Advice: either the buffer will be needed again in the near future, - * or wont be and could be discarded under memory pressure. + * or won't be and could be discarded under memory pressure. */ __u32 madv; @@ -3246,7 +3246,7 @@ struct drm_i915_query_topology_info { * // enough to hold our array of engines. The kernel will fill out the * // item.length for us, which is the number of bytes we need. * // - * // Alternatively a large buffer can be allocated straight away enabling + * // Alternatively a large buffer can be allocated straightaway enabling * // querying in one pass, in which case item.length should contain the * // length of the provided buffer. * err = ioctl(fd, DRM_IOCTL_I915_QUERY, &query); @@ -3256,7 +3256,7 @@ struct drm_i915_query_topology_info { * // Now that we allocated the required number of bytes, we call the ioctl * // again, this time with the data_ptr pointing to our newly allocated * // blob, which the kernel can then populate with info on all engines. - * item.data_ptr = (uintptr_t)&info, + * item.data_ptr = (uintptr_t)&info; * * err = ioctl(fd, DRM_IOCTL_I915_QUERY, &query); * if (err) ... @@ -3286,7 +3286,7 @@ struct drm_i915_query_topology_info { /** * struct drm_i915_engine_info * - * Describes one engine and it's capabilities as known to the driver. + * Describes one engine and its capabilities as known to the driver. */ struct drm_i915_engine_info { /** @engine: Engine class and instance. */ -- cgit v1.2.3 From 6872a189be508b9383bc081d462a5d99cbb8319d Mon Sep 17 00:00:00 2001 From: Joshua Ashton Date: Thu, 16 Nov 2023 18:58:12 -0100 Subject: drm/amd/display: Add 3x4 CTM support for plane CTM Create drm_color_ctm_3x4 to support 3x4-dimension plane CTM matrix and convert DRM CTM to DC CSC float matrix. v3: - rename ctm2 to ctm_3x4 (Harry) Reviewed-by: Harry Wentland Signed-off-by: Joshua Ashton Signed-off-by: Alex Deucher --- .../drm/amd/display/amdgpu_dm/amdgpu_dm_color.c | 28 +++++++++++++++++++--- .../drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c | 2 +- include/uapi/drm/drm_mode.h | 8 +++++++ 3 files changed, 34 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c index d52c3333ea13..96aecc1a71a3 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_color.c @@ -434,6 +434,28 @@ static void __drm_ctm_to_dc_matrix(const struct drm_color_ctm *ctm, } } +/** + * __drm_ctm_3x4_to_dc_matrix - converts a DRM CTM 3x4 to a DC CSC float matrix + * @ctm: DRM color transformation matrix with 3x4 dimensions + * @matrix: DC CSC float matrix + * + * The matrix needs to be a 3x4 (12 entry) matrix. + */ +static void __drm_ctm_3x4_to_dc_matrix(const struct drm_color_ctm_3x4 *ctm, + struct fixed31_32 *matrix) +{ + int i; + + /* The format provided is S31.32, using signed-magnitude representation. + * Our fixed31_32 is also S31.32, but is using 2's complement. We have + * to convert from signed-magnitude to 2's complement. + */ + for (i = 0; i < 12; i++) { + /* gamut_remap_matrix[i] = ctm[i - floor(i/4)] */ + matrix[i] = dc_fixpt_from_s3132(ctm->matrix[i]); + } +} + /** * __set_legacy_tf - Calculates the legacy transfer function * @func: transfer function @@ -1173,7 +1195,7 @@ int amdgpu_dm_update_plane_color_mgmt(struct dm_crtc_state *crtc, { struct amdgpu_device *adev = drm_to_adev(crtc->base.state->dev); struct dm_plane_state *dm_plane_state = to_dm_plane_state(plane_state); - struct drm_color_ctm *ctm = NULL; + struct drm_color_ctm_3x4 *ctm = NULL; struct dc_color_caps *color_caps = NULL; bool has_crtc_cm_degamma; int ret; @@ -1228,7 +1250,7 @@ int amdgpu_dm_update_plane_color_mgmt(struct dm_crtc_state *crtc, /* Setup CRTC CTM. */ if (dm_plane_state->ctm) { - ctm = (struct drm_color_ctm *)dm_plane_state->ctm->data; + ctm = (struct drm_color_ctm_3x4 *)dm_plane_state->ctm->data; /* * DCN2 and older don't support both pre-blending and * post-blending gamut remap. For this HW family, if we have @@ -1240,7 +1262,7 @@ int amdgpu_dm_update_plane_color_mgmt(struct dm_crtc_state *crtc, * mapping CRTC CTM to MPC and keeping plane CTM setup at DPP, * as it's done by dcn30_program_gamut_remap(). */ - __drm_ctm_to_dc_matrix(ctm, dc_plane_state->gamut_remap_matrix.matrix); + __drm_ctm_3x4_to_dc_matrix(ctm, dc_plane_state->gamut_remap_matrix.matrix); dc_plane_state->gamut_remap_matrix.enable_remap = true; dc_plane_state->input_csc_color_matrix.enable_adjustment = false; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c index f10c5154d06a..8a4c40b4c27e 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_plane.c @@ -1561,7 +1561,7 @@ dm_atomic_plane_set_property(struct drm_plane *plane, ret = drm_property_replace_blob_from_id(plane->dev, &dm_plane_state->ctm, val, - sizeof(struct drm_color_ctm), -1, + sizeof(struct drm_color_ctm_3x4), -1, &replaced); dm_plane_state->base.color_mgmt_changed |= replaced; return ret; diff --git a/include/uapi/drm/drm_mode.h b/include/uapi/drm/drm_mode.h index 95630f170110..39d9ac0c0a80 100644 --- a/include/uapi/drm/drm_mode.h +++ b/include/uapi/drm/drm_mode.h @@ -846,6 +846,14 @@ struct drm_color_ctm { __u64 matrix[9]; }; +struct drm_color_ctm_3x4 { + /* + * Conversion matrix with 3x4 dimensions in S31.32 sign-magnitude + * (not two's complement!) format. + */ + __u64 matrix[12]; +}; + struct drm_color_lut { /* * Values are mapped linearly to 0.0 - 1.0 range, with 0x0 == 0.0 and -- cgit v1.2.3 From 7259eb7b534735b9c1153654c0bb4c5f059c0dd3 Mon Sep 17 00:00:00 2001 From: Moti Haimovski Date: Sun, 12 Nov 2023 18:07:10 +0200 Subject: accel/habanalabs/gaudi2: add signed dev info uAPI User will provide a nonce via the INFO ioctl, and will retrieve the signed device info generated using given nonce. Signed-off-by: Moti Haimovski Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/accel/habanalabs/common/firmware_if.c | 8 ++++ drivers/accel/habanalabs/common/habanalabs.h | 2 + drivers/accel/habanalabs/common/habanalabs_ioctl.c | 53 ++++++++++++++++++++++ include/linux/habanalabs/cpucp_if.h | 8 +++- include/uapi/drm/habanalabs_accel.h | 28 ++++++++++++ 5 files changed, 98 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/drivers/accel/habanalabs/common/firmware_if.c b/drivers/accel/habanalabs/common/firmware_if.c index 9e9dfe013659..3558a6a8e192 100644 --- a/drivers/accel/habanalabs/common/firmware_if.c +++ b/drivers/accel/habanalabs/common/firmware_if.c @@ -3244,6 +3244,14 @@ int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_in HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC); } +int hl_fw_get_dev_info_signed(struct hl_device *hdev, + struct cpucp_dev_info_signed *dev_info_signed, u32 nonce) +{ + return hl_fw_get_sec_attest_data(hdev, CPUCP_PACKET_INFO_SIGNED_GET, dev_info_signed, + sizeof(struct cpucp_dev_info_signed), nonce, + HL_CPUCP_SEC_ATTEST_INFO_TINEOUT_USEC); +} + int hl_fw_send_generic_request(struct hl_device *hdev, enum hl_passthrough_type sub_opcode, dma_addr_t buff, u32 *size) { diff --git a/drivers/accel/habanalabs/common/habanalabs.h b/drivers/accel/habanalabs/common/habanalabs.h index 7b0209e5bad6..dd3fe3ddc00a 100644 --- a/drivers/accel/habanalabs/common/habanalabs.h +++ b/drivers/accel/habanalabs/common/habanalabs.h @@ -3964,6 +3964,8 @@ long hl_fw_get_max_power(struct hl_device *hdev); void hl_fw_set_max_power(struct hl_device *hdev); int hl_fw_get_sec_attest_info(struct hl_device *hdev, struct cpucp_sec_attest_info *sec_attest_info, u32 nonce); +int hl_fw_get_dev_info_signed(struct hl_device *hdev, + struct cpucp_dev_info_signed *dev_info_signed, u32 nonce); int hl_set_voltage(struct hl_device *hdev, int sensor_index, u32 attr, long value); int hl_set_current(struct hl_device *hdev, int sensor_index, u32 attr, long value); int hl_set_power(struct hl_device *hdev, int sensor_index, u32 attr, long value); diff --git a/drivers/accel/habanalabs/common/habanalabs_ioctl.c b/drivers/accel/habanalabs/common/habanalabs_ioctl.c index 8ef36effb95b..a92713e0e580 100644 --- a/drivers/accel/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/accel/habanalabs/common/habanalabs_ioctl.c @@ -19,6 +19,9 @@ #include +/* make sure there is space for all the signed info */ +static_assert(sizeof(struct cpucp_info) <= SEC_DEV_INFO_BUF_SZ); + static u32 hl_debug_struct_size[HL_DEBUG_OP_TIMESTAMP + 1] = { [HL_DEBUG_OP_ETR] = sizeof(struct hl_debug_params_etr), [HL_DEBUG_OP_ETF] = sizeof(struct hl_debug_params_etf), @@ -719,6 +722,53 @@ free_sec_attest_info: return rc; } +static int dev_info_signed(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + void __user *out = (void __user *) (uintptr_t) args->return_pointer; + struct cpucp_dev_info_signed *dev_info_signed; + struct hl_info_signed *info; + u32 max_size = args->return_size; + int rc; + + if ((!max_size) || (!out)) + return -EINVAL; + + dev_info_signed = kzalloc(sizeof(*dev_info_signed), GFP_KERNEL); + if (!dev_info_signed) + return -ENOMEM; + + info = kzalloc(sizeof(*info), GFP_KERNEL); + if (!info) { + rc = -ENOMEM; + goto free_dev_info_signed; + } + + rc = hl_fw_get_dev_info_signed(hpriv->hdev, + dev_info_signed, args->sec_attest_nonce); + if (rc) + goto free_info; + + info->nonce = le32_to_cpu(dev_info_signed->nonce); + info->info_sig_len = dev_info_signed->info_sig_len; + info->pub_data_len = le16_to_cpu(dev_info_signed->pub_data_len); + info->certificate_len = le16_to_cpu(dev_info_signed->certificate_len); + info->dev_info_len = sizeof(struct cpucp_info); + memcpy(&info->info_sig, &dev_info_signed->info_sig, sizeof(info->info_sig)); + memcpy(&info->public_data, &dev_info_signed->public_data, sizeof(info->public_data)); + memcpy(&info->certificate, &dev_info_signed->certificate, sizeof(info->certificate)); + memcpy(&info->dev_info, &dev_info_signed->info, info->dev_info_len); + + rc = copy_to_user(out, info, min_t(size_t, max_size, sizeof(*info))) ? -EFAULT : 0; + +free_info: + kfree(info); +free_dev_info_signed: + kfree(dev_info_signed); + + return rc; +} + + static int eventfd_register(struct hl_fpriv *hpriv, struct hl_info_args *args) { int rc; @@ -1089,6 +1139,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, case HL_INFO_FW_GENERIC_REQ: return send_fw_generic_request(hdev, args); + case HL_INFO_DEV_SIGNED: + return dev_info_signed(hpriv, args); + default: dev_err(dev, "Invalid request %d\n", args->op); rc = -EINVAL; diff --git a/include/linux/habanalabs/cpucp_if.h b/include/linux/habanalabs/cpucp_if.h index 86ea7c63a0d2..f316c8d0f3fc 100644 --- a/include/linux/habanalabs/cpucp_if.h +++ b/include/linux/habanalabs/cpucp_if.h @@ -659,6 +659,12 @@ enum pq_init_status { * number (nonce) provided by the host to prevent replay attacks. * public key and certificate also provided as part of the FW response. * + * CPUCP_PACKET_INFO_SIGNED_GET - + * Get the device information signed by the Trusted Platform device. + * device info data is also hashed with some unique number (nonce) provided + * by the host to prevent replay attacks. public key and certificate also + * provided as part of the FW response. + * * CPUCP_PACKET_MONITOR_DUMP_GET - * Get monitors registers dump from the CpuCP kernel. * The CPU will put the registers dump in the a buffer allocated by the driver @@ -733,7 +739,7 @@ enum cpucp_packet_id { CPUCP_PACKET_ENGINE_CORE_ASID_SET, /* internal */ CPUCP_PACKET_RESERVED2, /* not used */ CPUCP_PACKET_SEC_ATTEST_GET, /* internal */ - CPUCP_PACKET_RESERVED3, /* not used */ + CPUCP_PACKET_INFO_SIGNED_GET, /* internal */ CPUCP_PACKET_RESERVED4, /* not used */ CPUCP_PACKET_MONITOR_DUMP_GET, /* debugfs */ CPUCP_PACKET_RESERVED5, /* not used */ diff --git a/include/uapi/drm/habanalabs_accel.h b/include/uapi/drm/habanalabs_accel.h index 347c7b62e60e..a512dc4cffd0 100644 --- a/include/uapi/drm/habanalabs_accel.h +++ b/include/uapi/drm/habanalabs_accel.h @@ -846,6 +846,7 @@ enum hl_server_type { #define HL_INFO_HW_ERR_EVENT 36 #define HL_INFO_FW_ERR_EVENT 37 #define HL_INFO_USER_ENGINE_ERR_EVENT 38 +#define HL_INFO_DEV_SIGNED 40 #define HL_INFO_VERSION_MAX_LEN 128 #define HL_INFO_CARD_NAME_MAX_LEN 16 @@ -1256,6 +1257,7 @@ struct hl_info_dev_memalloc_page_sizes { #define SEC_SIGNATURE_BUF_SZ 255 /* (256 - 1) 1 byte used for size */ #define SEC_PUB_DATA_BUF_SZ 510 /* (512 - 2) 2 bytes used for size */ #define SEC_CERTIFICATE_BUF_SZ 2046 /* (2048 - 2) 2 bytes used for size */ +#define SEC_DEV_INFO_BUF_SZ 5120 /* * struct hl_info_sec_attest - attestation report of the boot @@ -1290,6 +1292,32 @@ struct hl_info_sec_attest { __u8 pad0[2]; }; +/* + * struct hl_info_signed - device information signed by a secured device. + * @nonce: number only used once. random number provided by host. this also passed to the quote + * command as a qualifying data. + * @pub_data_len: length of the public data (bytes) + * @certificate_len: length of the certificate (bytes) + * @info_sig_len: length of the attestation signature (bytes) + * @public_data: public key info signed info data (outPublic + name + qualifiedName) + * @certificate: certificate for the signing key + * @info_sig: signature of the info + nonce data. + * @dev_info_len: length of device info (bytes) + * @dev_info: device info as byte array. + */ +struct hl_info_signed { + __u32 nonce; + __u16 pub_data_len; + __u16 certificate_len; + __u8 info_sig_len; + __u8 public_data[SEC_PUB_DATA_BUF_SZ]; + __u8 certificate[SEC_CERTIFICATE_BUF_SZ]; + __u8 info_sig[SEC_SIGNATURE_BUF_SZ]; + __u16 dev_info_len; + __u8 dev_info[SEC_DEV_INFO_BUF_SZ]; + __u8 pad[2]; +}; + /** * struct hl_page_fault_info - page fault information. * @timestamp: timestamp of page fault. -- cgit v1.2.3 From 3949d57f1ef62ea00344617fd638ed6c778db8d8 Mon Sep 17 00:00:00 2001 From: José Roberto de Souza Date: Mon, 23 Jan 2023 08:43:10 -0800 Subject: drm/xe/uapi: Rename XE_ENGINE_PROPERTY_X to XE_ENGINE_SET_PROPERTY_X MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Engine property get uAPI will be added, so to avoid ambiguity here renaming XE_ENGINE_PROPERTY_X to XE_ENGINE_SET_PROPERTY_X. No changes in behavior. Cc: Matthew Brost Cc: Maarten Lankhorst Signed-off-by: José Roberto de Souza Reviewed-by: Matthew Brost Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_engine.c | 18 +++++++++--------- include/uapi/drm/xe_drm.h | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_engine.c b/drivers/gpu/drm/xe/xe_engine.c index 63219bd98be7..1b85bf4abe3d 100644 --- a/drivers/gpu/drm/xe/xe_engine.c +++ b/drivers/gpu/drm/xe/xe_engine.c @@ -314,15 +314,15 @@ typedef int (*xe_engine_set_property_fn)(struct xe_device *xe, u64 value, bool create); static const xe_engine_set_property_fn engine_set_property_funcs[] = { - [XE_ENGINE_PROPERTY_PRIORITY] = engine_set_priority, - [XE_ENGINE_PROPERTY_TIMESLICE] = engine_set_timeslice, - [XE_ENGINE_PROPERTY_PREEMPTION_TIMEOUT] = engine_set_preemption_timeout, - [XE_ENGINE_PROPERTY_COMPUTE_MODE] = engine_set_compute_mode, - [XE_ENGINE_PROPERTY_PERSISTENCE] = engine_set_persistence, - [XE_ENGINE_PROPERTY_JOB_TIMEOUT] = engine_set_job_timeout, - [XE_ENGINE_PROPERTY_ACC_TRIGGER] = engine_set_acc_trigger, - [XE_ENGINE_PROPERTY_ACC_NOTIFY] = engine_set_acc_notify, - [XE_ENGINE_PROPERTY_ACC_GRANULARITY] = engine_set_acc_granularity, + [XE_ENGINE_SET_PROPERTY_PRIORITY] = engine_set_priority, + [XE_ENGINE_SET_PROPERTY_TIMESLICE] = engine_set_timeslice, + [XE_ENGINE_SET_PROPERTY_PREEMPTION_TIMEOUT] = engine_set_preemption_timeout, + [XE_ENGINE_SET_PROPERTY_COMPUTE_MODE] = engine_set_compute_mode, + [XE_ENGINE_SET_PROPERTY_PERSISTENCE] = engine_set_persistence, + [XE_ENGINE_SET_PROPERTY_JOB_TIMEOUT] = engine_set_job_timeout, + [XE_ENGINE_SET_PROPERTY_ACC_TRIGGER] = engine_set_acc_trigger, + [XE_ENGINE_SET_PROPERTY_ACC_NOTIFY] = engine_set_acc_notify, + [XE_ENGINE_SET_PROPERTY_ACC_GRANULARITY] = engine_set_acc_granularity, }; static int engine_user_ext_set_property(struct xe_device *xe, diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index f64b1c785fad..8dc8ebbaf337 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -511,21 +511,21 @@ struct drm_xe_engine_set_property { __u32 engine_id; /** @property: property to set */ -#define XE_ENGINE_PROPERTY_PRIORITY 0 -#define XE_ENGINE_PROPERTY_TIMESLICE 1 -#define XE_ENGINE_PROPERTY_PREEMPTION_TIMEOUT 2 +#define XE_ENGINE_SET_PROPERTY_PRIORITY 0 +#define XE_ENGINE_SET_PROPERTY_TIMESLICE 1 +#define XE_ENGINE_SET_PROPERTY_PREEMPTION_TIMEOUT 2 /* * Long running or ULLS engine mode. DMA fences not allowed in this * mode. Must match the value of DRM_XE_VM_CREATE_COMPUTE_MODE, serves * as a sanity check the UMD knows what it is doing. Can only be set at * engine create time. */ -#define XE_ENGINE_PROPERTY_COMPUTE_MODE 3 -#define XE_ENGINE_PROPERTY_PERSISTENCE 4 -#define XE_ENGINE_PROPERTY_JOB_TIMEOUT 5 -#define XE_ENGINE_PROPERTY_ACC_TRIGGER 6 -#define XE_ENGINE_PROPERTY_ACC_NOTIFY 7 -#define XE_ENGINE_PROPERTY_ACC_GRANULARITY 8 +#define XE_ENGINE_SET_PROPERTY_COMPUTE_MODE 3 +#define XE_ENGINE_SET_PROPERTY_PERSISTENCE 4 +#define XE_ENGINE_SET_PROPERTY_JOB_TIMEOUT 5 +#define XE_ENGINE_SET_PROPERTY_ACC_TRIGGER 6 +#define XE_ENGINE_SET_PROPERTY_ACC_NOTIFY 7 +#define XE_ENGINE_SET_PROPERTY_ACC_GRANULARITY 8 __u32 property; /** @value: property value */ -- cgit v1.2.3 From 19431b029b8b5d095e77767f269cb142c687084e Mon Sep 17 00:00:00 2001 From: José Roberto de Souza Date: Mon, 23 Jan 2023 09:11:32 -0800 Subject: drm/xe/uapi: Add XE_ENGINE_GET_PROPERTY uAPI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is intended to get some properties that are of interest of UMDs like the ban state. Cc: Matthew Brost Cc: Maarten Lankhorst Signed-off-by: José Roberto de Souza Reviewed-by: Matthew Brost Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_device.c | 2 ++ drivers/gpu/drm/xe/xe_engine.c | 26 ++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_engine.h | 2 ++ include/uapi/drm/xe_drm.h | 22 +++++++++++++++++++++- 4 files changed, 51 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 104ab12cc2ed..9881b591bfdd 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -89,6 +89,8 @@ static const struct drm_ioctl_desc xe_ioctls[] = { DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_ENGINE_CREATE, xe_engine_create_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_ENGINE_GET_PROPERTY, xe_engine_get_property_ioctl, + DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_ENGINE_DESTROY, xe_engine_destroy_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), diff --git a/drivers/gpu/drm/xe/xe_engine.c b/drivers/gpu/drm/xe/xe_engine.c index 1b85bf4abe3d..b69dcbef0824 100644 --- a/drivers/gpu/drm/xe/xe_engine.c +++ b/drivers/gpu/drm/xe/xe_engine.c @@ -639,6 +639,32 @@ put_rpm: return err; } +int xe_engine_get_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct xe_file *xef = to_xe_file(file); + struct drm_xe_engine_get_property *args = data; + struct xe_engine *e; + + mutex_lock(&xef->engine.lock); + e = xa_load(&xef->engine.xa, args->engine_id); + mutex_unlock(&xef->engine.lock); + + if (XE_IOCTL_ERR(xe, !e)) + return -ENOENT; + + switch (args->property) { + case XE_ENGINE_GET_PROPERTY_BAN: + args->value = !!(e->flags & ENGINE_FLAG_BANNED); + break; + default: + return -EINVAL; + } + + return 0; +} + static void engine_kill_compute(struct xe_engine *e) { if (!xe_vm_in_compute_mode(e->vm)) diff --git a/drivers/gpu/drm/xe/xe_engine.h b/drivers/gpu/drm/xe/xe_engine.h index 4d1b609fea7e..a3a44534003f 100644 --- a/drivers/gpu/drm/xe/xe_engine.h +++ b/drivers/gpu/drm/xe/xe_engine.h @@ -50,5 +50,7 @@ int xe_engine_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int xe_engine_set_property_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +int xe_engine_get_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); #endif diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 8dc8ebbaf337..756c5994ae63 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -118,6 +118,7 @@ struct xe_user_extension { #define DRM_XE_ENGINE_SET_PROPERTY 0x0a #define DRM_XE_WAIT_USER_FENCE 0x0b #define DRM_XE_VM_MADVISE 0x0c +#define DRM_XE_ENGINE_GET_PROPERTY 0x0d /* Must be kept compact -- no holes */ #define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query) @@ -127,6 +128,7 @@ struct xe_user_extension { #define DRM_IOCTL_XE_VM_DESTROY DRM_IOW( DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) #define DRM_IOCTL_XE_VM_BIND DRM_IOW( DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) #define DRM_IOCTL_XE_ENGINE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_ENGINE_CREATE, struct drm_xe_engine_create) +#define DRM_IOCTL_XE_ENGINE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_ENGINE_GET_PROPERTY, struct drm_xe_engine_get_property) #define DRM_IOCTL_XE_ENGINE_DESTROY DRM_IOW( DRM_COMMAND_BASE + DRM_XE_ENGINE_DESTROY, struct drm_xe_engine_destroy) #define DRM_IOCTL_XE_EXEC DRM_IOW( DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) #define DRM_IOCTL_XE_MMIO DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_MMIO, struct drm_xe_mmio) @@ -568,8 +570,26 @@ struct drm_xe_engine_create { __u64 reserved[2]; }; +struct drm_xe_engine_get_property { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @engine_id: Engine ID */ + __u32 engine_id; + + /** @property: property to get */ +#define XE_ENGINE_GET_PROPERTY_BAN 0 + __u32 property; + + /** @value: property value */ + __u64 value; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + struct drm_xe_engine_destroy { - /** @vm_id: VM ID */ + /** @engine_id: Engine ID */ __u32 engine_id; /** @pad: MBZ */ -- cgit v1.2.3 From ccbb6ad52ab1a0fa4d386dc9f591240f5eb81646 Mon Sep 17 00:00:00 2001 From: Lucas De Marchi Date: Mon, 13 Mar 2023 14:16:28 -0700 Subject: drm/xe: Replace i915 with xe in uapi All structs and defines had already been renamed to "xe", but some comments with "i915" were left over. Rename them. Signed-off-by: Lucas De Marchi Reviewed-by: Matthew Auld Link: https://lore.kernel.org/r/20230313211628.2492587-1-lucas.demarchi@intel.com Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 756c5994ae63..32a4265de402 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -37,7 +37,7 @@ extern "C" { */ /** - * struct i915_user_extension - Base class for defining a chain of extensions + * struct xe_user_extension - Base class for defining a chain of extensions * * Many interfaces need to grow over time. In most cases we can simply * extend the struct and have userspace pass in more data. Another option, @@ -55,20 +55,20 @@ extern "C" { * * .. code-block:: C * - * struct i915_user_extension ext3 { + * struct xe_user_extension ext3 { * .next_extension = 0, // end * .name = ..., * }; - * struct i915_user_extension ext2 { + * struct xe_user_extension ext2 { * .next_extension = (uintptr_t)&ext3, * .name = ..., * }; - * struct i915_user_extension ext1 { + * struct xe_user_extension ext1 { * .next_extension = (uintptr_t)&ext2, * .name = ..., * }; * - * Typically the struct i915_user_extension would be embedded in some uAPI + * Typically the struct xe_user_extension would be embedded in some uAPI * struct, and in this case we would feed it the head of the chain(i.e ext1), * which would then apply all of the above extensions. * @@ -77,7 +77,7 @@ struct xe_user_extension { /** * @next_extension: * - * Pointer to the next struct i915_user_extension, or zero if the end. + * Pointer to the next struct xe_user_extension, or zero if the end. */ __u64 next_extension; /** @@ -87,7 +87,7 @@ struct xe_user_extension { * * Also note that the name space for this is not global for the whole * driver, but rather its scope/meaning is limited to the specific piece - * of uAPI which has embedded the struct i915_user_extension. + * of uAPI which has embedded the struct xe_user_extension. */ __u32 name; /** @@ -99,7 +99,7 @@ struct xe_user_extension { }; /* - * i915 specific ioctls. + * xe specific ioctls. * * The device specific ioctl range is [DRM_COMMAND_BASE, DRM_COMMAND_END) ie * [0x40, 0xa0) (a0 is excluded). The numbers below are defined as offset -- cgit v1.2.3 From ef5e3c2f703d05c9d296d8f8ad0a0f48f6c1fcc9 Mon Sep 17 00:00:00 2001 From: José Roberto de Souza Date: Thu, 23 Mar 2023 12:24:59 -0700 Subject: drm/xe: Add max engine priority to xe query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Intel Vulkan driver needs to know what is the maximum priority to fill a device info struct for applications. Right now we getting this information by creating a engine and setting priorities from min to high to know what is the maximum priority for running process but this leads to info messages to be printed to dmesg: xe 0000:03:00.0: [drm] Ioctl argument check failed at drivers/gpu/drm/xe/xe_engine.c:178: value == DRM_SCHED_PRIORITY_HIGH && !capable(CAP_SYS_NICE) It does not cause any harm but when executing a test suite like crucible it causes thousands of those messages to be printed. So here adding one more property to drm_xe_query_config to fetch the max engine priority. Cc: Matthew Brost Reviewed-by: Matthew Brost Signed-off-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_engine.c | 10 ++++++++-- drivers/gpu/drm/xe/xe_engine.h | 1 + drivers/gpu/drm/xe/xe_query.c | 3 +++ include/uapi/drm/xe_drm.h | 3 ++- 4 files changed, 14 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_engine.c b/drivers/gpu/drm/xe/xe_engine.c index 8011f5827cbe..141cb223ba02 100644 --- a/drivers/gpu/drm/xe/xe_engine.c +++ b/drivers/gpu/drm/xe/xe_engine.c @@ -169,14 +169,20 @@ struct xe_engine *xe_engine_lookup(struct xe_file *xef, u32 id) return e; } +enum xe_engine_priority +xe_engine_device_get_max_priority(struct xe_device *xe) +{ + return capable(CAP_SYS_NICE) ? XE_ENGINE_PRIORITY_HIGH : + XE_ENGINE_PRIORITY_NORMAL; +} + static int engine_set_priority(struct xe_device *xe, struct xe_engine *e, u64 value, bool create) { if (XE_IOCTL_ERR(xe, value > XE_ENGINE_PRIORITY_HIGH)) return -EINVAL; - if (XE_IOCTL_ERR(xe, value == XE_ENGINE_PRIORITY_HIGH && - !capable(CAP_SYS_NICE))) + if (XE_IOCTL_ERR(xe, value > xe_engine_device_get_max_priority(xe))) return -EPERM; return e->ops->set_priority(e, value); diff --git a/drivers/gpu/drm/xe/xe_engine.h b/drivers/gpu/drm/xe/xe_engine.h index 1cf7f23c4afd..b95d9b040877 100644 --- a/drivers/gpu/drm/xe/xe_engine.h +++ b/drivers/gpu/drm/xe/xe_engine.h @@ -54,5 +54,6 @@ int xe_engine_set_property_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int xe_engine_get_property_ioctl(struct drm_device *dev, void *data, struct drm_file *file); +enum xe_engine_priority xe_engine_device_get_max_priority(struct xe_device *xe); #endif diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 0f70945176f6..dd64ff0d2a57 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -12,6 +12,7 @@ #include "xe_bo.h" #include "xe_device.h" +#include "xe_engine.h" #include "xe_ggtt.h" #include "xe_gt.h" #include "xe_guc_hwconfig.h" @@ -194,6 +195,8 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) config->info[XE_QUERY_CONFIG_GT_COUNT] = xe->info.tile_count; config->info[XE_QUERY_CONFIG_MEM_REGION_COUNT] = hweight_long(xe->info.mem_region_mask); + config->info[XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY] = + xe_engine_device_get_max_priority(xe); if (copy_to_user(query_ptr, config, size)) { kfree(config); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 32a4265de402..b3bcb7106850 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -184,7 +184,8 @@ struct drm_xe_query_config { #define XE_QUERY_CONFIG_VA_BITS 3 #define XE_QUERY_CONFIG_GT_COUNT 4 #define XE_QUERY_CONFIG_MEM_REGION_COUNT 5 -#define XE_QUERY_CONFIG_NUM_PARAM XE_QUERY_CONFIG_MEM_REGION_COUNT + 1 +#define XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY 6 +#define XE_QUERY_CONFIG_NUM_PARAM XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY + 1 __u64 info[]; }; -- cgit v1.2.3 From e2bd81af05cb6dc9cbf7a367a48e43316207dd0e Mon Sep 17 00:00:00 2001 From: Christopher Snowhill Date: Wed, 24 May 2023 18:56:06 -0700 Subject: drm/xe: Add explicit padding to uAPI definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pad the uAPI definition so that it would align identically between 64-bit and 32-bit uarch, so consumers using this header will work correctly from 32-bit compat userspace on a 64-bit kernel. Do it in a minimally invasive way, so that 64-bit userspace will still work with the previous header, and so that no fields suddenly change sizes. Originally inspired by mlankhorst. Signed-off-by: Christopher Snowhill Reviewed-by: José Roberto de Souza Reviewed-by: Lucas De Marchi Signed-off-by: Lucas De Marchi Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index b3bcb7106850..34aff9e15fe6 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -91,7 +91,7 @@ struct xe_user_extension { */ __u32 name; /** - * @flags: MBZ + * @pad: MBZ * * All undefined bits must be zero. */ @@ -291,6 +291,9 @@ struct drm_xe_gem_create { */ __u32 handle; + /** @pad: MBZ */ + __u32 pad; + /** @reserved: Reserved */ __u64 reserved[2]; }; @@ -335,6 +338,9 @@ struct drm_xe_ext_vm_set_property { #define XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS 0 __u32 property; + /** @pad: MBZ */ + __u32 pad; + /** @value: property value */ __u64 value; @@ -379,6 +385,9 @@ struct drm_xe_vm_bind_op { */ __u32 obj; + /** @pad: MBZ */ + __u32 pad; + union { /** * @obj_offset: Offset into the object, MBZ for CLEAR_RANGE, @@ -469,6 +478,9 @@ struct drm_xe_vm_bind { /** @num_binds: number of binds in this IOCTL */ __u32 num_binds; + /** @pad: MBZ */ + __u32 pad; + union { /** @bind: used if num_binds == 1 */ struct drm_xe_vm_bind_op bind; @@ -482,6 +494,9 @@ struct drm_xe_vm_bind { /** @num_syncs: amount of syncs to wait on */ __u32 num_syncs; + /** @pad2: MBZ */ + __u32 pad2; + /** @syncs: pointer to struct drm_xe_sync array */ __u64 syncs; @@ -497,6 +512,9 @@ struct drm_xe_ext_engine_set_property { /** @property: property to set */ __u32 property; + /** @pad: MBZ */ + __u32 pad; + /** @value: property value */ __u64 value; }; @@ -612,6 +630,9 @@ struct drm_xe_sync { #define DRM_XE_SYNC_USER_FENCE 0x3 #define DRM_XE_SYNC_SIGNAL 0x10 + /** @pad: MBZ */ + __u32 pad; + union { __u32 handle; /** @@ -656,6 +677,9 @@ struct drm_xe_exec { */ __u16 num_batch_buffer; + /** @pad: MBZ */ + __u16 pad[3]; + /** @reserved: Reserved */ __u64 reserved[2]; }; @@ -718,6 +742,8 @@ struct drm_xe_wait_user_fence { #define DRM_XE_UFENCE_WAIT_ABSTIME (1 << 1) #define DRM_XE_UFENCE_WAIT_VM_ERROR (1 << 2) __u16 flags; + /** @pad: MBZ */ + __u32 pad; /** @value: compare value */ __u64 value; /** @mask: comparison mask */ @@ -750,6 +776,9 @@ struct drm_xe_vm_madvise { /** @vm_id: The ID VM in which the VMA exists */ __u32 vm_id; + /** @pad: MBZ */ + __u32 pad; + /** @range: Number of bytes in the VMA */ __u64 range; @@ -794,6 +823,9 @@ struct drm_xe_vm_madvise { /** @property: property to set */ __u32 property; + /** @pad2: MBZ */ + __u32 pad2; + /** @value: property value */ __u64 value; -- cgit v1.2.3 From 876611c2b75689c6bea43bdbbbef9b358f71526a Mon Sep 17 00:00:00 2001 From: Matt Roper Date: Thu, 1 Jun 2023 14:52:25 -0700 Subject: drm/xe: Memory allocations are tile-based, not GT-based Since memory and address spaces are a tile concept rather than a GT concept, we need to plumb tile-based handling through lots of memory-related code. Note that one remaining shortcoming here that will need to be addressed before media GT support can be re-enabled is that although the address space is shared between a tile's GTs, each GT caches the PTEs independently in their own TLB and thus TLB invalidation should be handled at the GT level. v2: - Fix kunit test build. Reviewed-by: Lucas De Marchi Link: https://lore.kernel.org/r/20230601215244.678611-13-matthew.d.roper@intel.com Signed-off-by: Matt Roper Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/tests/xe_bo.c | 2 +- drivers/gpu/drm/xe/tests/xe_migrate.c | 21 ++--- drivers/gpu/drm/xe/xe_bb.c | 3 +- drivers/gpu/drm/xe/xe_bo.c | 67 +++++++-------- drivers/gpu/drm/xe/xe_bo.h | 18 ++-- drivers/gpu/drm/xe/xe_bo_evict.c | 2 +- drivers/gpu/drm/xe/xe_bo_types.h | 4 +- drivers/gpu/drm/xe/xe_device_types.h | 7 ++ drivers/gpu/drm/xe/xe_ggtt.c | 5 +- drivers/gpu/drm/xe/xe_gt.c | 21 +---- drivers/gpu/drm/xe/xe_gt_debugfs.c | 6 +- drivers/gpu/drm/xe/xe_gt_pagefault.c | 14 ++-- drivers/gpu/drm/xe/xe_gt_types.h | 7 -- drivers/gpu/drm/xe/xe_guc_ads.c | 5 +- drivers/gpu/drm/xe/xe_guc_ct.c | 5 +- drivers/gpu/drm/xe/xe_guc_hwconfig.c | 5 +- drivers/gpu/drm/xe/xe_guc_log.c | 6 +- drivers/gpu/drm/xe/xe_guc_pc.c | 5 +- drivers/gpu/drm/xe/xe_hw_engine.c | 5 +- drivers/gpu/drm/xe/xe_lrc.c | 13 ++- drivers/gpu/drm/xe/xe_lrc_types.h | 4 +- drivers/gpu/drm/xe/xe_migrate.c | 23 ++--- drivers/gpu/drm/xe/xe_migrate.h | 5 +- drivers/gpu/drm/xe/xe_pt.c | 144 +++++++++++++++----------------- drivers/gpu/drm/xe/xe_pt.h | 14 ++-- drivers/gpu/drm/xe/xe_sa.c | 13 ++- drivers/gpu/drm/xe/xe_sa.h | 4 +- drivers/gpu/drm/xe/xe_tile.c | 7 ++ drivers/gpu/drm/xe/xe_uc_fw.c | 5 +- drivers/gpu/drm/xe/xe_vm.c | 152 +++++++++++++++++----------------- drivers/gpu/drm/xe/xe_vm.h | 2 +- drivers/gpu/drm/xe/xe_vm_types.h | 12 +-- include/uapi/drm/xe_drm.h | 4 +- 33 files changed, 298 insertions(+), 312 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/tests/xe_bo.c b/drivers/gpu/drm/xe/tests/xe_bo.c index 6235a6c73a06..f933e5df6c12 100644 --- a/drivers/gpu/drm/xe/tests/xe_bo.c +++ b/drivers/gpu/drm/xe/tests/xe_bo.c @@ -173,7 +173,7 @@ static int evict_test_run_gt(struct xe_device *xe, struct xe_gt *gt, struct kuni { struct xe_bo *bo, *external; unsigned int bo_flags = XE_BO_CREATE_USER_BIT | - XE_BO_CREATE_VRAM_IF_DGFX(gt); + XE_BO_CREATE_VRAM_IF_DGFX(gt_to_tile(gt)); struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->primary_gt.migrate); struct ww_acquire_ctx ww; int err, i; diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c index 4a3ca2960fd5..85ef9bacfe52 100644 --- a/drivers/gpu/drm/xe/tests/xe_migrate.c +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -63,7 +63,7 @@ static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe, static void sanity_populate_cb(struct xe_migrate_pt_update *pt_update, - struct xe_gt *gt, struct iosys_map *map, void *dst, + struct xe_tile *tile, struct iosys_map *map, void *dst, u32 qword_ofs, u32 num_qwords, const struct xe_vm_pgtable_update *update) { @@ -76,7 +76,7 @@ sanity_populate_cb(struct xe_migrate_pt_update *pt_update, for (i = 0; i < num_qwords; i++) { value = (qword_ofs + i - update->ofs) * 0x1111111111111111ULL; if (map) - xe_map_wr(gt_to_xe(gt), map, (qword_ofs + i) * + xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * sizeof(u64), u64, value); else ptr[i] = value; @@ -108,7 +108,7 @@ static void test_copy(struct xe_migrate *m, struct xe_bo *bo, const char *str = big ? "Copying big bo" : "Copying small bo"; int err; - struct xe_bo *sysmem = xe_bo_create_locked(xe, m->gt, NULL, + struct xe_bo *sysmem = xe_bo_create_locked(xe, gt_to_tile(m->gt), NULL, bo->size, ttm_bo_type_kernel, XE_BO_CREATE_SYSTEM_BIT); @@ -240,6 +240,7 @@ static void test_pt_update(struct xe_migrate *m, struct xe_bo *pt, static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) { struct xe_gt *gt = m->gt; + struct xe_tile *tile = gt_to_tile(m->gt); struct xe_device *xe = gt_to_xe(gt); struct xe_bo *pt, *bo = m->pt_bo, *big, *tiny; struct xe_res_cursor src_it; @@ -256,18 +257,18 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) return; } - big = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, SZ_4M, + big = xe_bo_create_pin_map(xe, tile, m->eng->vm, SZ_4M, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(m->gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_PINNED_BIT); if (IS_ERR(big)) { KUNIT_FAIL(test, "Failed to allocate bo: %li\n", PTR_ERR(big)); goto vunmap; } - pt = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, XE_PAGE_SIZE, + pt = xe_bo_create_pin_map(xe, tile, m->eng->vm, XE_PAGE_SIZE, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(m->gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_PINNED_BIT); if (IS_ERR(pt)) { KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", @@ -275,10 +276,10 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) goto free_big; } - tiny = xe_bo_create_pin_map(xe, m->gt, m->eng->vm, + tiny = xe_bo_create_pin_map(xe, tile, m->eng->vm, 2 * SZ_4K, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(m->gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_PINNED_BIT); if (IS_ERR(tiny)) { KUNIT_FAIL(test, "Failed to allocate fake pt: %li\n", @@ -286,7 +287,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) goto free_pt; } - bb = xe_bb_new(m->gt, 32, xe->info.supports_usm); + bb = xe_bb_new(gt, 32, xe->info.supports_usm); if (IS_ERR(bb)) { KUNIT_FAIL(test, "Failed to create batchbuffer: %li\n", PTR_ERR(bb)); diff --git a/drivers/gpu/drm/xe/xe_bb.c b/drivers/gpu/drm/xe/xe_bb.c index bf7c94b769d7..f9b6b7adf99f 100644 --- a/drivers/gpu/drm/xe/xe_bb.c +++ b/drivers/gpu/drm/xe/xe_bb.c @@ -30,6 +30,7 @@ static int bb_prefetch(struct xe_gt *gt) struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 dwords, bool usm) { + struct xe_tile *tile = gt_to_tile(gt); struct xe_bb *bb = kmalloc(sizeof(*bb), GFP_KERNEL); int err; @@ -42,7 +43,7 @@ struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 dwords, bool usm) * space to accomodate the platform-specific hardware prefetch * requirements. */ - bb->bo = xe_sa_bo_new(!usm ? gt->kernel_bb_pool : gt->usm.bb_pool, + bb->bo = xe_sa_bo_new(!usm ? tile->mem.kernel_bb_pool : gt->usm.bb_pool, 4 * (dwords + 1) + bb_prefetch(gt)); if (IS_ERR(bb->bo)) { err = PTR_ERR(bb->bo); diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 8ee6bad59a75..7c59487af86a 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -458,7 +458,7 @@ static int xe_bo_trigger_rebind(struct xe_device *xe, struct xe_bo *bo, } xe_vm_assert_held(vm); - if (list_empty(&vma->rebind_link) && vma->gt_present) + if (list_empty(&vma->rebind_link) && vma->tile_present) list_add_tail(&vma->rebind_link, &vm->rebind_list); if (vm_resv_locked) @@ -565,7 +565,7 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, struct xe_bo *bo = ttm_to_xe_bo(ttm_bo); struct ttm_resource *old_mem = ttm_bo->resource; struct ttm_tt *ttm = ttm_bo->ttm; - struct xe_gt *gt = NULL; + struct xe_tile *tile = NULL; struct dma_fence *fence; bool move_lacks_source; bool needs_clear; @@ -635,15 +635,15 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, goto out; } - if (bo->gt) - gt = bo->gt; + if (bo->tile) + tile = bo->tile; else if (resource_is_vram(new_mem)) - gt = &mem_type_to_tile(xe, new_mem->mem_type)->primary_gt; + tile = mem_type_to_tile(xe, new_mem->mem_type); else if (resource_is_vram(old_mem)) - gt = &mem_type_to_tile(xe, old_mem->mem_type)->primary_gt; + tile = mem_type_to_tile(xe, old_mem->mem_type); - XE_BUG_ON(!gt); - XE_BUG_ON(!gt->migrate); + XE_BUG_ON(!tile); + XE_BUG_ON(!tile->primary_gt.migrate); trace_xe_bo_move(bo); xe_device_mem_access_get(xe); @@ -664,7 +664,7 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, /* Create a new VMAP once kernel BO back in VRAM */ if (!ret && resource_is_vram(new_mem)) { - void *new_addr = gt_to_tile(gt)->mem.vram.mapping + + void *new_addr = tile->mem.vram.mapping + (new_mem->start << PAGE_SHIFT); if (XE_WARN_ON(new_mem->start == XE_BO_INVALID_OFFSET)) { @@ -681,9 +681,10 @@ static int xe_bo_move(struct ttm_buffer_object *ttm_bo, bool evict, } } else { if (move_lacks_source) - fence = xe_migrate_clear(gt->migrate, bo, new_mem); + fence = xe_migrate_clear(tile->primary_gt.migrate, bo, new_mem); else - fence = xe_migrate_copy(gt->migrate, bo, bo, old_mem, new_mem); + fence = xe_migrate_copy(tile->primary_gt.migrate, + bo, bo, old_mem, new_mem); if (IS_ERR(fence)) { ret = PTR_ERR(fence); xe_device_mem_access_put(xe); @@ -964,7 +965,7 @@ static void xe_ttm_bo_destroy(struct ttm_buffer_object *ttm_bo) WARN_ON(!list_empty(&bo->vmas)); if (bo->ggtt_node.size) - xe_ggtt_remove_bo(gt_to_tile(bo->gt)->mem.ggtt, bo); + xe_ggtt_remove_bo(bo->tile->mem.ggtt, bo); if (bo->vm && xe_bo_is_user(bo)) xe_vm_put(bo->vm); @@ -1086,7 +1087,7 @@ void xe_bo_free(struct xe_bo *bo) } struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, - struct xe_gt *gt, struct dma_resv *resv, + struct xe_tile *tile, struct dma_resv *resv, size_t size, enum ttm_bo_type type, u32 flags) { @@ -1099,7 +1100,7 @@ struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, int err; /* Only kernel objects should set GT */ - XE_BUG_ON(gt && type != ttm_bo_type_kernel); + XE_BUG_ON(tile && type != ttm_bo_type_kernel); if (XE_WARN_ON(!size)) return ERR_PTR(-EINVAL); @@ -1120,7 +1121,7 @@ struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, alignment = SZ_4K >> PAGE_SHIFT; } - bo->gt = gt; + bo->tile = tile; bo->size = size; bo->flags = flags; bo->ttm.base.funcs = &xe_gem_object_funcs; @@ -1202,7 +1203,7 @@ static int __xe_bo_fixed_placement(struct xe_device *xe, struct xe_bo * xe_bo_create_locked_range(struct xe_device *xe, - struct xe_gt *gt, struct xe_vm *vm, + struct xe_tile *tile, struct xe_vm *vm, size_t size, u64 start, u64 end, enum ttm_bo_type type, u32 flags) { @@ -1225,7 +1226,7 @@ xe_bo_create_locked_range(struct xe_device *xe, } } - bo = __xe_bo_create_locked(xe, bo, gt, vm ? &vm->resv : NULL, size, + bo = __xe_bo_create_locked(xe, bo, tile, vm ? &vm->resv : NULL, size, type, flags); if (IS_ERR(bo)) return bo; @@ -1235,16 +1236,16 @@ xe_bo_create_locked_range(struct xe_device *xe, bo->vm = vm; if (bo->flags & XE_BO_CREATE_GGTT_BIT) { - if (!gt && flags & XE_BO_CREATE_STOLEN_BIT) - gt = xe_device_get_gt(xe, 0); + if (!tile && flags & XE_BO_CREATE_STOLEN_BIT) + tile = xe_device_get_root_tile(xe); - XE_BUG_ON(!gt); + XE_BUG_ON(!tile); if (flags & XE_BO_CREATE_STOLEN_BIT && flags & XE_BO_FIXED_PLACEMENT_BIT) { - err = xe_ggtt_insert_bo_at(gt_to_tile(gt)->mem.ggtt, bo, start); + err = xe_ggtt_insert_bo_at(tile->mem.ggtt, bo, start); } else { - err = xe_ggtt_insert_bo(gt_to_tile(gt)->mem.ggtt, bo); + err = xe_ggtt_insert_bo(tile->mem.ggtt, bo); } if (err) goto err_unlock_put_bo; @@ -1258,18 +1259,18 @@ err_unlock_put_bo: return ERR_PTR(err); } -struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags) { - return xe_bo_create_locked_range(xe, gt, vm, size, 0, ~0ULL, type, flags); + return xe_bo_create_locked_range(xe, tile, vm, size, 0, ~0ULL, type, flags); } -struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags) { - struct xe_bo *bo = xe_bo_create_locked(xe, gt, vm, size, type, flags); + struct xe_bo *bo = xe_bo_create_locked(xe, tile, vm, size, type, flags); if (!IS_ERR(bo)) xe_bo_unlock_vm_held(bo); @@ -1277,7 +1278,7 @@ struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_gt *gt, return bo; } -struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, u64 offset, enum ttm_bo_type type, u32 flags) @@ -1291,7 +1292,7 @@ struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_gt *gt, xe_ttm_stolen_cpu_access_needs_ggtt(xe)) flags |= XE_BO_CREATE_GGTT_BIT; - bo = xe_bo_create_locked_range(xe, gt, vm, size, start, end, type, flags); + bo = xe_bo_create_locked_range(xe, tile, vm, size, start, end, type, flags); if (IS_ERR(bo)) return bo; @@ -1315,18 +1316,18 @@ err_put: return ERR_PTR(err); } -struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags) { - return xe_bo_create_pin_map_at(xe, gt, vm, size, ~0ull, type, flags); + return xe_bo_create_pin_map_at(xe, tile, vm, size, ~0ull, type, flags); } -struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile, const void *data, size_t size, enum ttm_bo_type type, u32 flags) { - struct xe_bo *bo = xe_bo_create_pin_map(xe, gt, NULL, + struct xe_bo *bo = xe_bo_create_pin_map(xe, tile, NULL, ALIGN(size, PAGE_SIZE), type, flags); if (IS_ERR(bo)) @@ -1957,7 +1958,7 @@ int xe_bo_dumb_create(struct drm_file *file_priv, page_size); bo = xe_bo_create(xe, NULL, NULL, args->size, ttm_bo_type_device, - XE_BO_CREATE_VRAM_IF_DGFX(to_gt(xe)) | + XE_BO_CREATE_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | XE_BO_CREATE_USER_BIT | XE_BO_SCANOUT_BIT); if (IS_ERR(bo)) return PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 6e29e45a90f2..29eb7474f018 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -21,8 +21,8 @@ XE_BO_CREATE_VRAM1_BIT) /* -- */ #define XE_BO_CREATE_STOLEN_BIT BIT(4) -#define XE_BO_CREATE_VRAM_IF_DGFX(gt) \ - (IS_DGFX(gt_to_xe(gt)) ? XE_BO_CREATE_VRAM0_BIT << gt_to_tile(gt)->id : \ +#define XE_BO_CREATE_VRAM_IF_DGFX(tile) \ + (IS_DGFX(tile_to_xe(tile)) ? XE_BO_CREATE_VRAM0_BIT << (tile)->id : \ XE_BO_CREATE_SYSTEM_BIT) #define XE_BO_CREATE_GGTT_BIT BIT(5) #define XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT BIT(6) @@ -81,27 +81,27 @@ struct xe_bo *xe_bo_alloc(void); void xe_bo_free(struct xe_bo *bo); struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, - struct xe_gt *gt, struct dma_resv *resv, + struct xe_tile *tile, struct dma_resv *resv, size_t size, enum ttm_bo_type type, u32 flags); struct xe_bo * xe_bo_create_locked_range(struct xe_device *xe, - struct xe_gt *gt, struct xe_vm *vm, + struct xe_tile *tile, struct xe_vm *vm, size_t size, u64 start, u64 end, enum ttm_bo_type type, u32 flags); -struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags); -struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags); -struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags); -struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create_pin_map_at(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, u64 offset, enum ttm_bo_type type, u32 flags); -struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_gt *gt, +struct xe_bo *xe_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile, const void *data, size_t size, enum ttm_bo_type type, u32 flags); diff --git a/drivers/gpu/drm/xe/xe_bo_evict.c b/drivers/gpu/drm/xe/xe_bo_evict.c index a72963c54bf3..9226195bd560 100644 --- a/drivers/gpu/drm/xe/xe_bo_evict.c +++ b/drivers/gpu/drm/xe/xe_bo_evict.c @@ -149,7 +149,7 @@ int xe_bo_restore_kernel(struct xe_device *xe) } if (bo->flags & XE_BO_CREATE_GGTT_BIT) { - struct xe_tile *tile = gt_to_tile(bo->gt); + struct xe_tile *tile = bo->tile; mutex_lock(&tile->mem.ggtt->lock); xe_ggtt_map_bo(tile->mem.ggtt, bo); diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index 06de3330211d..f6ee920303af 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -29,8 +29,8 @@ struct xe_bo { u32 flags; /** @vm: VM this BO is attached to, for extobj this will be NULL */ struct xe_vm *vm; - /** @gt: GT this BO is attached to (kernel BO only) */ - struct xe_gt *gt; + /** @tile: Tile this BO is attached to (kernel BO only) */ + struct xe_tile *tile; /** @vmas: List of VMAs for this BO */ struct list_head vmas; /** @placements: valid placements for this BO */ diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 9382d7f62f03..ee050b4b4d77 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -128,6 +128,13 @@ struct xe_tile { /** @ggtt: Global graphics translation table */ struct xe_ggtt *ggtt; + + /** + * @kernel_bb_pool: Pool from which batchbuffers are allocated. + * + * Media GT shares a pool with its primary GT. + */ + struct xe_sa_manager *kernel_bb_pool; } mem; }; diff --git a/drivers/gpu/drm/xe/xe_ggtt.c b/drivers/gpu/drm/xe/xe_ggtt.c index ff70a01f1591..d395d6fc1af6 100644 --- a/drivers/gpu/drm/xe/xe_ggtt.c +++ b/drivers/gpu/drm/xe/xe_ggtt.c @@ -151,7 +151,6 @@ static void xe_ggtt_initial_clear(struct xe_ggtt *ggtt) int xe_ggtt_init(struct xe_ggtt *ggtt) { struct xe_device *xe = tile_to_xe(ggtt->tile); - struct xe_gt *gt = &ggtt->tile->primary_gt; unsigned int flags; int err; @@ -164,9 +163,9 @@ int xe_ggtt_init(struct xe_ggtt *ggtt) if (ggtt->flags & XE_GGTT_FLAGS_64K) flags |= XE_BO_CREATE_SYSTEM_BIT; else - flags |= XE_BO_CREATE_VRAM_IF_DGFX(gt); + flags |= XE_BO_CREATE_VRAM_IF_DGFX(ggtt->tile); - ggtt->scratch = xe_bo_create_pin_map(xe, gt, NULL, XE_PAGE_SIZE, + ggtt->scratch = xe_bo_create_pin_map(xe, ggtt->tile, NULL, XE_PAGE_SIZE, ttm_bo_type_kernel, flags); diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 419fc471053c..74023a5dc8b2 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -95,7 +95,7 @@ static int emit_nop_job(struct xe_gt *gt, struct xe_engine *e) if (IS_ERR(bb)) return PTR_ERR(bb); - batch_ofs = xe_bo_ggtt_addr(gt->kernel_bb_pool->bo); + batch_ofs = xe_bo_ggtt_addr(gt_to_tile(gt)->mem.kernel_bb_pool->bo); job = xe_bb_create_wa_job(e, bb, batch_ofs); if (IS_ERR(job)) { xe_bb_free(bb, NULL); @@ -144,7 +144,7 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_engine *e) } } - batch_ofs = xe_bo_ggtt_addr(gt->kernel_bb_pool->bo); + batch_ofs = xe_bo_ggtt_addr(gt_to_tile(gt)->mem.kernel_bb_pool->bo); job = xe_bb_create_wa_job(e, bb, batch_ofs); if (IS_ERR(job)) { xe_bb_free(bb, NULL); @@ -370,31 +370,16 @@ static int all_fw_domain_init(struct xe_gt *gt) goto err_force_wake; if (!xe_gt_is_media_type(gt)) { - gt->kernel_bb_pool = xe_sa_bo_manager_init(gt, SZ_1M, 16); - if (IS_ERR(gt->kernel_bb_pool)) { - err = PTR_ERR(gt->kernel_bb_pool); - goto err_force_wake; - } - /* * USM has its only SA pool to non-block behind user operations */ if (gt_to_xe(gt)->info.supports_usm) { - gt->usm.bb_pool = xe_sa_bo_manager_init(gt, SZ_1M, 16); + gt->usm.bb_pool = xe_sa_bo_manager_init(gt_to_tile(gt), SZ_1M, 16); if (IS_ERR(gt->usm.bb_pool)) { err = PTR_ERR(gt->usm.bb_pool); goto err_force_wake; } } - } else { - struct xe_gt *full_gt = xe_find_full_gt(gt); - - /* - * Media GT's kernel_bb_pool is only used while recording the - * default context during GT init. The USM pool should never - * be needed on the media GT. - */ - gt->kernel_bb_pool = full_gt->kernel_bb_pool; } if (!xe_gt_is_media_type(gt)) { diff --git a/drivers/gpu/drm/xe/xe_gt_debugfs.c b/drivers/gpu/drm/xe/xe_gt_debugfs.c index 1114254bc519..b5a5538ae630 100644 --- a/drivers/gpu/drm/xe/xe_gt_debugfs.c +++ b/drivers/gpu/drm/xe/xe_gt_debugfs.c @@ -64,11 +64,11 @@ static int force_reset(struct seq_file *m, void *data) static int sa_info(struct seq_file *m, void *data) { - struct xe_gt *gt = node_to_gt(m->private); + struct xe_tile *tile = gt_to_tile(node_to_gt(m->private)); struct drm_printer p = drm_seq_file_printer(m); - drm_suballoc_dump_debug_info(>->kernel_bb_pool->base, &p, - gt->kernel_bb_pool->gpu_addr); + drm_suballoc_dump_debug_info(&tile->mem.kernel_bb_pool->base, &p, + tile->mem.kernel_bb_pool->gpu_addr); return 0; } diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c index f4f3d95ae6b1..1ec140aaf2a7 100644 --- a/drivers/gpu/drm/xe/xe_gt_pagefault.c +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c @@ -69,10 +69,10 @@ static bool access_is_atomic(enum access_type access_type) return access_type == ACCESS_TYPE_ATOMIC; } -static bool vma_is_valid(struct xe_gt *gt, struct xe_vma *vma) +static bool vma_is_valid(struct xe_tile *tile, struct xe_vma *vma) { - return BIT(gt->info.id) & vma->gt_present && - !(BIT(gt->info.id) & vma->usm.gt_invalidated); + return BIT(tile->id) & vma->tile_present && + !(BIT(tile->id) & vma->usm.tile_invalidated); } static bool vma_matches(struct xe_vma *vma, struct xe_vma *lookup) @@ -152,7 +152,7 @@ retry_userptr: atomic = access_is_atomic(pf->access_type); /* Check if VMA is valid */ - if (vma_is_valid(gt, vma) && !atomic) + if (vma_is_valid(tile, vma) && !atomic) goto unlock_vm; /* TODO: Validate fault */ @@ -208,8 +208,8 @@ retry_userptr: /* Bind VMA only to the GT that has faulted */ trace_xe_vma_pf_bind(vma); - fence = __xe_pt_bind_vma(gt, vma, xe_gt_migrate_engine(gt), NULL, 0, - vma->gt_present & BIT(gt->info.id)); + fence = __xe_pt_bind_vma(tile, vma, xe_gt_migrate_engine(gt), NULL, 0, + vma->tile_present & BIT(tile->id)); if (IS_ERR(fence)) { ret = PTR_ERR(fence); goto unlock_dma_resv; @@ -225,7 +225,7 @@ retry_userptr: if (xe_vma_is_userptr(vma)) ret = xe_vma_userptr_check_repin(vma); - vma->usm.gt_invalidated &= ~BIT(gt->info.id); + vma->usm.tile_invalidated &= ~BIT(tile->id); unlock_dma_resv: if (only_needs_bo_lock(bo)) diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index a040ec896e70..c44560b6dc71 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -278,13 +278,6 @@ struct xe_gt { /** @hw_engines: hardware engines on the GT */ struct xe_hw_engine hw_engines[XE_NUM_HW_ENGINES]; - /** - * @kernel_bb_pool: Pool from which batchbuffers are allocated. - * - * Media GT shares a pool with its primary GT. - */ - struct xe_sa_manager *kernel_bb_pool; - /** @migrate: Migration helper for vram blits and clearing */ struct xe_migrate *migrate; diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index 6d550d746909..dd69d097b920 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -273,16 +273,17 @@ int xe_guc_ads_init(struct xe_guc_ads *ads) { struct xe_device *xe = ads_to_xe(ads); struct xe_gt *gt = ads_to_gt(ads); + struct xe_tile *tile = gt_to_tile(gt); struct xe_bo *bo; int err; ads->golden_lrc_size = calculate_golden_lrc_size(ads); ads->regset_size = calculate_regset_size(gt); - bo = xe_bo_create_pin_map(xe, gt, NULL, guc_ads_size(ads) + + bo = xe_bo_create_pin_map(xe, tile, NULL, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(bo)) return PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 9dc906f2651a..137c184df487 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -130,6 +130,7 @@ int xe_guc_ct_init(struct xe_guc_ct *ct) { struct xe_device *xe = ct_to_xe(ct); struct xe_gt *gt = ct_to_gt(ct); + struct xe_tile *tile = gt_to_tile(gt); struct xe_bo *bo; int err; @@ -145,9 +146,9 @@ int xe_guc_ct_init(struct xe_guc_ct *ct) primelockdep(ct); - bo = xe_bo_create_pin_map(xe, gt, NULL, guc_ct_size(), + bo = xe_bo_create_pin_map(xe, tile, NULL, guc_ct_size(), ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(bo)) return PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_guc_hwconfig.c b/drivers/gpu/drm/xe/xe_guc_hwconfig.c index a6982f323ed1..c8f875e970ab 100644 --- a/drivers/gpu/drm/xe/xe_guc_hwconfig.c +++ b/drivers/gpu/drm/xe/xe_guc_hwconfig.c @@ -70,6 +70,7 @@ int xe_guc_hwconfig_init(struct xe_guc *guc) { struct xe_device *xe = guc_to_xe(guc); struct xe_gt *gt = guc_to_gt(guc); + struct xe_tile *tile = gt_to_tile(gt); struct xe_bo *bo; u32 size; int err; @@ -94,9 +95,9 @@ int xe_guc_hwconfig_init(struct xe_guc *guc) if (!size) return -EINVAL; - bo = xe_bo_create_pin_map(xe, gt, NULL, PAGE_ALIGN(size), + bo = xe_bo_create_pin_map(xe, tile, NULL, PAGE_ALIGN(size), ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(bo)) return PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_guc_log.c b/drivers/gpu/drm/xe/xe_guc_log.c index 9a7b5d5906c1..403aaafcaba6 100644 --- a/drivers/gpu/drm/xe/xe_guc_log.c +++ b/drivers/gpu/drm/xe/xe_guc_log.c @@ -87,13 +87,13 @@ static void guc_log_fini(struct drm_device *drm, void *arg) int xe_guc_log_init(struct xe_guc_log *log) { struct xe_device *xe = log_to_xe(log); - struct xe_gt *gt = log_to_gt(log); + struct xe_tile *tile = gt_to_tile(log_to_gt(log)); struct xe_bo *bo; int err; - bo = xe_bo_create_pin_map(xe, gt, NULL, guc_log_size(), + bo = xe_bo_create_pin_map(xe, tile, NULL, guc_log_size(), ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(bo)) return PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index e799faa1c6b8..67faa9ee0006 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -888,6 +888,7 @@ static void pc_fini(struct drm_device *drm, void *arg) int xe_guc_pc_init(struct xe_guc_pc *pc) { struct xe_gt *gt = pc_to_gt(pc); + struct xe_tile *tile = gt_to_tile(gt); struct xe_device *xe = gt_to_xe(gt); struct xe_bo *bo; u32 size = PAGE_ALIGN(sizeof(struct slpc_shared_data)); @@ -895,9 +896,9 @@ int xe_guc_pc_init(struct xe_guc_pc *pc) mutex_init(&pc->freq_lock); - bo = xe_bo_create_pin_map(xe, gt, NULL, size, + bo = xe_bo_create_pin_map(xe, tile, NULL, size, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(bo)) diff --git a/drivers/gpu/drm/xe/xe_hw_engine.c b/drivers/gpu/drm/xe/xe_hw_engine.c index 7e4b0b465244..b12f65a2bab3 100644 --- a/drivers/gpu/drm/xe/xe_hw_engine.c +++ b/drivers/gpu/drm/xe/xe_hw_engine.c @@ -373,6 +373,7 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, enum xe_hw_engine_id id) { struct xe_device *xe = gt_to_xe(gt); + struct xe_tile *tile = gt_to_tile(gt); int err; XE_BUG_ON(id >= ARRAY_SIZE(engine_infos) || !engine_infos[id].name); @@ -381,8 +382,8 @@ static int hw_engine_init(struct xe_gt *gt, struct xe_hw_engine *hwe, xe_reg_sr_apply_mmio(&hwe->reg_sr, gt); xe_reg_sr_apply_whitelist(&hwe->reg_whitelist, hwe->mmio_base, gt); - hwe->hwsp = xe_bo_create_pin_map(xe, gt, NULL, SZ_4K, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + hwe->hwsp = xe_bo_create_pin_map(xe, tile, NULL, SZ_4K, ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(hwe->hwsp)) { err = PTR_ERR(hwe->hwsp); diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index ae605e7805de..8f25a38f36a5 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -592,7 +592,7 @@ static void *empty_lrc_data(struct xe_hw_engine *hwe) static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm) { - u64 desc = xe_vm_pdp4_descriptor(vm, lrc->full_gt); + u64 desc = xe_vm_pdp4_descriptor(vm, lrc->tile); xe_lrc_write_ctx_reg(lrc, CTX_PDP0_UDW, upper_32_bits(desc)); xe_lrc_write_ctx_reg(lrc, CTX_PDP0_LDW, lower_32_bits(desc)); @@ -607,6 +607,7 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, struct xe_engine *e, struct xe_vm *vm, u32 ring_size) { struct xe_gt *gt = hwe->gt; + struct xe_tile *tile = gt_to_tile(gt); struct xe_device *xe = gt_to_xe(gt); struct iosys_map map; void *init_data = NULL; @@ -619,19 +620,15 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, * FIXME: Perma-pinning LRC as we don't yet support moving GGTT address * via VM bind calls. */ - lrc->bo = xe_bo_create_pin_map(xe, hwe->gt, vm, + lrc->bo = xe_bo_create_pin_map(xe, tile, vm, ring_size + xe_lrc_size(xe, hwe->class), ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(hwe->gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(lrc->bo)) return PTR_ERR(lrc->bo); - if (xe_gt_is_media_type(hwe->gt)) - lrc->full_gt = xe_find_full_gt(hwe->gt); - else - lrc->full_gt = hwe->gt; - + lrc->tile = gt_to_tile(hwe->gt); lrc->ring.size = ring_size; lrc->ring.tail = 0; diff --git a/drivers/gpu/drm/xe/xe_lrc_types.h b/drivers/gpu/drm/xe/xe_lrc_types.h index 8fe08535873d..78220336062c 100644 --- a/drivers/gpu/drm/xe/xe_lrc_types.h +++ b/drivers/gpu/drm/xe/xe_lrc_types.h @@ -20,8 +20,8 @@ struct xe_lrc { */ struct xe_bo *bo; - /** @full_gt: full GT which this LRC belongs to */ - struct xe_gt *full_gt; + /** @tile: tile which this LRC belongs to */ + struct xe_tile *tile; /** @flags: LRC flags */ u32 flags; diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 7a2188f02a86..3031a45db490 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -129,6 +129,7 @@ static u64 xe_migrate_vram_ofs(u64 addr) static int xe_migrate_create_cleared_bo(struct xe_migrate *m, struct xe_vm *vm) { struct xe_gt *gt = m->gt; + struct xe_tile *tile = gt_to_tile(gt); struct xe_device *xe = vm->xe; size_t cleared_size; u64 vram_addr; @@ -139,9 +140,9 @@ static int xe_migrate_create_cleared_bo(struct xe_migrate *m, struct xe_vm *vm) cleared_size = xe_device_ccs_bytes(xe, MAX_PREEMPTDISABLE_TRANSFER); cleared_size = PAGE_ALIGN(cleared_size); - m->cleared_bo = xe_bo_create_pin_map(xe, gt, vm, cleared_size, + m->cleared_bo = xe_bo_create_pin_map(xe, tile, vm, cleared_size, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_PINNED_BIT); if (IS_ERR(m->cleared_bo)) return PTR_ERR(m->cleared_bo); @@ -161,7 +162,8 @@ static int xe_migrate_prepare_vm(struct xe_gt *gt, struct xe_migrate *m, u32 num_entries = NUM_PT_SLOTS, num_level = vm->pt_root[id]->level; u32 map_ofs, level, i; struct xe_device *xe = gt_to_xe(m->gt); - struct xe_bo *bo, *batch = gt->kernel_bb_pool->bo; + struct xe_tile *tile = gt_to_tile(m->gt); + struct xe_bo *bo, *batch = tile->mem.kernel_bb_pool->bo; u64 entry; int ret; @@ -175,10 +177,10 @@ static int xe_migrate_prepare_vm(struct xe_gt *gt, struct xe_migrate *m, /* Need to be sure everything fits in the first PT, or create more */ XE_BUG_ON(m->batch_base_ofs + batch->size >= SZ_2M); - bo = xe_bo_create_pin_map(vm->xe, m->gt, vm, + bo = xe_bo_create_pin_map(vm->xe, tile, vm, num_entries * XE_PAGE_SIZE, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(m->gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_PINNED_BIT); if (IS_ERR(bo)) return PTR_ERR(bo); @@ -984,7 +986,7 @@ err_sync: return fence; } -static void write_pgtable(struct xe_gt *gt, struct xe_bb *bb, u64 ppgtt_ofs, +static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, const struct xe_vm_pgtable_update *update, struct xe_migrate_pt_update *pt_update) { @@ -1023,7 +1025,7 @@ static void write_pgtable(struct xe_gt *gt, struct xe_bb *bb, u64 ppgtt_ofs, (chunk * 2 + 1); bb->cs[bb->len++] = lower_32_bits(addr); bb->cs[bb->len++] = upper_32_bits(addr); - ops->populate(pt_update, gt, NULL, bb->cs + bb->len, ofs, chunk, + ops->populate(pt_update, tile, NULL, bb->cs + bb->len, ofs, chunk, update); bb->len += chunk * 2; @@ -1081,7 +1083,7 @@ xe_migrate_update_pgtables_cpu(struct xe_migrate *m, for (i = 0; i < num_updates; i++) { const struct xe_vm_pgtable_update *update = &updates[i]; - ops->populate(pt_update, m->gt, &update->pt_bo->vmap, NULL, + ops->populate(pt_update, gt_to_tile(m->gt), &update->pt_bo->vmap, NULL, update->ofs, update->qwords, update); } @@ -1149,6 +1151,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m, { const struct xe_migrate_pt_update_ops *ops = pt_update->ops; struct xe_gt *gt = m->gt; + struct xe_tile *tile = gt_to_tile(m->gt); struct xe_device *xe = gt_to_xe(gt); struct xe_sched_job *job; struct dma_fence *fence; @@ -1243,7 +1246,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m, addr = xe_migrate_vm_addr(ppgtt_ofs, 0) + (page_ofs / sizeof(u64)) * XE_PAGE_SIZE; for (i = 0; i < num_updates; i++) - write_pgtable(m->gt, bb, addr + i * XE_PAGE_SIZE, + write_pgtable(tile, bb, addr + i * XE_PAGE_SIZE, &updates[i], pt_update); } else { /* phys pages, no preamble required */ @@ -1253,7 +1256,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m, /* Preemption is enabled again by the ring ops. */ emit_arb_clear(bb); for (i = 0; i < num_updates; i++) - write_pgtable(m->gt, bb, 0, &updates[i], pt_update); + write_pgtable(tile, bb, 0, &updates[i], pt_update); } if (!eng) diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index c283b626c21c..e627f4023d5a 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -19,6 +19,7 @@ struct xe_migrate; struct xe_migrate_pt_update; struct xe_sync_entry; struct xe_pt; +struct xe_tile; struct xe_vm; struct xe_vm_pgtable_update; struct xe_vma; @@ -31,7 +32,7 @@ struct xe_migrate_pt_update_ops { /** * @populate: Populate a command buffer or page-table with ptes. * @pt_update: Embeddable callback argument. - * @gt: The gt for the current operation. + * @tile: The tile for the current operation. * @map: struct iosys_map into the memory to be populated. * @pos: If @map is NULL, map into the memory to be populated. * @ofs: qword offset into @map, unused if @map is NULL. @@ -43,7 +44,7 @@ struct xe_migrate_pt_update_ops { * page-tables with PTEs. */ void (*populate)(struct xe_migrate_pt_update *pt_update, - struct xe_gt *gt, struct iosys_map *map, + struct xe_tile *tile, struct iosys_map *map, void *pos, u32 ofs, u32 num_qwords, const struct xe_vm_pgtable_update *update); diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index e2cd1946af5a..094058cb5f93 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -165,12 +165,10 @@ u64 gen8_pte_encode(struct xe_vma *vma, struct xe_bo *bo, return __gen8_pte_encode(pte, cache, flags, pt_level); } -static u64 __xe_pt_empty_pte(struct xe_gt *gt, struct xe_vm *vm, +static u64 __xe_pt_empty_pte(struct xe_tile *tile, struct xe_vm *vm, unsigned int level) { - u8 id = gt->info.id; - - XE_BUG_ON(xe_gt_is_media_type(gt)); + u8 id = tile->id; if (!vm->scratch_bo[id]) return 0; @@ -189,7 +187,7 @@ static u64 __xe_pt_empty_pte(struct xe_gt *gt, struct xe_vm *vm, /** * xe_pt_create() - Create a page-table. * @vm: The vm to create for. - * @gt: The gt to create for. + * @tile: The tile to create for. * @level: The page-table level. * * Allocate and initialize a single struct xe_pt metadata structure. Also @@ -201,7 +199,7 @@ static u64 __xe_pt_empty_pte(struct xe_gt *gt, struct xe_vm *vm, * Return: A valid struct xe_pt pointer on success, Pointer error code on * error. */ -struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_gt *gt, +struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile, unsigned int level) { struct xe_pt *pt; @@ -215,9 +213,9 @@ struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_gt *gt, if (!pt) return ERR_PTR(-ENOMEM); - bo = xe_bo_create_pin_map(vm->xe, gt, vm, SZ_4K, + bo = xe_bo_create_pin_map(vm->xe, tile, vm, SZ_4K, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT | XE_BO_CREATE_PINNED_BIT | XE_BO_CREATE_NO_RESV_EVICT); @@ -241,30 +239,28 @@ err_kfree: /** * xe_pt_populate_empty() - Populate a page-table bo with scratch- or zero * entries. - * @gt: The gt the scratch pagetable of which to use. + * @tile: The tile the scratch pagetable of which to use. * @vm: The vm we populate for. * @pt: The pagetable the bo of which to initialize. * - * Populate the page-table bo of @pt with entries pointing into the gt's + * Populate the page-table bo of @pt with entries pointing into the tile's * scratch page-table tree if any. Otherwise populate with zeros. */ -void xe_pt_populate_empty(struct xe_gt *gt, struct xe_vm *vm, +void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm, struct xe_pt *pt) { struct iosys_map *map = &pt->bo->vmap; u64 empty; int i; - XE_BUG_ON(xe_gt_is_media_type(gt)); - - if (!vm->scratch_bo[gt->info.id]) { + if (!vm->scratch_bo[tile->id]) { /* * FIXME: Some memory is allocated already allocated to zero? * Find out which memory that is and avoid this memset... */ xe_map_memset(vm->xe, map, 0, 0, SZ_4K); } else { - empty = __xe_pt_empty_pte(gt, vm, pt->level); + empty = __xe_pt_empty_pte(tile, vm, pt->level); for (i = 0; i < XE_PDES; i++) xe_pt_write(vm->xe, map, i, empty); } @@ -318,9 +314,9 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) /** * xe_pt_create_scratch() - Setup a scratch memory pagetable tree for the - * given gt and vm. + * given tile and vm. * @xe: xe device. - * @gt: gt to set up for. + * @tile: tile to set up for. * @vm: vm to set up for. * * Sets up a pagetable tree with one page-table per level and a single @@ -329,10 +325,10 @@ void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred) * * Return: 0 on success, negative error code on error. */ -int xe_pt_create_scratch(struct xe_device *xe, struct xe_gt *gt, +int xe_pt_create_scratch(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm) { - u8 id = gt->info.id; + u8 id = tile->id; unsigned int flags; int i; @@ -345,9 +341,9 @@ int xe_pt_create_scratch(struct xe_device *xe, struct xe_gt *gt, if (vm->flags & XE_VM_FLAGS_64K) flags |= XE_BO_CREATE_SYSTEM_BIT; else - flags |= XE_BO_CREATE_VRAM_IF_DGFX(gt); + flags |= XE_BO_CREATE_VRAM_IF_DGFX(tile); - vm->scratch_bo[id] = xe_bo_create_pin_map(xe, gt, vm, SZ_4K, + vm->scratch_bo[id] = xe_bo_create_pin_map(xe, tile, vm, SZ_4K, ttm_bo_type_kernel, flags); if (IS_ERR(vm->scratch_bo[id])) @@ -357,11 +353,11 @@ int xe_pt_create_scratch(struct xe_device *xe, struct xe_gt *gt, vm->scratch_bo[id]->size); for (i = 0; i < vm->pt_root[id]->level; i++) { - vm->scratch_pt[id][i] = xe_pt_create(vm, gt, i); + vm->scratch_pt[id][i] = xe_pt_create(vm, tile, i); if (IS_ERR(vm->scratch_pt[id][i])) return PTR_ERR(vm->scratch_pt[id][i]); - xe_pt_populate_empty(gt, vm, vm->scratch_pt[id][i]); + xe_pt_populate_empty(tile, vm, vm->scratch_pt[id][i]); } return 0; @@ -410,8 +406,8 @@ struct xe_pt_stage_bind_walk { /* Input parameters for the walk */ /** @vm: The vm we're building for. */ struct xe_vm *vm; - /** @gt: The gt we're building for. */ - struct xe_gt *gt; + /** @tile: The tile we're building for. */ + struct xe_tile *tile; /** @cache: Desired cache level for the ptes */ enum xe_cache_level cache; /** @default_pte: PTE flag only template. No address is associated */ @@ -679,7 +675,7 @@ xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, if (covers || !*child) { u64 flags = 0; - xe_child = xe_pt_create(xe_walk->vm, xe_walk->gt, level - 1); + xe_child = xe_pt_create(xe_walk->vm, xe_walk->tile, level - 1); if (IS_ERR(xe_child)) return PTR_ERR(xe_child); @@ -687,7 +683,7 @@ xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, round_down(addr, 1ull << walk->shifts[level])); if (!covers) - xe_pt_populate_empty(xe_walk->gt, xe_walk->vm, xe_child); + xe_pt_populate_empty(xe_walk->tile, xe_walk->vm, xe_child); *child = &xe_child->base; @@ -696,7 +692,7 @@ xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, * TODO: Suballocate the pt bo to avoid wasting a lot of * memory. */ - if (GRAPHICS_VERx100(gt_to_xe(xe_walk->gt)) >= 1250 && level == 1 && + if (GRAPHICS_VERx100(tile_to_xe(xe_walk->tile)) >= 1250 && level == 1 && covers && xe_pt_scan_64K(addr, next, xe_walk)) { walk->shifts = xe_compact_pt_shifts; flags |= XE_PDE_64K; @@ -719,7 +715,7 @@ static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = { /** * xe_pt_stage_bind() - Build a disconnected page-table tree for a given address * range. - * @gt: The gt we're building for. + * @tile: The tile we're building for. * @vma: The vma indicating the address range. * @entries: Storage for the update entries used for connecting the tree to * the main tree at commit time. @@ -735,7 +731,7 @@ static const struct xe_pt_walk_ops xe_pt_stage_bind_ops = { * Return 0 on success, negative error code on error. */ static int -xe_pt_stage_bind(struct xe_gt *gt, struct xe_vma *vma, +xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, struct xe_vm_pgtable_update *entries, u32 *num_entries) { struct xe_bo *bo = vma->bo; @@ -748,14 +744,14 @@ xe_pt_stage_bind(struct xe_gt *gt, struct xe_vma *vma, .max_level = XE_PT_HIGHEST_LEVEL, }, .vm = vma->vm, - .gt = gt, + .tile = tile, .curs = &curs, .va_curs_start = vma->start, .pte_flags = vma->pte_flags, .wupd.entries = entries, .needs_64K = (vma->vm->flags & XE_VM_FLAGS_64K) && is_vram, }; - struct xe_pt *pt = vma->vm->pt_root[gt->info.id]; + struct xe_pt *pt = vma->vm->pt_root[tile->id]; int ret; if (is_vram) { @@ -849,8 +845,8 @@ struct xe_pt_zap_ptes_walk { struct xe_pt_walk base; /* Input parameters for the walk */ - /** @gt: The gt we're building for */ - struct xe_gt *gt; + /** @tile: The tile we're building for */ + struct xe_tile *tile; /* Output */ /** @needs_invalidate: Whether we need to invalidate TLB*/ @@ -878,7 +874,7 @@ static int xe_pt_zap_ptes_entry(struct xe_ptw *parent, pgoff_t offset, */ if (xe_pt_nonshared_offsets(addr, next, --level, walk, action, &offset, &end_offset)) { - xe_map_memset(gt_to_xe(xe_walk->gt), &xe_child->bo->vmap, + xe_map_memset(tile_to_xe(xe_walk->tile), &xe_child->bo->vmap, offset * sizeof(u64), 0, (end_offset - offset) * sizeof(u64)); xe_walk->needs_invalidate = true; @@ -893,7 +889,7 @@ static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = { /** * xe_pt_zap_ptes() - Zap (zero) gpu ptes of an address range - * @gt: The gt we're zapping for. + * @tile: The tile we're zapping for. * @vma: GPU VMA detailing address range. * * Eviction and Userptr invalidation needs to be able to zap the @@ -907,7 +903,7 @@ static const struct xe_pt_walk_ops xe_pt_zap_ptes_ops = { * Return: Whether ptes were actually updated and a TLB invalidation is * required. */ -bool xe_pt_zap_ptes(struct xe_gt *gt, struct xe_vma *vma) +bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma) { struct xe_pt_zap_ptes_walk xe_walk = { .base = { @@ -915,11 +911,11 @@ bool xe_pt_zap_ptes(struct xe_gt *gt, struct xe_vma *vma) .shifts = xe_normal_pt_shifts, .max_level = XE_PT_HIGHEST_LEVEL, }, - .gt = gt, + .tile = tile, }; - struct xe_pt *pt = vma->vm->pt_root[gt->info.id]; + struct xe_pt *pt = vma->vm->pt_root[tile->id]; - if (!(vma->gt_present & BIT(gt->info.id))) + if (!(vma->tile_present & BIT(tile->id))) return false; (void)xe_pt_walk_shared(&pt->base, pt->level, vma->start, vma->end + 1, @@ -929,7 +925,7 @@ bool xe_pt_zap_ptes(struct xe_gt *gt, struct xe_vma *vma) } static void -xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_gt *gt, +xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_tile *tile, struct iosys_map *map, void *data, u32 qword_ofs, u32 num_qwords, const struct xe_vm_pgtable_update *update) @@ -938,11 +934,9 @@ xe_vm_populate_pgtable(struct xe_migrate_pt_update *pt_update, struct xe_gt *gt, u64 *ptr = data; u32 i; - XE_BUG_ON(xe_gt_is_media_type(gt)); - for (i = 0; i < num_qwords; i++) { if (map) - xe_map_wr(gt_to_xe(gt), map, (qword_ofs + i) * + xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * sizeof(u64), u64, ptes[i].pte); else ptr[i] = ptes[i].pte; @@ -1016,14 +1010,14 @@ static void xe_pt_commit_bind(struct xe_vma *vma, } static int -xe_pt_prepare_bind(struct xe_gt *gt, struct xe_vma *vma, +xe_pt_prepare_bind(struct xe_tile *tile, struct xe_vma *vma, struct xe_vm_pgtable_update *entries, u32 *num_entries, bool rebind) { int err; *num_entries = 0; - err = xe_pt_stage_bind(gt, vma, entries, num_entries); + err = xe_pt_stage_bind(tile, vma, entries, num_entries); if (!err) BUG_ON(!*num_entries); else /* abort! */ @@ -1250,7 +1244,7 @@ static int invalidation_fence_init(struct xe_gt *gt, /** * __xe_pt_bind_vma() - Build and connect a page-table tree for the vma * address range. - * @gt: The gt to bind for. + * @tile: The tile to bind for. * @vma: The vma to bind. * @e: The engine with which to do pipelined page-table updates. * @syncs: Entries to sync on before binding the built tree to the live vm tree. @@ -1270,7 +1264,7 @@ static int invalidation_fence_init(struct xe_gt *gt, * on success, an error pointer on error. */ struct dma_fence * -__xe_pt_bind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, +__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, struct xe_sync_entry *syncs, u32 num_syncs, bool rebind) { @@ -1291,18 +1285,17 @@ __xe_pt_bind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, bind_pt_update.locked = false; xe_bo_assert_held(vma->bo); xe_vm_assert_held(vm); - XE_BUG_ON(xe_gt_is_media_type(gt)); vm_dbg(&vma->vm->xe->drm, "Preparing bind, with range [%llx...%llx) engine %p.\n", vma->start, vma->end, e); - err = xe_pt_prepare_bind(gt, vma, entries, &num_entries, rebind); + err = xe_pt_prepare_bind(tile, vma, entries, &num_entries, rebind); if (err) goto err; XE_BUG_ON(num_entries > ARRAY_SIZE(entries)); - xe_vm_dbg_print_entries(gt_to_xe(gt), entries, num_entries); + xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries); if (rebind && !xe_vm_no_dma_fences(vma->vm)) { ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); @@ -1310,9 +1303,9 @@ __xe_pt_bind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, return ERR_PTR(-ENOMEM); } - fence = xe_migrate_update_pgtables(gt->migrate, + fence = xe_migrate_update_pgtables(tile->primary_gt.migrate, vm, vma->bo, - e ? e : vm->eng[gt->info.id], + e ? e : vm->eng[tile->id], entries, num_entries, syncs, num_syncs, &bind_pt_update.base); @@ -1321,7 +1314,7 @@ __xe_pt_bind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, /* TLB invalidation must be done before signaling rebind */ if (rebind && !xe_vm_no_dma_fences(vma->vm)) { - int err = invalidation_fence_init(gt, ifence, fence, + int err = invalidation_fence_init(&tile->primary_gt, ifence, fence, vma); if (err) { dma_fence_put(fence); @@ -1344,7 +1337,7 @@ __xe_pt_bind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, bind_pt_update.locked ? &deferred : NULL); /* This vma is live (again?) now */ - vma->gt_present |= BIT(gt->info.id); + vma->tile_present |= BIT(tile->id); if (bind_pt_update.locked) { vma->userptr.initial_bind = true; @@ -1373,8 +1366,8 @@ struct xe_pt_stage_unbind_walk { struct xe_pt_walk base; /* Input parameters for the walk */ - /** @gt: The gt we're unbinding from. */ - struct xe_gt *gt; + /** @tile: The tile we're unbinding from. */ + struct xe_tile *tile; /** * @modified_start: Walk range start, modified to include any @@ -1479,7 +1472,7 @@ static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = { /** * xe_pt_stage_unbind() - Build page-table update structures for an unbind * operation - * @gt: The gt we're unbinding for. + * @tile: The tile we're unbinding for. * @vma: The vma we're unbinding. * @entries: Caller-provided storage for the update structures. * @@ -1490,7 +1483,7 @@ static const struct xe_pt_walk_ops xe_pt_stage_unbind_ops = { * * Return: The number of entries used. */ -static unsigned int xe_pt_stage_unbind(struct xe_gt *gt, struct xe_vma *vma, +static unsigned int xe_pt_stage_unbind(struct xe_tile *tile, struct xe_vma *vma, struct xe_vm_pgtable_update *entries) { struct xe_pt_stage_unbind_walk xe_walk = { @@ -1499,12 +1492,12 @@ static unsigned int xe_pt_stage_unbind(struct xe_gt *gt, struct xe_vma *vma, .shifts = xe_normal_pt_shifts, .max_level = XE_PT_HIGHEST_LEVEL, }, - .gt = gt, + .tile = tile, .modified_start = vma->start, .modified_end = vma->end + 1, .wupd.entries = entries, }; - struct xe_pt *pt = vma->vm->pt_root[gt->info.id]; + struct xe_pt *pt = vma->vm->pt_root[tile->id]; (void)xe_pt_walk_shared(&pt->base, pt->level, vma->start, vma->end + 1, &xe_walk.base); @@ -1514,19 +1507,17 @@ static unsigned int xe_pt_stage_unbind(struct xe_gt *gt, struct xe_vma *vma, static void xe_migrate_clear_pgtable_callback(struct xe_migrate_pt_update *pt_update, - struct xe_gt *gt, struct iosys_map *map, + struct xe_tile *tile, struct iosys_map *map, void *ptr, u32 qword_ofs, u32 num_qwords, const struct xe_vm_pgtable_update *update) { struct xe_vma *vma = pt_update->vma; - u64 empty = __xe_pt_empty_pte(gt, vma->vm, update->pt->level); + u64 empty = __xe_pt_empty_pte(tile, vma->vm, update->pt->level); int i; - XE_BUG_ON(xe_gt_is_media_type(gt)); - if (map && map->is_iomem) for (i = 0; i < num_qwords; ++i) - xe_map_wr(gt_to_xe(gt), map, (qword_ofs + i) * + xe_map_wr(tile_to_xe(tile), map, (qword_ofs + i) * sizeof(u64), u64, empty); else if (map) memset64(map->vaddr + qword_ofs * sizeof(u64), empty, @@ -1577,7 +1568,7 @@ static const struct xe_migrate_pt_update_ops userptr_unbind_ops = { /** * __xe_pt_unbind_vma() - Disconnect and free a page-table tree for the vma * address range. - * @gt: The gt to unbind for. + * @tile: The tile to unbind for. * @vma: The vma to unbind. * @e: The engine with which to do pipelined page-table updates. * @syncs: Entries to sync on before disconnecting the tree to be destroyed. @@ -1595,7 +1586,7 @@ static const struct xe_migrate_pt_update_ops userptr_unbind_ops = { * on success, an error pointer on error. */ struct dma_fence * -__xe_pt_unbind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, +__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, struct xe_sync_entry *syncs, u32 num_syncs) { struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; @@ -1614,16 +1605,15 @@ __xe_pt_unbind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, xe_bo_assert_held(vma->bo); xe_vm_assert_held(vm); - XE_BUG_ON(xe_gt_is_media_type(gt)); vm_dbg(&vma->vm->xe->drm, "Preparing unbind, with range [%llx...%llx) engine %p.\n", vma->start, vma->end, e); - num_entries = xe_pt_stage_unbind(gt, vma, entries); + num_entries = xe_pt_stage_unbind(tile, vma, entries); XE_BUG_ON(num_entries > ARRAY_SIZE(entries)); - xe_vm_dbg_print_entries(gt_to_xe(gt), entries, num_entries); + xe_vm_dbg_print_entries(tile_to_xe(tile), entries, num_entries); ifence = kzalloc(sizeof(*ifence), GFP_KERNEL); if (!ifence) @@ -1634,9 +1624,9 @@ __xe_pt_unbind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, * clear again here. The eviction may have updated pagetables at a * lower level, because it needs to be more conservative. */ - fence = xe_migrate_update_pgtables(gt->migrate, + fence = xe_migrate_update_pgtables(tile->primary_gt.migrate, vm, NULL, e ? e : - vm->eng[gt->info.id], + vm->eng[tile->id], entries, num_entries, syncs, num_syncs, &unbind_pt_update.base); @@ -1644,7 +1634,7 @@ __xe_pt_unbind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, int err; /* TLB invalidation must be done before signaling unbind */ - err = invalidation_fence_init(gt, ifence, fence, vma); + err = invalidation_fence_init(&tile->primary_gt, ifence, fence, vma); if (err) { dma_fence_put(fence); kfree(ifence); @@ -1662,18 +1652,18 @@ __xe_pt_unbind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, DMA_RESV_USAGE_BOOKKEEP); xe_pt_commit_unbind(vma, entries, num_entries, unbind_pt_update.locked ? &deferred : NULL); - vma->gt_present &= ~BIT(gt->info.id); + vma->tile_present &= ~BIT(tile->id); } else { kfree(ifence); } - if (!vma->gt_present) + if (!vma->tile_present) list_del_init(&vma->rebind_link); if (unbind_pt_update.locked) { XE_WARN_ON(!xe_vma_is_userptr(vma)); - if (!vma->gt_present) { + if (!vma->tile_present) { spin_lock(&vm->userptr.invalidated_lock); list_del_init(&vma->userptr.invalidate_link); spin_unlock(&vm->userptr.invalidated_lock); diff --git a/drivers/gpu/drm/xe/xe_pt.h b/drivers/gpu/drm/xe/xe_pt.h index 1152043e5c63..10f334b9c004 100644 --- a/drivers/gpu/drm/xe/xe_pt.h +++ b/drivers/gpu/drm/xe/xe_pt.h @@ -13,8 +13,8 @@ struct dma_fence; struct xe_bo; struct xe_device; struct xe_engine; -struct xe_gt; struct xe_sync_entry; +struct xe_tile; struct xe_vm; struct xe_vma; @@ -23,27 +23,27 @@ struct xe_vma; unsigned int xe_pt_shift(unsigned int level); -struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_gt *gt, +struct xe_pt *xe_pt_create(struct xe_vm *vm, struct xe_tile *tile, unsigned int level); -int xe_pt_create_scratch(struct xe_device *xe, struct xe_gt *gt, +int xe_pt_create_scratch(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm); -void xe_pt_populate_empty(struct xe_gt *gt, struct xe_vm *vm, +void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm, struct xe_pt *pt); void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred); struct dma_fence * -__xe_pt_bind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, +__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, struct xe_sync_entry *syncs, u32 num_syncs, bool rebind); struct dma_fence * -__xe_pt_unbind_vma(struct xe_gt *gt, struct xe_vma *vma, struct xe_engine *e, +__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, struct xe_sync_entry *syncs, u32 num_syncs); -bool xe_pt_zap_ptes(struct xe_gt *gt, struct xe_vma *vma); +bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma); u64 gen8_pde_encode(struct xe_bo *bo, u64 bo_offset, const enum xe_cache_level level); diff --git a/drivers/gpu/drm/xe/xe_sa.c b/drivers/gpu/drm/xe/xe_sa.c index c16f7c14ff52..fee71080bd31 100644 --- a/drivers/gpu/drm/xe/xe_sa.c +++ b/drivers/gpu/drm/xe/xe_sa.c @@ -11,7 +11,6 @@ #include "xe_bo.h" #include "xe_device.h" -#include "xe_gt.h" #include "xe_map.h" static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg) @@ -33,14 +32,14 @@ static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg) sa_manager->bo = NULL; } -struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_gt *gt, u32 size, u32 align) +struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 align) { - struct xe_device *xe = gt_to_xe(gt); + struct xe_device *xe = tile_to_xe(tile); u32 managed_size = size - SZ_4K; struct xe_bo *bo; int ret; - struct xe_sa_manager *sa_manager = drmm_kzalloc(>_to_xe(gt)->drm, + struct xe_sa_manager *sa_manager = drmm_kzalloc(&tile_to_xe(tile)->drm, sizeof(*sa_manager), GFP_KERNEL); if (!sa_manager) @@ -48,8 +47,8 @@ struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_gt *gt, u32 size, u32 alig sa_manager->bo = NULL; - bo = xe_bo_create_pin_map(xe, gt, NULL, size, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + bo = xe_bo_create_pin_map(xe, tile, NULL, size, ttm_bo_type_kernel, + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(bo)) { drm_err(&xe->drm, "failed to allocate bo for sa manager: %ld\n", @@ -90,7 +89,7 @@ struct drm_suballoc *xe_sa_bo_new(struct xe_sa_manager *sa_manager, void xe_sa_bo_flush_write(struct drm_suballoc *sa_bo) { struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager); - struct xe_device *xe = gt_to_xe(sa_manager->bo->gt); + struct xe_device *xe = tile_to_xe(sa_manager->bo->tile); if (!sa_manager->bo->vmap.is_iomem) return; diff --git a/drivers/gpu/drm/xe/xe_sa.h b/drivers/gpu/drm/xe/xe_sa.h index 3063fb34c720..4e96483057d7 100644 --- a/drivers/gpu/drm/xe/xe_sa.h +++ b/drivers/gpu/drm/xe/xe_sa.h @@ -9,9 +9,9 @@ struct dma_fence; struct xe_bo; -struct xe_gt; +struct xe_tile; -struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_gt *gt, u32 size, u32 align); +struct xe_sa_manager *xe_sa_bo_manager_init(struct xe_tile *tile, u32 size, u32 align); struct drm_suballoc *xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size); diff --git a/drivers/gpu/drm/xe/xe_tile.c b/drivers/gpu/drm/xe/xe_tile.c index 5530a6b6ef31..59d3e25ea550 100644 --- a/drivers/gpu/drm/xe/xe_tile.c +++ b/drivers/gpu/drm/xe/xe_tile.c @@ -7,6 +7,7 @@ #include "xe_device.h" #include "xe_ggtt.h" +#include "xe_sa.h" #include "xe_tile.h" #include "xe_ttm_vram_mgr.h" @@ -76,6 +77,12 @@ int xe_tile_init_noalloc(struct xe_tile *tile) goto err_mem_access; err = xe_ggtt_init_noalloc(tile->mem.ggtt); + if (err) + goto err_mem_access; + + tile->mem.kernel_bb_pool = xe_sa_bo_manager_init(tile, SZ_1M, 16); + if (IS_ERR(tile->mem.kernel_bb_pool)) + err = PTR_ERR(tile->mem.kernel_bb_pool); err_mem_access: xe_device_mem_access_put(tile_to_xe(tile)); diff --git a/drivers/gpu/drm/xe/xe_uc_fw.c b/drivers/gpu/drm/xe/xe_uc_fw.c index 5703213bdf1b..2b9b9b4a6711 100644 --- a/drivers/gpu/drm/xe/xe_uc_fw.c +++ b/drivers/gpu/drm/xe/xe_uc_fw.c @@ -322,6 +322,7 @@ int xe_uc_fw_init(struct xe_uc_fw *uc_fw) { struct xe_device *xe = uc_fw_to_xe(uc_fw); struct xe_gt *gt = uc_fw_to_gt(uc_fw); + struct xe_tile *tile = gt_to_tile(gt); struct device *dev = xe->drm.dev; const struct firmware *fw = NULL; struct uc_css_header *css; @@ -411,9 +412,9 @@ int xe_uc_fw_init(struct xe_uc_fw *uc_fw) if (uc_fw->type == XE_UC_FW_TYPE_GUC) guc_read_css_info(uc_fw, css); - obj = xe_bo_create_from_data(xe, gt, fw->data, fw->size, + obj = xe_bo_create_from_data(xe, tile, fw->data, fw->size, ttm_bo_type_kernel, - XE_BO_CREATE_VRAM_IF_DGFX(gt) | + XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_GGTT_BIT); if (IS_ERR(obj)) { drm_notice(&xe->drm, "%s firmware %s: failed to create / populate bo", diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 798cba1bda6b..ecfff4ffd00e 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -465,7 +465,7 @@ int xe_vm_lock_dma_resv(struct xe_vm *vm, struct ww_acquire_ctx *ww, xe_bo_assert_held(vma->bo); list_del_init(&vma->notifier.rebind_link); - if (vma->gt_present && !vma->destroyed) + if (vma->tile_present && !vma->destroyed) list_move_tail(&vma->rebind_link, &vm->rebind_list); } spin_unlock(&vm->notifier.list_lock); @@ -703,7 +703,7 @@ static bool vma_userptr_invalidate(struct mmu_interval_notifier *mni, * Tell exec and rebind worker they need to repin and rebind this * userptr. */ - if (!xe_vm_in_fault_mode(vm) && !vma->destroyed && vma->gt_present) { + if (!xe_vm_in_fault_mode(vm) && !vma->destroyed && vma->tile_present) { spin_lock(&vm->userptr.invalidated_lock); list_move_tail(&vma->userptr.invalidate_link, &vm->userptr.invalidated); @@ -821,7 +821,7 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker) xe_vm_assert_held(vm); list_for_each_entry_safe(vma, next, &vm->rebind_list, rebind_link) { - XE_WARN_ON(!vma->gt_present); + XE_WARN_ON(!vma->tile_present); list_del_init(&vma->rebind_link); dma_fence_put(fence); @@ -842,10 +842,10 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, u64 bo_offset_or_userptr, u64 start, u64 end, bool read_only, - u64 gt_mask) + u64 tile_mask) { struct xe_vma *vma; - struct xe_gt *gt; + struct xe_tile *tile; u8 id; XE_BUG_ON(start >= end); @@ -870,12 +870,11 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, if (read_only) vma->pte_flags = XE_PTE_READ_ONLY; - if (gt_mask) { - vma->gt_mask = gt_mask; + if (tile_mask) { + vma->tile_mask = tile_mask; } else { - for_each_gt(gt, vm->xe, id) - if (!xe_gt_is_media_type(gt)) - vma->gt_mask |= 0x1 << id; + for_each_tile(tile, vm->xe, id) + vma->tile_mask |= 0x1 << id; } if (vm->xe->info.platform == XE_PVC) @@ -1162,8 +1161,8 @@ static void vm_destroy_work_func(struct work_struct *w); struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) { struct xe_vm *vm; - int err, i = 0, number_gts = 0; - struct xe_gt *gt; + int err, i = 0, number_tiles = 0; + struct xe_tile *tile; u8 id; vm = kzalloc(sizeof(*vm), GFP_KERNEL); @@ -1215,15 +1214,12 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) if (IS_DGFX(xe) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) vm->flags |= XE_VM_FLAGS_64K; - for_each_gt(gt, xe, id) { - if (xe_gt_is_media_type(gt)) - continue; - + for_each_tile(tile, xe, id) { if (flags & XE_VM_FLAG_MIGRATION && - gt->info.id != XE_VM_FLAG_GT_ID(flags)) + tile->id != XE_VM_FLAG_GT_ID(flags)) continue; - vm->pt_root[id] = xe_pt_create(vm, gt, xe->info.vm_max_level); + vm->pt_root[id] = xe_pt_create(vm, tile, xe->info.vm_max_level); if (IS_ERR(vm->pt_root[id])) { err = PTR_ERR(vm->pt_root[id]); vm->pt_root[id] = NULL; @@ -1232,11 +1228,11 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) } if (flags & XE_VM_FLAG_SCRATCH_PAGE) { - for_each_gt(gt, xe, id) { + for_each_tile(tile, xe, id) { if (!vm->pt_root[id]) continue; - err = xe_pt_create_scratch(xe, gt, vm); + err = xe_pt_create_scratch(xe, tile, vm); if (err) goto err_scratch_pt; } @@ -1253,17 +1249,18 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) } /* Fill pt_root after allocating scratch tables */ - for_each_gt(gt, xe, id) { + for_each_tile(tile, xe, id) { if (!vm->pt_root[id]) continue; - xe_pt_populate_empty(gt, vm, vm->pt_root[id]); + xe_pt_populate_empty(tile, vm, vm->pt_root[id]); } dma_resv_unlock(&vm->resv); /* Kernel migration VM shouldn't have a circular loop.. */ if (!(flags & XE_VM_FLAG_MIGRATION)) { - for_each_gt(gt, xe, id) { + for_each_tile(tile, xe, id) { + struct xe_gt *gt = &tile->primary_gt; struct xe_vm *migrate_vm; struct xe_engine *eng; @@ -1280,11 +1277,11 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) return ERR_CAST(eng); } vm->eng[id] = eng; - number_gts++; + number_tiles++; } } - if (number_gts > 1) + if (number_tiles > 1) vm->composite_fence_ctx = dma_fence_context_alloc(1); mutex_lock(&xe->usm.lock); @@ -1299,7 +1296,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) return vm; err_scratch_pt: - for_each_gt(gt, xe, id) { + for_each_tile(tile, xe, id) { if (!vm->pt_root[id]) continue; @@ -1312,7 +1309,7 @@ err_scratch_pt: xe_bo_put(vm->scratch_bo[id]); } err_destroy_root: - for_each_gt(gt, xe, id) { + for_each_tile(tile, xe, id) { if (vm->pt_root[id]) xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); } @@ -1369,7 +1366,7 @@ void xe_vm_close_and_put(struct xe_vm *vm) struct rb_root contested = RB_ROOT; struct ww_acquire_ctx ww; struct xe_device *xe = vm->xe; - struct xe_gt *gt; + struct xe_tile *tile; u8 id; XE_BUG_ON(vm->preempt.num_engines); @@ -1380,7 +1377,7 @@ void xe_vm_close_and_put(struct xe_vm *vm) if (xe_vm_in_compute_mode(vm)) flush_work(&vm->preempt.rebind_work); - for_each_gt(gt, xe, id) { + for_each_tile(tile, xe, id) { if (vm->eng[id]) { xe_engine_kill(vm->eng[id]); xe_engine_put(vm->eng[id]); @@ -1417,7 +1414,7 @@ void xe_vm_close_and_put(struct xe_vm *vm) * install a fence to resv. Hence it's safe to * destroy the pagetables immediately. */ - for_each_gt(gt, xe, id) { + for_each_tile(tile, xe, id) { if (vm->scratch_bo[id]) { u32 i; @@ -1467,7 +1464,7 @@ static void vm_destroy_work_func(struct work_struct *w) container_of(w, struct xe_vm, destroy_work); struct ww_acquire_ctx ww; struct xe_device *xe = vm->xe; - struct xe_gt *gt; + struct xe_tile *tile; u8 id; void *lookup; @@ -1492,7 +1489,7 @@ static void vm_destroy_work_func(struct work_struct *w) * can be moved to xe_vm_close_and_put. */ xe_vm_lock(vm, &ww, 0, false); - for_each_gt(gt, xe, id) { + for_each_tile(tile, xe, id) { if (vm->pt_root[id]) { xe_pt_destroy(vm->pt_root[id], vm->flags, NULL); vm->pt_root[id] = NULL; @@ -1528,11 +1525,9 @@ struct xe_vm *xe_vm_lookup(struct xe_file *xef, u32 id) return vm; } -u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_gt *full_gt) +u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) { - XE_BUG_ON(xe_gt_is_media_type(full_gt)); - - return gen8_pde_encode(vm->pt_root[full_gt->info.id]->bo, 0, + return gen8_pde_encode(vm->pt_root[tile->id]->bo, 0, XE_CACHE_WB); } @@ -1540,32 +1535,30 @@ static struct dma_fence * xe_vm_unbind_vma(struct xe_vma *vma, struct xe_engine *e, struct xe_sync_entry *syncs, u32 num_syncs) { - struct xe_gt *gt; + struct xe_tile *tile; struct dma_fence *fence = NULL; struct dma_fence **fences = NULL; struct dma_fence_array *cf = NULL; struct xe_vm *vm = vma->vm; int cur_fence = 0, i; - int number_gts = hweight_long(vma->gt_present); + int number_tiles = hweight_long(vma->tile_present); int err; u8 id; trace_xe_vma_unbind(vma); - if (number_gts > 1) { - fences = kmalloc_array(number_gts, sizeof(*fences), + if (number_tiles > 1) { + fences = kmalloc_array(number_tiles, sizeof(*fences), GFP_KERNEL); if (!fences) return ERR_PTR(-ENOMEM); } - for_each_gt(gt, vm->xe, id) { - if (!(vma->gt_present & BIT(id))) + for_each_tile(tile, vm->xe, id) { + if (!(vma->tile_present & BIT(id))) goto next; - XE_BUG_ON(xe_gt_is_media_type(gt)); - - fence = __xe_pt_unbind_vma(gt, vma, e, syncs, num_syncs); + fence = __xe_pt_unbind_vma(tile, vma, e, syncs, num_syncs); if (IS_ERR(fence)) { err = PTR_ERR(fence); goto err_fences; @@ -1580,7 +1573,7 @@ next: } if (fences) { - cf = dma_fence_array_create(number_gts, fences, + cf = dma_fence_array_create(number_tiles, fences, vm->composite_fence_ctx, vm->composite_fence_seqno++, false); @@ -1612,32 +1605,31 @@ static struct dma_fence * xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e, struct xe_sync_entry *syncs, u32 num_syncs) { - struct xe_gt *gt; + struct xe_tile *tile; struct dma_fence *fence; struct dma_fence **fences = NULL; struct dma_fence_array *cf = NULL; struct xe_vm *vm = vma->vm; int cur_fence = 0, i; - int number_gts = hweight_long(vma->gt_mask); + int number_tiles = hweight_long(vma->tile_mask); int err; u8 id; trace_xe_vma_bind(vma); - if (number_gts > 1) { - fences = kmalloc_array(number_gts, sizeof(*fences), + if (number_tiles > 1) { + fences = kmalloc_array(number_tiles, sizeof(*fences), GFP_KERNEL); if (!fences) return ERR_PTR(-ENOMEM); } - for_each_gt(gt, vm->xe, id) { - if (!(vma->gt_mask & BIT(id))) + for_each_tile(tile, vm->xe, id) { + if (!(vma->tile_mask & BIT(id))) goto next; - XE_BUG_ON(xe_gt_is_media_type(gt)); - fence = __xe_pt_bind_vma(gt, vma, e, syncs, num_syncs, - vma->gt_present & BIT(id)); + fence = __xe_pt_bind_vma(tile, vma, e, syncs, num_syncs, + vma->tile_present & BIT(id)); if (IS_ERR(fence)) { err = PTR_ERR(fence); goto err_fences; @@ -1652,7 +1644,7 @@ next: } if (fences) { - cf = dma_fence_array_create(number_gts, fences, + cf = dma_fence_array_create(number_tiles, fences, vm->composite_fence_ctx, vm->composite_fence_seqno++, false); @@ -2047,7 +2039,7 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, return err; } - if (vma->gt_mask != (vma->gt_present & ~vma->usm.gt_invalidated)) { + if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) { return xe_vm_bind(vm, vma, e, vma->bo, syncs, num_syncs, afence); } else { @@ -2649,7 +2641,7 @@ static struct xe_vma *vm_unbind_lookup_vmas(struct xe_vm *vm, first->start, lookup->start - 1, (first->pte_flags & XE_PTE_READ_ONLY), - first->gt_mask); + first->tile_mask); if (first->bo) xe_bo_unlock(first->bo, &ww); if (!new_first) { @@ -2680,7 +2672,7 @@ static struct xe_vma *vm_unbind_lookup_vmas(struct xe_vm *vm, last->start + chunk, last->end, (last->pte_flags & XE_PTE_READ_ONLY), - last->gt_mask); + last->tile_mask); if (last->bo) xe_bo_unlock(last->bo, &ww); if (!new_last) { @@ -2816,7 +2808,7 @@ static struct xe_vma *vm_bind_ioctl_lookup_vma(struct xe_vm *vm, struct xe_bo *bo, u64 bo_offset_or_userptr, u64 addr, u64 range, u32 op, - u64 gt_mask, u32 region) + u64 tile_mask, u32 region) { struct ww_acquire_ctx ww; struct xe_vma *vma, lookup; @@ -2837,7 +2829,7 @@ static struct xe_vma *vm_bind_ioctl_lookup_vma(struct xe_vm *vm, vma = xe_vma_create(vm, bo, bo_offset_or_userptr, addr, addr + range - 1, op & XE_VM_BIND_FLAG_READONLY, - gt_mask); + tile_mask); xe_bo_unlock(bo, &ww); if (!vma) return ERR_PTR(-ENOMEM); @@ -2877,7 +2869,7 @@ static struct xe_vma *vm_bind_ioctl_lookup_vma(struct xe_vm *vm, vma = xe_vma_create(vm, NULL, bo_offset_or_userptr, addr, addr + range - 1, op & XE_VM_BIND_FLAG_READONLY, - gt_mask); + tile_mask); if (!vma) return ERR_PTR(-ENOMEM); @@ -3114,11 +3106,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto put_engine; } - if (bind_ops[i].gt_mask) { - u64 valid_gts = BIT(xe->info.tile_count) - 1; + if (bind_ops[i].tile_mask) { + u64 valid_tiles = BIT(xe->info.tile_count) - 1; - if (XE_IOCTL_ERR(xe, bind_ops[i].gt_mask & - ~valid_gts)) { + if (XE_IOCTL_ERR(xe, bind_ops[i].tile_mask & + ~valid_tiles)) { err = -EINVAL; goto put_engine; } @@ -3209,11 +3201,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) u64 addr = bind_ops[i].addr; u32 op = bind_ops[i].op; u64 obj_offset = bind_ops[i].obj_offset; - u64 gt_mask = bind_ops[i].gt_mask; + u64 tile_mask = bind_ops[i].tile_mask; u32 region = bind_ops[i].region; vmas[i] = vm_bind_ioctl_lookup_vma(vm, bos[i], obj_offset, - addr, range, op, gt_mask, + addr, range, op, tile_mask, region); if (IS_ERR(vmas[i])) { err = PTR_ERR(vmas[i]); @@ -3387,8 +3379,8 @@ void xe_vm_unlock(struct xe_vm *vm, struct ww_acquire_ctx *ww) int xe_vm_invalidate_vma(struct xe_vma *vma) { struct xe_device *xe = vma->vm->xe; - struct xe_gt *gt; - u32 gt_needs_invalidate = 0; + struct xe_tile *tile; + u32 tile_needs_invalidate = 0; int seqno[XE_MAX_TILES_PER_DEVICE]; u8 id; int ret; @@ -3410,25 +3402,29 @@ int xe_vm_invalidate_vma(struct xe_vma *vma) } } - for_each_gt(gt, xe, id) { - if (xe_pt_zap_ptes(gt, vma)) { - gt_needs_invalidate |= BIT(id); + for_each_tile(tile, xe, id) { + if (xe_pt_zap_ptes(tile, vma)) { + tile_needs_invalidate |= BIT(id); xe_device_wmb(xe); - seqno[id] = xe_gt_tlb_invalidation_vma(gt, NULL, vma); + /* + * FIXME: We potentially need to invalidate multiple + * GTs within the tile + */ + seqno[id] = xe_gt_tlb_invalidation_vma(&tile->primary_gt, NULL, vma); if (seqno[id] < 0) return seqno[id]; } } - for_each_gt(gt, xe, id) { - if (gt_needs_invalidate & BIT(id)) { - ret = xe_gt_tlb_invalidation_wait(gt, seqno[id]); + for_each_tile(tile, xe, id) { + if (tile_needs_invalidate & BIT(id)) { + ret = xe_gt_tlb_invalidation_wait(&tile->primary_gt, seqno[id]); if (ret < 0) return ret; } } - vma->usm.gt_invalidated = vma->gt_mask; + vma->usm.tile_invalidated = vma->tile_mask; return 0; } diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index 748dc16ebed9..372f26153209 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -54,7 +54,7 @@ xe_vm_find_overlapping_vma(struct xe_vm *vm, const struct xe_vma *vma); #define xe_vm_assert_held(vm) dma_resv_assert_held(&(vm)->resv) -u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_gt *full_gt); +u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile); int xe_vm_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file); diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 203ba9d946b8..c45c5daeeaa7 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -37,17 +37,17 @@ struct xe_vma { /** @bo_offset: offset into BO if not a userptr, unused for userptr */ u64 bo_offset; - /** @gt_mask: GT mask of where to create binding for this VMA */ - u64 gt_mask; + /** @tile_mask: Tile mask of where to create binding for this VMA */ + u64 tile_mask; /** - * @gt_present: GT mask of binding are present for this VMA. + * @tile_present: GT mask of binding are present for this VMA. * protected by vm->lock, vm->resv and for userptrs, * vm->userptr.notifier_lock for writing. Needs either for reading, * but if reading is done under the vm->lock only, it needs to be held * in write mode. */ - u64 gt_present; + u64 tile_present; /** * @destroyed: VMA is destroyed, in the sense that it shouldn't be @@ -132,8 +132,8 @@ struct xe_vma { /** @usm: unified shared memory state */ struct { - /** @gt_invalidated: VMA has been invalidated */ - u64 gt_invalidated; + /** @tile_invalidated: VMA has been invalidated */ + u64 tile_invalidated; } usm; struct { diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 34aff9e15fe6..edd29e7f39eb 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -407,10 +407,10 @@ struct drm_xe_vm_bind_op { __u64 addr; /** - * @gt_mask: Mask for which GTs to create binds for, 0 == All GTs, + * @tile_mask: Mask for which tiles to create binds for, 0 == All tiles, * only applies to creating new VMAs */ - __u64 gt_mask; + __u64 tile_mask; /** @op: Operation to perform (lower 16 bits) and flags (upper 16 bits) */ __u32 op; -- cgit v1.2.3 From a4f08dbb712135680d086ffa9e8ee5c07e5fc661 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 31 May 2023 15:23:34 +0000 Subject: drm/xe: Use SPDX-License-Identifier instead of license text Replace the license text with its SPDX-License-Identifier for quick identification of the license and consistency with the rest of the driver. Reported-by: Oded Gabbay Signed-off-by: Francois Dugast Reviewed-by: Lucas De Marchi Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index edd29e7f39eb..4266760faf05 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1,26 +1,6 @@ +/* SPDX-License-Identifier: MIT */ /* - * Copyright 2021 Intel Corporation. All Rights Reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the - * "Software"), to deal in the Software without restriction, including - * without limitation the rights to use, copy, modify, merge, publish, - * distribute, sub license, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to - * the following conditions: - * - * The above copyright notice and this permission notice (including the - * next paragraph) shall be included in all copies or substantial portions - * of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. - * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR - * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * + * Copyright © 2023 Intel Corporation */ #ifndef _UAPI_XE_DRM_H_ -- cgit v1.2.3 From fcca94c69b9539ed741ba5875ab4f1157cd781f8 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 31 May 2023 15:23:35 +0000 Subject: drm/xe: Group engine related structs Move the definition of drm_xe_engine_class_instance to group it with other engine related structs and to follow the ioctls order. Reported-by: Oded Gabbay Signed-off-by: Francois Dugast Reviewed-by: Lucas De Marchi Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 4266760faf05..7d317b9564e9 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -116,24 +116,6 @@ struct xe_user_extension { #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) #define DRM_IOCTL_XE_VM_MADVISE DRM_IOW( DRM_COMMAND_BASE + DRM_XE_VM_MADVISE, struct drm_xe_vm_madvise) -struct drm_xe_engine_class_instance { - __u16 engine_class; - -#define DRM_XE_ENGINE_CLASS_RENDER 0 -#define DRM_XE_ENGINE_CLASS_COPY 1 -#define DRM_XE_ENGINE_CLASS_VIDEO_DECODE 2 -#define DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE 3 -#define DRM_XE_ENGINE_CLASS_COMPUTE 4 - /* - * Kernel only class (not actual hardware engine class). Used for - * creating ordered queues of VM bind operations. - */ -#define DRM_XE_ENGINE_CLASS_VM_BIND 5 - - __u16 engine_instance; - __u16 gt_id; -}; - #define XE_MEM_REGION_CLASS_SYSMEM 0 #define XE_MEM_REGION_CLASS_VRAM 1 @@ -536,6 +518,24 @@ struct drm_xe_engine_set_property { __u64 reserved[2]; }; +struct drm_xe_engine_class_instance { + __u16 engine_class; + +#define DRM_XE_ENGINE_CLASS_RENDER 0 +#define DRM_XE_ENGINE_CLASS_COPY 1 +#define DRM_XE_ENGINE_CLASS_VIDEO_DECODE 2 +#define DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE 3 +#define DRM_XE_ENGINE_CLASS_COMPUTE 4 + /* + * Kernel only class (not actual hardware engine class). Used for + * creating ordered queues of VM bind operations. + */ +#define DRM_XE_ENGINE_CLASS_VM_BIND 5 + + __u16 engine_instance; + __u16 gt_id; +}; + struct drm_xe_engine_create { /** @extensions: Pointer to the first extension struct, if any */ #define XE_ENGINE_EXTENSION_SET_PROPERTY 0 -- cgit v1.2.3 From a0385a840ca02585d16a1ed4b10b501d17853d33 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Thu, 8 Jun 2023 09:59:14 +0200 Subject: drm/xe: Fix some formatting issues in uAPI Fix spacing, alignment, and repeated words in the documentation. Reported-by: Oded Gabbay Signed-off-by: Francois Dugast Reviewed-by: Lucas De Marchi Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 7d317b9564e9..83868af45984 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -105,16 +105,16 @@ struct xe_user_extension { #define DRM_IOCTL_XE_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_CREATE, struct drm_xe_gem_create) #define DRM_IOCTL_XE_GEM_MMAP_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_MMAP_OFFSET, struct drm_xe_gem_mmap_offset) #define DRM_IOCTL_XE_VM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_CREATE, struct drm_xe_vm_create) -#define DRM_IOCTL_XE_VM_DESTROY DRM_IOW( DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) -#define DRM_IOCTL_XE_VM_BIND DRM_IOW( DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) +#define DRM_IOCTL_XE_VM_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) +#define DRM_IOCTL_XE_VM_BIND DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) #define DRM_IOCTL_XE_ENGINE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_ENGINE_CREATE, struct drm_xe_engine_create) #define DRM_IOCTL_XE_ENGINE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_ENGINE_GET_PROPERTY, struct drm_xe_engine_get_property) -#define DRM_IOCTL_XE_ENGINE_DESTROY DRM_IOW( DRM_COMMAND_BASE + DRM_XE_ENGINE_DESTROY, struct drm_xe_engine_destroy) -#define DRM_IOCTL_XE_EXEC DRM_IOW( DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) +#define DRM_IOCTL_XE_ENGINE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_ENGINE_DESTROY, struct drm_xe_engine_destroy) +#define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) #define DRM_IOCTL_XE_MMIO DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_MMIO, struct drm_xe_mmio) -#define DRM_IOCTL_XE_ENGINE_SET_PROPERTY DRM_IOW( DRM_COMMAND_BASE + DRM_XE_ENGINE_SET_PROPERTY, struct drm_xe_engine_set_property) +#define DRM_IOCTL_XE_ENGINE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_ENGINE_SET_PROPERTY, struct drm_xe_engine_set_property) #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) -#define DRM_IOCTL_XE_VM_MADVISE DRM_IOW( DRM_COMMAND_BASE + DRM_XE_VM_MADVISE, struct drm_xe_vm_madvise) +#define DRM_IOCTL_XE_VM_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_MADVISE, struct drm_xe_vm_madvise) #define XE_MEM_REGION_CLASS_SYSMEM 0 #define XE_MEM_REGION_CLASS_VRAM 1 @@ -147,7 +147,7 @@ struct drm_xe_query_config { #define XE_QUERY_CONFIG_GT_COUNT 4 #define XE_QUERY_CONFIG_MEM_REGION_COUNT 5 #define XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY 6 -#define XE_QUERY_CONFIG_NUM_PARAM XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY + 1 +#define XE_QUERY_CONFIG_NUM_PARAM (XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY + 1) __u64 info[]; }; @@ -399,8 +399,8 @@ struct drm_xe_vm_bind_op { * If this flag is clear and the IOCTL doesn't return an error, in * practice the bind op is good and will complete. * - * If this flag is set and doesn't return return an error, the bind op - * can still fail and recovery is needed. If configured, the bind op that + * If this flag is set and doesn't return an error, the bind op can + * still fail and recovery is needed. If configured, the bind op that * caused the error will be captured in drm_xe_vm_bind_op_error_capture. * Once the user sees the error (via a ufence + * XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS), it should free memory @@ -646,9 +646,9 @@ struct drm_xe_exec { __u64 syncs; /** - * @address: address of batch buffer if num_batch_buffer == 1 or an - * array of batch buffer addresses - */ + * @address: address of batch buffer if num_batch_buffer == 1 or an + * array of batch buffer addresses + */ __u64 address; /** -- cgit v1.2.3 From 37430402618db90b53aa782a6c49f66ab0efced0 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 15 Jun 2023 11:22:36 -0700 Subject: drm/xe: NULL binding implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add uAPI and implementation for NULL bindings. A NULL binding is defined as writes dropped and read zero. A single bit in the uAPI has been added which results in a single bit in the PTEs being set. NULL bindings are intendedd to be used to implement VK sparse bindings, in particular residencyNonResidentStrict property. v2: Fix BUG_ON shown in VK testing, fix check patch warning, fix xe_pt_scan_64K, update __gen8_pte_encode to understand NULL bindings, remove else if vma_addr Reviewed-by: Thomas Hellström Suggested-by: Paulo Zanoni Signed-off-by: Matthew Brost Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_bo.h | 1 + drivers/gpu/drm/xe/xe_exec.c | 2 + drivers/gpu/drm/xe/xe_gt_pagefault.c | 4 +- drivers/gpu/drm/xe/xe_pt.c | 54 ++++++++++++++------ drivers/gpu/drm/xe/xe_vm.c | 99 +++++++++++++++++++++++------------- drivers/gpu/drm/xe/xe_vm.h | 12 ++++- drivers/gpu/drm/xe/xe_vm_types.h | 1 + include/uapi/drm/xe_drm.h | 8 +++ 8 files changed, 126 insertions(+), 55 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index dd3d448fee0b..3a148cc6e811 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -61,6 +61,7 @@ #define XE_PPGTT_PTE_LM BIT_ULL(11) #define XE_PDE_64K BIT_ULL(6) #define XE_PTE_PS64 BIT_ULL(8) +#define XE_PTE_NULL BIT_ULL(9) #define XE_PAGE_PRESENT BIT_ULL(0) #define XE_PAGE_RW BIT_ULL(1) diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index e44076ee2e11..4f7694a29348 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -120,6 +120,8 @@ retry: * to a location where the GPU can access it). */ list_for_each_entry(vma, &vm->rebind_list, rebind_link) { + XE_WARN_ON(xe_vma_is_null(vma)); + if (xe_vma_is_userptr(vma)) continue; diff --git a/drivers/gpu/drm/xe/xe_gt_pagefault.c b/drivers/gpu/drm/xe/xe_gt_pagefault.c index 5436667ba82b..9dd8e5097e65 100644 --- a/drivers/gpu/drm/xe/xe_gt_pagefault.c +++ b/drivers/gpu/drm/xe/xe_gt_pagefault.c @@ -533,8 +533,8 @@ static int handle_acc(struct xe_gt *gt, struct acc *acc) trace_xe_vma_acc(vma); - /* Userptr can't be migrated, nothing to do */ - if (xe_vma_is_userptr(vma)) + /* Userptr or null can't be migrated, nothing to do */ + if (xe_vma_has_no_bo(vma)) goto unlock_vm; /* Lock VM and BOs dma-resv */ diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index 29c1b1f0bd7c..fe1c77b139e4 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -81,6 +81,9 @@ u64 xe_pde_encode(struct xe_bo *bo, u64 bo_offset, static dma_addr_t vma_addr(struct xe_vma *vma, u64 offset, size_t page_size, bool *is_vram) { + if (xe_vma_is_null(vma)) + return 0; + if (xe_vma_is_userptr(vma)) { struct xe_res_cursor cur; u64 page; @@ -105,6 +108,9 @@ static u64 __pte_encode(u64 pte, enum xe_cache_level cache, u32 flags, if (unlikely(flags & XE_PTE_FLAG_READ_ONLY)) pte &= ~XE_PAGE_RW; + if (unlikely(flags & XE_PTE_FLAG_NULL)) + pte |= XE_PTE_NULL; + /* FIXME: I don't think the PPAT handling is correct for MTL */ switch (cache) { @@ -557,6 +563,10 @@ static bool xe_pt_hugepte_possible(u64 addr, u64 next, unsigned int level, if (next - xe_walk->va_curs_start > xe_walk->curs->size) return false; + /* null VMA's do not have dma addresses */ + if (xe_walk->pte_flags & XE_PTE_FLAG_NULL) + return true; + /* Is the DMA address huge PTE size aligned? */ size = next - addr; dma = addr - xe_walk->va_curs_start + xe_res_dma(xe_walk->curs); @@ -579,6 +589,10 @@ xe_pt_scan_64K(u64 addr, u64 next, struct xe_pt_stage_bind_walk *xe_walk) if (next > xe_walk->l0_end_addr) return false; + /* null VMA's do not have dma addresses */ + if (xe_walk->pte_flags & XE_PTE_FLAG_NULL) + return true; + xe_res_next(&curs, addr - xe_walk->va_curs_start); for (; addr < next; addr += SZ_64K) { if (!IS_ALIGNED(xe_res_dma(&curs), SZ_64K) || curs.size < SZ_64K) @@ -629,10 +643,12 @@ xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, /* Is this a leaf entry ?*/ if (level == 0 || xe_pt_hugepte_possible(addr, next, level, xe_walk)) { struct xe_res_cursor *curs = xe_walk->curs; + bool is_null = xe_walk->pte_flags & XE_PTE_FLAG_NULL; XE_WARN_ON(xe_walk->va_curs_start != addr); - pte = __pte_encode(xe_res_dma(curs) + xe_walk->dma_offset, + pte = __pte_encode(is_null ? 0 : + xe_res_dma(curs) + xe_walk->dma_offset, xe_walk->cache, xe_walk->pte_flags, level); pte |= xe_walk->default_pte; @@ -652,7 +668,8 @@ xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, if (unlikely(ret)) return ret; - xe_res_next(curs, next - addr); + if (!is_null) + xe_res_next(curs, next - addr); xe_walk->va_curs_start = next; *action = ACTION_CONTINUE; @@ -759,24 +776,29 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource); xe_walk.cache = XE_CACHE_WB; } else { - if (!xe_vma_is_userptr(vma) && bo->flags & XE_BO_SCANOUT_BIT) + if (!xe_vma_has_no_bo(vma) && bo->flags & XE_BO_SCANOUT_BIT) xe_walk.cache = XE_CACHE_WT; else xe_walk.cache = XE_CACHE_WB; } - if (!xe_vma_is_userptr(vma) && xe_bo_is_stolen(bo)) + if (!xe_vma_has_no_bo(vma) && xe_bo_is_stolen(bo)) xe_walk.dma_offset = xe_ttm_stolen_gpu_offset(xe_bo_device(bo)); xe_bo_assert_held(bo); - if (xe_vma_is_userptr(vma)) - xe_res_first_sg(vma->userptr.sg, 0, vma->end - vma->start + 1, - &curs); - else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo)) - xe_res_first(bo->ttm.resource, vma->bo_offset, - vma->end - vma->start + 1, &curs); - else - xe_res_first_sg(xe_bo_get_sg(bo), vma->bo_offset, - vma->end - vma->start + 1, &curs); + + if (!xe_vma_is_null(vma)) { + if (xe_vma_is_userptr(vma)) + xe_res_first_sg(vma->userptr.sg, 0, + vma->end - vma->start + 1, &curs); + else if (xe_bo_is_vram(bo) || xe_bo_is_stolen(bo)) + xe_res_first(bo->ttm.resource, vma->bo_offset, + vma->end - vma->start + 1, &curs); + else + xe_res_first_sg(xe_bo_get_sg(bo), vma->bo_offset, + vma->end - vma->start + 1, &curs); + } else { + curs.size = vma->end - vma->start + 1; + } ret = xe_pt_walk_range(&pt->base, pt->level, vma->start, vma->end + 1, &xe_walk.base); @@ -965,7 +987,7 @@ static void xe_pt_commit_locks_assert(struct xe_vma *vma) if (xe_vma_is_userptr(vma)) lockdep_assert_held_read(&vm->userptr.notifier_lock); - else + else if (!xe_vma_is_null(vma)) dma_resv_assert_held(vma->bo->ttm.base.resv); dma_resv_assert_held(&vm->resv); @@ -1341,7 +1363,7 @@ __xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, DMA_RESV_USAGE_KERNEL : DMA_RESV_USAGE_BOOKKEEP); - if (!xe_vma_is_userptr(vma) && !vma->bo->vm) + if (!xe_vma_has_no_bo(vma) && !vma->bo->vm) dma_resv_add_fence(vma->bo->ttm.base.resv, fence, DMA_RESV_USAGE_BOOKKEEP); xe_pt_commit_bind(vma, entries, num_entries, rebind, @@ -1658,7 +1680,7 @@ __xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e DMA_RESV_USAGE_BOOKKEEP); /* This fence will be installed by caller when doing eviction */ - if (!xe_vma_is_userptr(vma) && !vma->bo->vm) + if (!xe_vma_has_no_bo(vma) && !vma->bo->vm) dma_resv_add_fence(vma->bo->ttm.base.resv, fence, DMA_RESV_USAGE_BOOKKEEP); xe_pt_commit_unbind(vma, entries, num_entries, diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 6edac7d4af87..5ac819a65cf1 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -590,7 +590,7 @@ retry: goto out_unlock; list_for_each_entry(vma, &vm->rebind_list, rebind_link) { - if (xe_vma_is_userptr(vma) || vma->destroyed) + if (xe_vma_has_no_bo(vma) || vma->destroyed) continue; err = xe_bo_validate(vma->bo, vm, false); @@ -843,6 +843,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, u64 bo_offset_or_userptr, u64 start, u64 end, bool read_only, + bool is_null, u64 tile_mask) { struct xe_vma *vma; @@ -868,8 +869,11 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, vma->vm = vm; vma->start = start; vma->end = end; + vma->pte_flags = 0; if (read_only) - vma->pte_flags = XE_PTE_FLAG_READ_ONLY; + vma->pte_flags |= XE_PTE_FLAG_READ_ONLY; + if (is_null) + vma->pte_flags |= XE_PTE_FLAG_NULL; if (tile_mask) { vma->tile_mask = tile_mask; @@ -886,23 +890,26 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, vma->bo_offset = bo_offset_or_userptr; vma->bo = xe_bo_get(bo); list_add_tail(&vma->bo_link, &bo->vmas); - } else /* userptr */ { - u64 size = end - start + 1; - int err; + } else /* userptr or null */ { + if (!is_null) { + u64 size = end - start + 1; + int err; - vma->userptr.ptr = bo_offset_or_userptr; + vma->userptr.ptr = bo_offset_or_userptr; - err = mmu_interval_notifier_insert(&vma->userptr.notifier, - current->mm, - vma->userptr.ptr, size, - &vma_userptr_notifier_ops); - if (err) { - kfree(vma); - vma = ERR_PTR(err); - return vma; + err = mmu_interval_notifier_insert(&vma->userptr.notifier, + current->mm, + vma->userptr.ptr, size, + &vma_userptr_notifier_ops); + if (err) { + kfree(vma); + vma = ERR_PTR(err); + return vma; + } + + vma->userptr.notifier_seq = LONG_MAX; } - vma->userptr.notifier_seq = LONG_MAX; xe_vm_get(vm); } @@ -942,6 +949,8 @@ static void xe_vma_destroy_late(struct xe_vma *vma) */ mmu_interval_notifier_remove(&vma->userptr.notifier); xe_vm_put(vm); + } else if (xe_vma_is_null(vma)) { + xe_vm_put(vm); } else { xe_bo_put(vma->bo); } @@ -1024,7 +1033,7 @@ static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) list_del_init(&vma->userptr.invalidate_link); spin_unlock(&vm->userptr.invalidated_lock); list_del(&vma->userptr_link); - } else { + } else if (!xe_vma_is_null(vma)) { xe_bo_assert_held(vma->bo); list_del(&vma->bo_link); @@ -1393,7 +1402,7 @@ void xe_vm_close_and_put(struct xe_vm *vm) while (vm->vmas.rb_node) { struct xe_vma *vma = to_xe_vma(vm->vmas.rb_node); - if (xe_vma_is_userptr(vma)) { + if (xe_vma_has_no_bo(vma)) { down_read(&vm->userptr.notifier_lock); vma->destroyed = true; up_read(&vm->userptr.notifier_lock); @@ -1402,7 +1411,7 @@ void xe_vm_close_and_put(struct xe_vm *vm) rb_erase(&vma->vm_node, &vm->vmas); /* easy case, remove from VMA? */ - if (xe_vma_is_userptr(vma) || vma->bo->vm) { + if (xe_vma_has_no_bo(vma) || vma->bo->vm) { xe_vma_destroy(vma, NULL); continue; } @@ -2036,7 +2045,7 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, XE_BUG_ON(region > ARRAY_SIZE(region_to_mem_type)); - if (!xe_vma_is_userptr(vma)) { + if (!xe_vma_has_no_bo(vma)) { err = xe_bo_migrate(vma->bo, region_to_mem_type[region]); if (err) return err; @@ -2645,6 +2654,8 @@ static struct xe_vma *vm_unbind_lookup_vmas(struct xe_vm *vm, lookup->start - 1, (first->pte_flags & XE_PTE_FLAG_READ_ONLY), + (first->pte_flags & + XE_PTE_FLAG_NULL), first->tile_mask); if (first->bo) xe_bo_unlock(first->bo, &ww); @@ -2652,7 +2663,7 @@ static struct xe_vma *vm_unbind_lookup_vmas(struct xe_vm *vm, err = -ENOMEM; goto unwind; } - if (!first->bo) { + if (xe_vma_is_userptr(first)) { err = xe_vma_userptr_pin_pages(new_first); if (err) goto unwind; @@ -2677,6 +2688,7 @@ static struct xe_vma *vm_unbind_lookup_vmas(struct xe_vm *vm, last->end, (last->pte_flags & XE_PTE_FLAG_READ_ONLY), + (last->pte_flags & XE_PTE_FLAG_NULL), last->tile_mask); if (last->bo) xe_bo_unlock(last->bo, &ww); @@ -2684,7 +2696,7 @@ static struct xe_vma *vm_unbind_lookup_vmas(struct xe_vm *vm, err = -ENOMEM; goto unwind; } - if (!last->bo) { + if (xe_vma_is_userptr(last)) { err = xe_vma_userptr_pin_pages(new_last); if (err) goto unwind; @@ -2744,7 +2756,7 @@ static struct xe_vma *vm_prefetch_lookup_vmas(struct xe_vm *vm, *next; struct rb_node *node; - if (!xe_vma_is_userptr(vma)) { + if (!xe_vma_has_no_bo(vma)) { if (!xe_bo_can_migrate(vma->bo, region_to_mem_type[region])) return ERR_PTR(-EINVAL); } @@ -2753,7 +2765,7 @@ static struct xe_vma *vm_prefetch_lookup_vmas(struct xe_vm *vm, while ((node = rb_next(node))) { if (!xe_vma_cmp_vma_cb(lookup, node)) { __vma = to_xe_vma(node); - if (!xe_vma_is_userptr(__vma)) { + if (!xe_vma_has_no_bo(__vma)) { if (!xe_bo_can_migrate(__vma->bo, region_to_mem_type[region])) goto flush_list; } @@ -2767,7 +2779,7 @@ static struct xe_vma *vm_prefetch_lookup_vmas(struct xe_vm *vm, while ((node = rb_prev(node))) { if (!xe_vma_cmp_vma_cb(lookup, node)) { __vma = to_xe_vma(node); - if (!xe_vma_is_userptr(__vma)) { + if (!xe_vma_has_no_bo(__vma)) { if (!xe_bo_can_migrate(__vma->bo, region_to_mem_type[region])) goto flush_list; } @@ -2826,21 +2838,23 @@ static struct xe_vma *vm_bind_ioctl_lookup_vma(struct xe_vm *vm, switch (VM_BIND_OP(op)) { case XE_VM_BIND_OP_MAP: - XE_BUG_ON(!bo); - - err = xe_bo_lock(bo, &ww, 0, true); - if (err) - return ERR_PTR(err); + if (bo) { + err = xe_bo_lock(bo, &ww, 0, true); + if (err) + return ERR_PTR(err); + } vma = xe_vma_create(vm, bo, bo_offset_or_userptr, addr, addr + range - 1, op & XE_VM_BIND_FLAG_READONLY, + op & XE_VM_BIND_FLAG_NULL, tile_mask); - xe_bo_unlock(bo, &ww); + if (bo) + xe_bo_unlock(bo, &ww); if (!vma) return ERR_PTR(-ENOMEM); xe_vm_insert_vma(vm, vma); - if (!bo->vm) { + if (bo && !bo->vm) { vm_insert_extobj(vm, vma); err = add_preempt_fences(vm, bo); if (err) { @@ -2874,6 +2888,7 @@ static struct xe_vma *vm_bind_ioctl_lookup_vma(struct xe_vm *vm, vma = xe_vma_create(vm, NULL, bo_offset_or_userptr, addr, addr + range - 1, op & XE_VM_BIND_FLAG_READONLY, + op & XE_VM_BIND_FLAG_NULL, tile_mask); if (!vma) return ERR_PTR(-ENOMEM); @@ -2899,11 +2914,12 @@ static struct xe_vma *vm_bind_ioctl_lookup_vma(struct xe_vm *vm, #ifdef TEST_VM_ASYNC_OPS_ERROR #define SUPPORTED_FLAGS \ (FORCE_ASYNC_OP_ERROR | XE_VM_BIND_FLAG_ASYNC | \ - XE_VM_BIND_FLAG_READONLY | XE_VM_BIND_FLAG_IMMEDIATE | 0xffff) + XE_VM_BIND_FLAG_READONLY | XE_VM_BIND_FLAG_IMMEDIATE | \ + XE_VM_BIND_FLAG_NULL | 0xffff) #else #define SUPPORTED_FLAGS \ (XE_VM_BIND_FLAG_ASYNC | XE_VM_BIND_FLAG_READONLY | \ - XE_VM_BIND_FLAG_IMMEDIATE | 0xffff) + XE_VM_BIND_FLAG_IMMEDIATE | XE_VM_BIND_FLAG_NULL | 0xffff) #endif #define XE_64K_PAGE_MASK 0xffffull @@ -2951,6 +2967,7 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, u32 obj = (*bind_ops)[i].obj; u64 obj_offset = (*bind_ops)[i].obj_offset; u32 region = (*bind_ops)[i].region; + bool is_null = op & XE_VM_BIND_FLAG_NULL; if (XE_IOCTL_ERR(xe, (*bind_ops)[i].pad) || XE_IOCTL_ERR(xe, (*bind_ops)[i].reserved[0] || @@ -2984,8 +3001,13 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, if (XE_IOCTL_ERR(xe, VM_BIND_OP(op) > XE_VM_BIND_OP_PREFETCH) || XE_IOCTL_ERR(xe, op & ~SUPPORTED_FLAGS) || + XE_IOCTL_ERR(xe, obj && is_null) || + XE_IOCTL_ERR(xe, obj_offset && is_null) || + XE_IOCTL_ERR(xe, VM_BIND_OP(op) != XE_VM_BIND_OP_MAP && + is_null) || XE_IOCTL_ERR(xe, !obj && - VM_BIND_OP(op) == XE_VM_BIND_OP_MAP) || + VM_BIND_OP(op) == XE_VM_BIND_OP_MAP && + !is_null) || XE_IOCTL_ERR(xe, !obj && VM_BIND_OP(op) == XE_VM_BIND_OP_UNMAP_ALL) || XE_IOCTL_ERR(xe, addr && @@ -3390,6 +3412,7 @@ int xe_vm_invalidate_vma(struct xe_vma *vma) int ret; XE_BUG_ON(!xe_vm_in_fault_mode(vma->vm)); + XE_WARN_ON(xe_vma_is_null(vma)); trace_xe_vma_usm_invalidate(vma); /* Check that we don't race with page-table updates */ @@ -3452,8 +3475,11 @@ int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id) for (node = rb_first(&vm->vmas); node; node = rb_next(node)) { struct xe_vma *vma = to_xe_vma(node); bool is_userptr = xe_vma_is_userptr(vma); + bool is_null = xe_vma_is_null(vma); - if (is_userptr) { + if (is_null) { + addr = 0; + } else if (is_userptr) { struct xe_res_cursor cur; if (vma->userptr.sg) { @@ -3468,7 +3494,8 @@ int xe_analyze_vm(struct drm_printer *p, struct xe_vm *vm, int gt_id) } drm_printf(p, " [%016llx-%016llx] S:0x%016llx A:%016llx %s\n", vma->start, vma->end, vma->end - vma->start + 1ull, - addr, is_userptr ? "USR" : is_vram ? "VRAM" : "SYS"); + addr, is_null ? "NULL" : is_userptr ? "USR" : + is_vram ? "VRAM" : "SYS"); } up_read(&vm->lock); diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index bb2996856841..5edb7771629c 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -115,11 +115,21 @@ static inline void xe_vm_reactivate_rebind(struct xe_vm *vm) } } -static inline bool xe_vma_is_userptr(struct xe_vma *vma) +static inline bool xe_vma_is_null(struct xe_vma *vma) +{ + return vma->pte_flags & XE_PTE_FLAG_NULL; +} + +static inline bool xe_vma_has_no_bo(struct xe_vma *vma) { return !vma->bo; } +static inline bool xe_vma_is_userptr(struct xe_vma *vma) +{ + return xe_vma_has_no_bo(vma) && !xe_vma_is_null(vma); +} + int xe_vma_userptr_pin_pages(struct xe_vma *vma); int xe_vma_userptr_check_repin(struct xe_vma *vma); diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index a51e84e584b4..9b39c5f64afa 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -31,6 +31,7 @@ struct xe_vma { u64 end; /** @pte_flags: pte flags for this VMA */ #define XE_PTE_FLAG_READ_ONLY BIT(0) +#define XE_PTE_FLAG_NULL BIT(1) u32 pte_flags; /** @bo: BO if not a userptr, must be NULL is userptr */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 83868af45984..6a991afc563d 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -418,6 +418,14 @@ struct drm_xe_vm_bind_op { * than differing the MAP to the page fault handler. */ #define XE_VM_BIND_FLAG_IMMEDIATE (0x1 << 18) + /* + * When the NULL flag is set, the page tables are setup with a special + * bit which indicates writes are dropped and all reads return zero. In + * the future, the NULL flags will only be valid for XE_VM_BIND_OP_MAP + * operations, the BO handle MBZ, and the BO offset MBZ. This flag is + * intended to implement VK sparse bindings. + */ +#define XE_VM_BIND_FLAG_NULL (0x1 << 19) /** @reserved: Reserved */ __u64 reserved[2]; -- cgit v1.2.3 From ffd6620fb746c59ad82070f1975c4a0e3d30520e Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 9 Jun 2023 07:37:12 +0000 Subject: drm/xe: Document structures for device query This adds documentation to the various structures used to query memory, GTs, topology, engines, and so on. It includes a functional code snippet to query engines. v2: - Rebase on drm-xe-next - Also document structures related to drm_xe_device_query, changed pseudo code to snippet (Lucas De Marchi) v3: - Move changelog to commit - Fix warnings showed only using dim checkpath Reported-by: Oded Gabbay Link: https://lists.freedesktop.org/archives/intel-xe/2023-May/004704.html Signed-off-by: Francois Dugast Reviewed-by: Lucas De Marchi Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 75 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 6a991afc563d..445f7b7689dd 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -119,8 +119,18 @@ struct xe_user_extension { #define XE_MEM_REGION_CLASS_SYSMEM 0 #define XE_MEM_REGION_CLASS_VRAM 1 +/** + * struct drm_xe_query_mem_usage - describe memory regions and usage + * + * If a query is made with a struct drm_xe_device_query where .query + * is equal to DRM_XE_DEVICE_QUERY_MEM_USAGE, then the reply uses + * struct drm_xe_query_mem_usage in .data. + */ struct drm_xe_query_mem_usage { + /** @num_params: number of memory regions returned in regions */ __u32 num_regions; + + /** @pad: MBZ */ __u32 pad; struct drm_xe_query_mem_region { @@ -135,9 +145,20 @@ struct drm_xe_query_mem_usage { } regions[]; }; +/** + * struct drm_xe_query_config - describe the device configuration + * + * If a query is made with a struct drm_xe_device_query where .query + * is equal to DRM_XE_DEVICE_QUERY_CONFIG, then the reply uses + * struct drm_xe_query_config in .data. + */ struct drm_xe_query_config { + /** @num_params: number of parameters returned in info */ __u32 num_params; + + /** @pad: MBZ */ __u32 pad; + #define XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 #define XE_QUERY_CONFIG_FLAGS 1 #define XE_QUERY_CONFIG_FLAGS_HAS_VRAM (0x1 << 0) @@ -148,11 +169,22 @@ struct drm_xe_query_config { #define XE_QUERY_CONFIG_MEM_REGION_COUNT 5 #define XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY 6 #define XE_QUERY_CONFIG_NUM_PARAM (XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY + 1) + /** @info: array of elements containing the config info */ __u64 info[]; }; +/** + * struct drm_xe_query_gts - describe GTs + * + * If a query is made with a struct drm_xe_device_query where .query + * is equal to DRM_XE_DEVICE_QUERY_GTS, then the reply uses struct + * drm_xe_query_gts in .data. + */ struct drm_xe_query_gts { + /** @num_gt: number of GTs returned in gts */ __u32 num_gt; + + /** @pad: MBZ */ __u32 pad; /* @@ -175,6 +207,13 @@ struct drm_xe_query_gts { } gts[]; }; +/** + * struct drm_xe_query_topology_mask - describe the topology mask of a GT + * + * If a query is made with a struct drm_xe_device_query where .query + * is equal to DRM_XE_DEVICE_QUERY_GT_TOPOLOGY, then the reply uses + * struct drm_xe_query_topology_mask in .data. + */ struct drm_xe_query_topology_mask { /** @gt_id: GT ID the mask is associated with */ __u16 gt_id; @@ -192,6 +231,41 @@ struct drm_xe_query_topology_mask { __u8 mask[]; }; +/** + * struct drm_xe_device_query - main structure to query device information + * + * If size is set to 0, the driver fills it with the required size for the + * requested type of data to query. If size is equal to the required size, + * the queried information is copied into data. + * + * For example the following code snippet allows retrieving and printing + * information about the device engines with DRM_XE_DEVICE_QUERY_ENGINES: + * + * .. code-block:: C + * + * struct drm_xe_engine_class_instance *hwe; + * struct drm_xe_device_query query = { + * .extensions = 0, + * .query = DRM_XE_DEVICE_QUERY_ENGINES, + * .size = 0, + * .data = 0, + * }; + * ioctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query); + * hwe = malloc(query.size); + * query.data = (uintptr_t)hwe; + * ioctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query); + * int num_engines = query.size / sizeof(*hwe); + * for (int i = 0; i < num_engines; i++) { + * printf("Engine %d: %s\n", i, + * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_RENDER ? "RENDER": + * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_COPY ? "COPY": + * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_VIDEO_DECODE ? "VIDEO_DECODE": + * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE ? "VIDEO_ENHANCE": + * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_COMPUTE ? "COMPUTE": + * "UNKNOWN"); + * } + * free(hwe); + */ struct drm_xe_device_query { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -526,6 +600,7 @@ struct drm_xe_engine_set_property { __u64 reserved[2]; }; +/** struct drm_xe_engine_class_instance - instance of an engine class */ struct drm_xe_engine_class_instance { __u16 engine_class; -- cgit v1.2.3 From 4f082f2c3a37d1b2fb90e048cc61616885b69648 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Thu, 22 Jun 2023 13:59:20 +0200 Subject: drm/xe: Move defines before relevant fields Align on same rule in the whole file: defines then doc then relevant field, with an empty line to separate fields. v2: - Rebase on drm-xe-next - Fix ordering of defines and fields in uAPI (Lucas De Marchi) v3: Remove useless empty lines (Lucas De Marchi) v4: Move changelog to commit v5: Rebase Reported-by: Oded Gabbay Link: https://lists.freedesktop.org/archives/intel-xe/2023-May/004704.html Signed-off-by: Francois Dugast Reviewed-by: Lucas De Marchi Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 73 +++++++++++++++++++++++++++-------------------- 1 file changed, 42 insertions(+), 31 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 445f7b7689dd..be62b3a06db9 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -60,6 +60,7 @@ struct xe_user_extension { * Pointer to the next struct xe_user_extension, or zero if the end. */ __u64 next_extension; + /** * @name: Name of the extension. * @@ -70,6 +71,7 @@ struct xe_user_extension { * of uAPI which has embedded the struct xe_user_extension. */ __u32 name; + /** * @pad: MBZ * @@ -218,11 +220,11 @@ struct drm_xe_query_topology_mask { /** @gt_id: GT ID the mask is associated with */ __u16 gt_id; - /** @type: type of mask */ - __u16 type; #define XE_TOPO_DSS_GEOMETRY (1 << 0) #define XE_TOPO_DSS_COMPUTE (1 << 1) #define XE_TOPO_EU_PER_DSS (1 << 2) + /** @type: type of mask */ + __u16 type; /** @num_bytes: number of bytes in requested mask */ __u32 num_bytes; @@ -270,15 +272,14 @@ struct drm_xe_device_query { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; - /** @query: The type of data to query */ - __u32 query; - #define DRM_XE_DEVICE_QUERY_ENGINES 0 #define DRM_XE_DEVICE_QUERY_MEM_USAGE 1 #define DRM_XE_DEVICE_QUERY_CONFIG 2 #define DRM_XE_DEVICE_QUERY_GTS 3 #define DRM_XE_DEVICE_QUERY_HWCONFIG 4 #define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5 + /** @query: The type of data to query */ + __u32 query; /** @size: Size of the queried data */ __u32 size; @@ -301,12 +302,12 @@ struct drm_xe_gem_create { */ __u64 size; +#define XE_GEM_CREATE_FLAG_DEFER_BACKING (0x1 << 24) +#define XE_GEM_CREATE_FLAG_SCANOUT (0x1 << 25) /** * @flags: Flags, currently a mask of memory instances of where BO can * be placed */ -#define XE_GEM_CREATE_FLAG_DEFER_BACKING (0x1 << 24) -#define XE_GEM_CREATE_FLAG_SCANOUT (0x1 << 25) __u32 flags; /** @@ -357,10 +358,13 @@ struct drm_xe_gem_mmap_offset { struct drm_xe_vm_bind_op_error_capture { /** @error: errno that occured */ __s32 error; + /** @op: operation that encounter an error */ __u32 op; + /** @addr: address of bind op */ __u64 addr; + /** @size: size of bind */ __u64 size; }; @@ -370,8 +374,8 @@ struct drm_xe_ext_vm_set_property { /** @base: base user extension */ struct xe_user_extension base; - /** @property: property to set */ #define XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS 0 + /** @property: property to set */ __u32 property; /** @pad: MBZ */ @@ -385,17 +389,16 @@ struct drm_xe_ext_vm_set_property { }; struct drm_xe_vm_create { - /** @extensions: Pointer to the first extension struct, if any */ #define XE_VM_EXTENSION_SET_PROPERTY 0 + /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; - /** @flags: Flags */ - __u32 flags; - #define DRM_XE_VM_CREATE_SCRATCH_PAGE (0x1 << 0) #define DRM_XE_VM_CREATE_COMPUTE_MODE (0x1 << 1) #define DRM_XE_VM_CREATE_ASYNC_BIND_OPS (0x1 << 2) #define DRM_XE_VM_CREATE_FAULT_MODE (0x1 << 3) + /** @flags: Flags */ + __u32 flags; /** @vm_id: Returned VM ID */ __u32 vm_id; @@ -430,6 +433,7 @@ struct drm_xe_vm_bind_op { * ignored for unbind */ __u64 obj_offset; + /** @userptr: user pointer to bind on */ __u64 userptr; }; @@ -448,12 +452,6 @@ struct drm_xe_vm_bind_op { */ __u64 tile_mask; - /** @op: Operation to perform (lower 16 bits) and flags (upper 16 bits) */ - __u32 op; - - /** @mem_region: Memory region to prefetch VMA to, instance not a mask */ - __u32 region; - #define XE_VM_BIND_OP_MAP 0x0 #define XE_VM_BIND_OP_UNMAP 0x1 #define XE_VM_BIND_OP_MAP_USERPTR 0x2 @@ -500,6 +498,11 @@ struct drm_xe_vm_bind_op { * intended to implement VK sparse bindings. */ #define XE_VM_BIND_FLAG_NULL (0x1 << 19) + /** @op: Operation to perform (lower 16 bits) and flags (upper 16 bits) */ + __u32 op; + + /** @mem_region: Memory region to prefetch VMA to, instance not a mask */ + __u32 region; /** @reserved: Reserved */ __u64 reserved[2]; @@ -528,6 +531,7 @@ struct drm_xe_vm_bind { union { /** @bind: used if num_binds == 1 */ struct drm_xe_vm_bind_op bind; + /** * @vector_of_binds: userptr to array of struct * drm_xe_vm_bind_op if num_binds > 1 @@ -575,7 +579,6 @@ struct drm_xe_engine_set_property { /** @engine_id: Engine ID */ __u32 engine_id; - /** @property: property to set */ #define XE_ENGINE_SET_PROPERTY_PRIORITY 0 #define XE_ENGINE_SET_PROPERTY_TIMESLICE 1 #define XE_ENGINE_SET_PROPERTY_PREEMPTION_TIMEOUT 2 @@ -591,6 +594,7 @@ struct drm_xe_engine_set_property { #define XE_ENGINE_SET_PROPERTY_ACC_TRIGGER 6 #define XE_ENGINE_SET_PROPERTY_ACC_NOTIFY 7 #define XE_ENGINE_SET_PROPERTY_ACC_GRANULARITY 8 + /** @property: property to set */ __u32 property; /** @value: property value */ @@ -602,8 +606,6 @@ struct drm_xe_engine_set_property { /** struct drm_xe_engine_class_instance - instance of an engine class */ struct drm_xe_engine_class_instance { - __u16 engine_class; - #define DRM_XE_ENGINE_CLASS_RENDER 0 #define DRM_XE_ENGINE_CLASS_COPY 1 #define DRM_XE_ENGINE_CLASS_VIDEO_DECODE 2 @@ -614,14 +616,15 @@ struct drm_xe_engine_class_instance { * creating ordered queues of VM bind operations. */ #define DRM_XE_ENGINE_CLASS_VM_BIND 5 + __u16 engine_class; __u16 engine_instance; __u16 gt_id; }; struct drm_xe_engine_create { - /** @extensions: Pointer to the first extension struct, if any */ #define XE_ENGINE_EXTENSION_SET_PROPERTY 0 + /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; /** @width: submission width (number BB per exec) for this engine */ @@ -659,8 +662,8 @@ struct drm_xe_engine_get_property { /** @engine_id: Engine ID */ __u32 engine_id; - /** @property: property to get */ #define XE_ENGINE_GET_PROPERTY_BAN 0 + /** @property: property to get */ __u32 property; /** @value: property value */ @@ -685,19 +688,19 @@ struct drm_xe_sync { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; - __u32 flags; - #define DRM_XE_SYNC_SYNCOBJ 0x0 #define DRM_XE_SYNC_TIMELINE_SYNCOBJ 0x1 #define DRM_XE_SYNC_DMA_BUF 0x2 #define DRM_XE_SYNC_USER_FENCE 0x3 #define DRM_XE_SYNC_SIGNAL 0x10 + __u32 flags; /** @pad: MBZ */ __u32 pad; union { __u32 handle; + /** * @addr: Address of user fence. When sync passed in via exec * IOCTL this a GPU address in the VM. When sync passed in via @@ -753,8 +756,6 @@ struct drm_xe_mmio { __u32 addr; - __u32 flags; - #define DRM_XE_MMIO_8BIT 0x0 #define DRM_XE_MMIO_16BIT 0x1 #define DRM_XE_MMIO_32BIT 0x2 @@ -762,6 +763,7 @@ struct drm_xe_mmio { #define DRM_XE_MMIO_BITS_MASK 0x3 #define DRM_XE_MMIO_READ 0x4 #define DRM_XE_MMIO_WRITE 0x8 + __u32 flags; __u64 value; @@ -781,47 +783,57 @@ struct drm_xe_mmio { struct drm_xe_wait_user_fence { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; + union { /** * @addr: user pointer address to wait on, must qword aligned */ __u64 addr; + /** * @vm_id: The ID of the VM which encounter an error used with * DRM_XE_UFENCE_WAIT_VM_ERROR. Upper 32 bits must be clear. */ __u64 vm_id; }; - /** @op: wait operation (type of comparison) */ + #define DRM_XE_UFENCE_WAIT_EQ 0 #define DRM_XE_UFENCE_WAIT_NEQ 1 #define DRM_XE_UFENCE_WAIT_GT 2 #define DRM_XE_UFENCE_WAIT_GTE 3 #define DRM_XE_UFENCE_WAIT_LT 4 #define DRM_XE_UFENCE_WAIT_LTE 5 + /** @op: wait operation (type of comparison) */ __u16 op; - /** @flags: wait flags */ + #define DRM_XE_UFENCE_WAIT_SOFT_OP (1 << 0) /* e.g. Wait on VM bind */ #define DRM_XE_UFENCE_WAIT_ABSTIME (1 << 1) #define DRM_XE_UFENCE_WAIT_VM_ERROR (1 << 2) + /** @flags: wait flags */ __u16 flags; + /** @pad: MBZ */ __u32 pad; + /** @value: compare value */ __u64 value; - /** @mask: comparison mask */ + #define DRM_XE_UFENCE_WAIT_U8 0xffu #define DRM_XE_UFENCE_WAIT_U16 0xffffu #define DRM_XE_UFENCE_WAIT_U32 0xffffffffu #define DRM_XE_UFENCE_WAIT_U64 0xffffffffffffffffu + /** @mask: comparison mask */ __u64 mask; + /** @timeout: how long to wait before bailing, value in jiffies */ __s64 timeout; + /** * @num_engines: number of engine instances to wait on, must be zero * when DRM_XE_UFENCE_WAIT_SOFT_OP set */ __u64 num_engines; + /** * @instances: user pointer to array of drm_xe_engine_class_instance to * wait on, must be NULL when DRM_XE_UFENCE_WAIT_SOFT_OP set @@ -882,7 +894,6 @@ struct drm_xe_vm_madvise { #define DRM_XE_VMA_PRIORITY_HIGH 2 /* Must be elevated user */ /* Pin the VMA in memory, must be elevated user */ #define DRM_XE_VM_MADVISE_PIN 6 - /** @property: property to set */ __u32 property; -- cgit v1.2.3 From 1bc56a934f11cc9bb859116d30e828ccf2df54cf Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Thu, 22 Jun 2023 14:32:03 +0200 Subject: drm/xe: Document topology mask query Provide information on the types of topology masks that can be queried and add some examples. Signed-off-by: Francois Dugast Reviewed-by: Lucas De Marchi Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index be62b3a06db9..fef5e26aad2a 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -212,6 +212,9 @@ struct drm_xe_query_gts { /** * struct drm_xe_query_topology_mask - describe the topology mask of a GT * + * This is the hardware topology which reflects the internal physical + * structure of the GPU. + * * If a query is made with a struct drm_xe_device_query where .query * is equal to DRM_XE_DEVICE_QUERY_GT_TOPOLOGY, then the reply uses * struct drm_xe_query_topology_mask in .data. @@ -220,8 +223,29 @@ struct drm_xe_query_topology_mask { /** @gt_id: GT ID the mask is associated with */ __u16 gt_id; + /* + * To query the mask of Dual Sub Slices (DSS) available for geometry + * operations. For example a query response containing the following + * in mask: + * DSS_GEOMETRY ff ff ff ff 00 00 00 00 + * means 32 DSS are available for geometry. + */ #define XE_TOPO_DSS_GEOMETRY (1 << 0) + /* + * To query the mask of Dual Sub Slices (DSS) available for compute + * operations. For example a query response containing the following + * in mask: + * DSS_COMPUTE ff ff ff ff 00 00 00 00 + * means 32 DSS are available for compute. + */ #define XE_TOPO_DSS_COMPUTE (1 << 1) + /* + * To query the mask of Execution Units (EU) available per Dual Sub + * Slices (DSS). For example a query response containing the following + * in mask: + * EU_PER_DSS ff ff 00 00 00 00 00 00 + * means each DSS has 16 EU. + */ #define XE_TOPO_EU_PER_DSS (1 << 2) /** @type: type of mask */ __u16 type; -- cgit v1.2.3 From a9c4a069fbc3a1e115fead47145bc0257a7b3509 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Fri, 31 Mar 2023 09:46:25 +0100 Subject: drm/xe/uapi: add some kernel-doc for region query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since we need to extend this, we should also take the time to add some basic kernel-doc here for the existing bits. Note that this is all still subject to change when upstreaming. Also convert XE_MEM_REGION_CLASS_* into an enum, so we can more easily create links to it from other parts of the uapi. Suggested-by: Gwan-gyeong Mun Signed-off-by: Matthew Auld Cc: Maarten Lankhorst Cc: Thomas Hellström Cc: Lucas De Marchi Cc: José Roberto de Souza Cc: Filip Hazubski Cc: Carl Zhang Cc: Effie Yu Reviewed-by: Gwan-gyeong Mun Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 86 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 15 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index fef5e26aad2a..0808b21de29a 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -118,8 +118,71 @@ struct xe_user_extension { #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) #define DRM_IOCTL_XE_VM_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_MADVISE, struct drm_xe_vm_madvise) -#define XE_MEM_REGION_CLASS_SYSMEM 0 -#define XE_MEM_REGION_CLASS_VRAM 1 +/** + * enum drm_xe_memory_class - Supported memory classes. + */ +enum drm_xe_memory_class { + /** @XE_MEM_REGION_CLASS_SYSMEM: Represents system memory. */ + XE_MEM_REGION_CLASS_SYSMEM = 0, + /** + * @XE_MEM_REGION_CLASS_VRAM: On discrete platforms, this + * represents the memory that is local to the device, which we + * call VRAM. Not valid on integrated platforms. + */ + XE_MEM_REGION_CLASS_VRAM +}; + +/** + * struct drm_xe_query_mem_region - Describes some region as known to + * the driver. + */ +struct drm_xe_query_mem_region { + /** + * @mem_class: The memory class describing this region. + * + * See enum drm_xe_memory_class for supported values. + */ + __u16 mem_class; + /** + * @instance: The instance for this region. + * + * The @mem_class and @instance taken together will always give + * a unique pair. + */ + __u16 instance; + /** @pad: MBZ */ + __u32 pad; + /** + * @min_page_size: Min page-size in bytes for this region. + * + * When the kernel allocates memory for this region, the + * underlying pages will be at least @min_page_size in size. + * + * Important note: When userspace allocates a GTT address which + * can point to memory allocated from this region, it must also + * respect this minimum alignment. This is enforced by the + * kernel. + */ + __u32 min_page_size; + /** + * @max_page_size: Max page-size in bytes for this region. + */ + __u32 max_page_size; + /** + * @total_size: The usable size in bytes for this region. + */ + __u64 total_size; + /** + * @used: Estimate of the memory used in bytes for this region. + * + * Requires CAP_PERFMON or CAP_SYS_ADMIN to get reliable + * accounting. Without this the value here will always equal + * zero. + */ + __u64 used; + /** @reserved: MBZ */ + __u64 reserved[8]; +}; /** * struct drm_xe_query_mem_usage - describe memory regions and usage @@ -129,22 +192,12 @@ struct xe_user_extension { * struct drm_xe_query_mem_usage in .data. */ struct drm_xe_query_mem_usage { - /** @num_params: number of memory regions returned in regions */ + /** @num_regions: number of memory regions returned in @regions */ __u32 num_regions; - /** @pad: MBZ */ __u32 pad; - - struct drm_xe_query_mem_region { - __u16 mem_class; - __u16 instance; /* unique ID even among different classes */ - __u32 pad; - __u32 min_page_size; - __u32 max_page_size; - __u64 total_size; - __u64 used; - __u64 reserved[8]; - } regions[]; + /** @regions: The returned regions for this device */ + struct drm_xe_query_mem_region regions[]; }; /** @@ -888,6 +941,9 @@ struct drm_xe_vm_madvise { * Setting the preferred location will trigger a migrate of the VMA * backing store to new location if the backing store is already * allocated. + * + * For DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS usage, see enum + * drm_xe_memory_class. */ #define DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS 0 #define DRM_XE_VM_MADVISE_PREFERRED_GT 1 -- cgit v1.2.3 From 63f9c3cd36cad69d4422d86b2f86675f93df521a Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Mon, 26 Jun 2023 09:25:07 +0100 Subject: drm/xe/uapi: silence kernel-doc errors ./include/uapi/drm/xe_drm.h:263: warning: Function parameter or member 'gts' not described in 'drm_xe_query_gts' ./include/uapi/drm/xe_drm.h:854: WARNING: Inline emphasis start-string without end-string. With the idea to also include the uapi file in the pre-merge CI hooks when building the kernel-doc, so first make sure it's clean: https://gitlab.freedesktop.org/drm/xe/ci/-/merge_requests/16 v2: (Francois) - It makes more sense to just fix the kernel-doc for 'gts' Signed-off-by: Matthew Auld Cc: Francois Dugast Cc: Lucas De Marchi Reviewed-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 0808b21de29a..8e7be1551333 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -242,11 +242,13 @@ struct drm_xe_query_gts { /** @pad: MBZ */ __u32 pad; - /* + /** + * @gts: The GTs returned for this device + * + * TODO: convert drm_xe_query_gt to proper kernel-doc. * TODO: Perhaps info about every mem region relative to this GT? e.g. * bandwidth between this GT and remote region? */ - struct drm_xe_query_gt { #define XE_QUERY_GT_TYPE_MAIN 0 #define XE_QUERY_GT_TYPE_REMOTE 1 @@ -852,8 +854,9 @@ struct drm_xe_mmio { * struct drm_xe_wait_user_fence - wait user fence * * Wait on user fence, XE will wakeup on every HW engine interrupt in the - * instances list and check if user fence is complete: - * (*addr & MASK) OP (VALUE & MASK) + * instances list and check if user fence is complete:: + * + * (*addr & MASK) OP (VALUE & MASK) * * Returns to user on user fence completion or timeout. */ -- cgit v1.2.3 From 5572a004685770f8daad7661c5494b65148ede9f Mon Sep 17 00:00:00 2001 From: Zbigniew Kempczyński Date: Wed, 28 Jun 2023 07:51:41 +0200 Subject: drm/xe: Use nanoseconds instead of jiffies in uapi for user fence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Using jiffies as a timeout from userspace is weird even if theoretically exists possiblity of acquiring jiffies via getconf. Unfortunately this method is unreliable and the returned value may vary from the one configured in the kernel config. Now timeout is expressed in nanoseconds and its interpretation depends on setting DRM_XE_UFENCE_WAIT_ABSTIME flag. Relative timeout (flag is not set) means fence expire at now() + timeout. Absolute timeout (flag is set) means that the fence expires at exact point of time. Passing negative timeout means we will wait "forever" by setting wait time to MAX_SCHEDULE_TIMEOUT. Cc: Andi Shyti Reviewed-by: Andi Shyti Link: https://lore.kernel.org/r/20230628055141.398036-2-zbigniew.kempczynski@intel.com Signed-off-by: Zbigniew Kempczyński Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_wait_user_fence.c | 47 ++++++++++++++++++++++++++------- include/uapi/drm/xe_drm.h | 16 +++++++++-- 2 files changed, 51 insertions(+), 12 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.c b/drivers/gpu/drm/xe/xe_wait_user_fence.c index 098e2a4cff3f..c4420c0dbf9c 100644 --- a/drivers/gpu/drm/xe/xe_wait_user_fence.c +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.c @@ -7,6 +7,7 @@ #include #include +#include #include #include "xe_device.h" @@ -84,6 +85,21 @@ static int check_hw_engines(struct xe_device *xe, DRM_XE_UFENCE_WAIT_VM_ERROR) #define MAX_OP DRM_XE_UFENCE_WAIT_LTE +static unsigned long to_jiffies_timeout(struct drm_xe_wait_user_fence *args) +{ + unsigned long timeout; + + if (args->flags & DRM_XE_UFENCE_WAIT_ABSTIME) + return drm_timeout_abs_to_jiffies(args->timeout); + + if (args->timeout == MAX_SCHEDULE_TIMEOUT || args->timeout == 0) + return args->timeout; + + timeout = nsecs_to_jiffies(args->timeout); + + return timeout ?: 1; +} + int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { @@ -98,7 +114,8 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, int err; bool no_engines = args->flags & DRM_XE_UFENCE_WAIT_SOFT_OP || args->flags & DRM_XE_UFENCE_WAIT_VM_ERROR; - unsigned long timeout = args->timeout; + unsigned long timeout; + ktime_t start; if (XE_IOCTL_ERR(xe, args->extensions) || XE_IOCTL_ERR(xe, args->pad) || XE_IOCTL_ERR(xe, args->reserved[0] || args->reserved[1])) @@ -152,8 +169,18 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, addr = vm->async_ops.error_capture.addr; } - if (XE_IOCTL_ERR(xe, timeout > MAX_SCHEDULE_TIMEOUT)) - return -EINVAL; + /* + * For negative timeout we want to wait "forever" by setting + * MAX_SCHEDULE_TIMEOUT. But we have to assign this value also + * to args->timeout to avoid being zeroed on the signal delivery + * (see arithmetics after wait). + */ + if (args->timeout < 0) + args->timeout = MAX_SCHEDULE_TIMEOUT; + + timeout = to_jiffies_timeout(args); + + start = ktime_get(); /* * FIXME: Very simple implementation at the moment, single wait queue @@ -192,17 +219,17 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, } else { remove_wait_queue(&xe->ufence_wq, &w_wait); } + + if (!(args->flags & DRM_XE_UFENCE_WAIT_ABSTIME)) { + args->timeout -= ktime_to_ns(ktime_sub(ktime_get(), start)); + if (args->timeout < 0) + args->timeout = 0; + } + if (XE_IOCTL_ERR(xe, err < 0)) return err; else if (XE_IOCTL_ERR(xe, !timeout)) return -ETIME; - /* - * Again very simple, return the time in jiffies that has past, may need - * a more precision - */ - if (args->flags & DRM_XE_UFENCE_WAIT_ABSTIME) - args->timeout = args->timeout - timeout; - return 0; } diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 8e7be1551333..347351a8f618 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -904,8 +904,20 @@ struct drm_xe_wait_user_fence { #define DRM_XE_UFENCE_WAIT_U64 0xffffffffffffffffu /** @mask: comparison mask */ __u64 mask; - - /** @timeout: how long to wait before bailing, value in jiffies */ + /** + * @timeout: how long to wait before bailing, value in nanoseconds. + * Without DRM_XE_UFENCE_WAIT_ABSTIME flag set (relative timeout) + * it contains timeout expressed in nanoseconds to wait (fence will + * expire at now() + timeout). + * When DRM_XE_UFENCE_WAIT_ABSTIME flat is set (absolute timeout) wait + * will end at timeout (uses system MONOTONIC_CLOCK). + * Passing negative timeout leads to neverending wait. + * + * On relative timeout this value is updated with timeout left + * (for restarting the call in case of signal delivery). + * On absolute timeout this value stays intact (restarted call still + * expire at the same point of time). + */ __s64 timeout; /** -- cgit v1.2.3 From cd928fced9968558f1c7d724c23b1f8868c39774 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Fri, 31 Mar 2023 09:46:27 +0100 Subject: drm/xe/uapi: add the userspace bits for small-bar MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mostly the same as i915. We add a new hint for userspace to force an object into the mappable part of vram. We also need to tell userspace how large the mappable part is. In Vulkan for example, there will be two vram heaps for small-bar systems. And here the size of each heap needs to be known. Likewise the used/avail tracking needs to account for the mappable part. We also limit the available tracking going forward, such that we limit to privileged users only, since these values are system wide and are technically considered an info leak. v2 (Maarten): - s/NEEDS_CPU_ACCESS/NEEDS_VISIBLE_VRAM/ in the uapi. We also no longer require smem as an extra placement. This is more flexible, and lets us use this for clear-color surfaces, since we need CPU access there but we don't want to attach smem, since that effectively disables CCS from kernel pov. - Reject clear-color CCS buffers where NEEDS_VISIBLE_VRAM is not set, instead of migrating it behind the scenes. v3 (José): - Split the changes that limit the accounting for perfmon_capable() into a separate patch. - Use XE_BO_CREATE_VRAM_MASK. v4 (Gwan-gyeong Mun): - Add some kernel-doc for the query bits. v5: - One small kernel-doc correction. The cpu_visible_size and corresponding used tracking are always zero for non XE_MEM_REGION_CLASS_VRAM. v6: - Without perfmon_capable() it likely makes more sense to report as zero, instead of reporting as used == total size. This should give similar behaviour as i915 which rather tracks free instead of used. - Only enforce NEEDS_VISIBLE_VRAM on rc_ccs_cc_plane surfaces when the device is actually small-bar. Testcase: igt/tests/xe_query Testcase: igt/tests/xe_mmap@small-bar Signed-off-by: Matthew Auld Cc: Maarten Lankhorst Cc: Thomas Hellström Cc: Gwan-gyeong Mun Cc: Lucas De Marchi Cc: José Roberto de Souza Cc: Filip Hazubski Cc: Carl Zhang Cc: Effie Yu Reviewed-by: José Roberto de Souza Reviewed-by: Gwan-gyeong Mun Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_bo.c | 13 ++++++++-- drivers/gpu/drm/xe/xe_query.c | 8 ++++-- drivers/gpu/drm/xe/xe_ttm_vram_mgr.c | 18 ++++++++++++++ drivers/gpu/drm/xe/xe_ttm_vram_mgr.h | 4 +++ include/uapi/drm/xe_drm.h | 47 +++++++++++++++++++++++++++++++++++- 5 files changed, 85 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index fa3fc825b730..d89cf93acb61 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1109,7 +1109,6 @@ static vm_fault_t xe_gem_fault(struct vm_fault *vmf) ret = ttm_bo_vm_fault_reserved(vmf, vmf->vma->vm_page_prot, TTM_BO_VM_NUM_PREFAULT); - drm_dev_exit(idx); } else { ret = ttm_bo_vm_dummy_page(vmf, vmf->vma->vm_page_prot); @@ -1760,6 +1759,7 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, args->flags & ~(XE_GEM_CREATE_FLAG_DEFER_BACKING | XE_GEM_CREATE_FLAG_SCANOUT | + XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM | xe->info.mem_region_mask))) return -EINVAL; @@ -1797,6 +1797,14 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, bo_flags |= XE_BO_SCANOUT_BIT; bo_flags |= args->flags << (ffs(XE_BO_CREATE_SYSTEM_BIT) - 1); + + if (args->flags & XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) { + if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_CREATE_VRAM_MASK))) + return -EINVAL; + + bo_flags |= XE_BO_NEEDS_CPU_ACCESS; + } + bo = xe_bo_create(xe, NULL, vm, args->size, ttm_bo_type_device, bo_flags); if (IS_ERR(bo)) { @@ -2081,7 +2089,8 @@ int xe_bo_dumb_create(struct drm_file *file_priv, bo = xe_bo_create(xe, NULL, NULL, args->size, ttm_bo_type_device, XE_BO_CREATE_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | - XE_BO_CREATE_USER_BIT | XE_BO_SCANOUT_BIT); + XE_BO_CREATE_USER_BIT | XE_BO_SCANOUT_BIT | + XE_BO_NEEDS_CPU_ACCESS); if (IS_ERR(bo)) return PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index f880c9af1651..3997c644f8fc 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -17,6 +17,7 @@ #include "xe_gt.h" #include "xe_guc_hwconfig.h" #include "xe_macros.h" +#include "xe_ttm_vram_mgr.h" static const enum xe_engine_class xe_to_user_engine_class[] = { [XE_ENGINE_CLASS_RENDER] = DRM_XE_ENGINE_CLASS_RENDER, @@ -148,10 +149,13 @@ static int query_memory_usage(struct xe_device *xe, man->size; if (perfmon_capable()) { - usage->regions[usage->num_regions].used = - ttm_resource_manager_usage(man); + xe_ttm_vram_get_used(man, + &usage->regions[usage->num_regions].used, + &usage->regions[usage->num_regions].cpu_visible_used); } + usage->regions[usage->num_regions].cpu_visible_size = + xe_ttm_vram_get_cpu_visible_size(man); usage->num_regions++; } } diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c index 27e0d40daca8..06a54c8bd46f 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.c @@ -457,3 +457,21 @@ void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, sg_free_table(sgt); kfree(sgt); } + +u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man) +{ + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + + return mgr->visible_size; +} + +void xe_ttm_vram_get_used(struct ttm_resource_manager *man, + u64 *used, u64 *used_visible) +{ + struct xe_ttm_vram_mgr *mgr = to_xe_ttm_vram_mgr(man); + + mutex_lock(&mgr->lock); + *used = mgr->mm.size - mgr->mm.avail; + *used_visible = mgr->visible_size - mgr->visible_avail; + mutex_unlock(&mgr->lock); +} diff --git a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h index 6e1d6033d739..d184e19a9230 100644 --- a/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h +++ b/drivers/gpu/drm/xe/xe_ttm_vram_mgr.h @@ -25,6 +25,10 @@ int xe_ttm_vram_mgr_alloc_sgt(struct xe_device *xe, void xe_ttm_vram_mgr_free_sgt(struct device *dev, enum dma_data_direction dir, struct sg_table *sgt); +u64 xe_ttm_vram_get_cpu_visible_size(struct ttm_resource_manager *man); +void xe_ttm_vram_get_used(struct ttm_resource_manager *man, + u64 *used, u64 *used_visible); + static inline struct xe_ttm_vram_mgr_resource * to_xe_ttm_vram_mgr_resource(struct ttm_resource *res) { diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 347351a8f618..7f29c58f87a3 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -180,8 +180,37 @@ struct drm_xe_query_mem_region { * zero. */ __u64 used; + /** + * @cpu_visible_size: How much of this region can be CPU + * accessed, in bytes. + * + * This will always be <= @total_size, and the remainder (if + * any) will not be CPU accessible. If the CPU accessible part + * is smaller than @total_size then this is referred to as a + * small BAR system. + * + * On systems without small BAR (full BAR), the probed_size will + * always equal the @total_size, since all of it will be CPU + * accessible. + * + * Note this is only tracked for XE_MEM_REGION_CLASS_VRAM + * regions (for other types the value here will always equal + * zero). + */ + __u64 cpu_visible_size; + /** + * @cpu_visible_used: Estimate of CPU visible memory used, in + * bytes. + * + * Requires CAP_PERFMON or CAP_SYS_ADMIN to get reliable + * accounting. Without this the value here will always equal + * zero. Note this is only currently tracked for + * XE_MEM_REGION_CLASS_VRAM regions (for other types the value + * here will always be zero). + */ + __u64 cpu_visible_used; /** @reserved: MBZ */ - __u64 reserved[8]; + __u64 reserved[6]; }; /** @@ -383,6 +412,22 @@ struct drm_xe_gem_create { #define XE_GEM_CREATE_FLAG_DEFER_BACKING (0x1 << 24) #define XE_GEM_CREATE_FLAG_SCANOUT (0x1 << 25) +/* + * When using VRAM as a possible placement, ensure that the corresponding VRAM + * allocation will always use the CPU accessible part of VRAM. This is important + * for small-bar systems (on full-bar systems this gets turned into a noop). + * + * Note: System memory can be used as an extra placement if the kernel should + * spill the allocation to system memory, if space can't be made available in + * the CPU accessible part of VRAM (giving the same behaviour as the i915 + * interface, see I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS). + * + * Note: For clear-color CCS surfaces the kernel needs to read the clear-color + * value stored in the buffer, and on discrete platforms we need to use VRAM for + * display surfaces, therefore the kernel requires setting this flag for such + * objects, otherwise an error is thrown on small-bar systems. + */ +#define XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM (0x1 << 26) /** * @flags: Flags, currently a mask of memory instances of where BO can * be placed -- cgit v1.2.3 From c856cc138bf39aa38f1b97def8927c71b2a057c2 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Fri, 21 Jul 2023 15:44:50 -0400 Subject: drm/xe/uapi: Remove XE_QUERY_CONFIG_FLAGS_USE_GUC MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This config is the only real one. If execlist remains in the code it will forever be experimental and we shouldn't maintain an uapi like that for that experimental piece of code that should never be used by real users. Signed-off-by: Rodrigo Vivi Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_query.c | 3 --- include/uapi/drm/xe_drm.h | 1 - 2 files changed, 4 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 3997c644f8fc..6ba7baf7c777 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -195,9 +195,6 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) if (xe_device_get_root_tile(xe)->mem.vram.usable_size) config->info[XE_QUERY_CONFIG_FLAGS] = XE_QUERY_CONFIG_FLAGS_HAS_VRAM; - if (xe->info.enable_guc) - config->info[XE_QUERY_CONFIG_FLAGS] |= - XE_QUERY_CONFIG_FLAGS_USE_GUC; config->info[XE_QUERY_CONFIG_MIN_ALIGNEMENT] = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; config->info[XE_QUERY_CONFIG_VA_BITS] = 12 + diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 7f29c58f87a3..259de80376b4 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -246,7 +246,6 @@ struct drm_xe_query_config { #define XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 #define XE_QUERY_CONFIG_FLAGS 1 #define XE_QUERY_CONFIG_FLAGS_HAS_VRAM (0x1 << 0) - #define XE_QUERY_CONFIG_FLAGS_USE_GUC (0x1 << 1) #define XE_QUERY_CONFIG_MIN_ALIGNEMENT 2 #define XE_QUERY_CONFIG_VA_BITS 3 #define XE_QUERY_CONFIG_GT_COUNT 4 -- cgit v1.2.3 From 4f027e304a6c7ae77150965d10b8a1edee0398a2 Mon Sep 17 00:00:00 2001 From: Himal Prasad Ghimiray Date: Thu, 27 Jul 2023 04:56:49 +0530 Subject: drm/xe: Notify Userspace when gt reset fails Send uevent in case of gt reset failure. This intimation can be used by userspace monitoring tool to do the device level reset/reboot when GT reset fails. udevadm can be used to monitor the uevents. v2: - Support only gt failure notification (Rodrigo) v3 - Rectify the comments in header file. v4 - Use pci kobj instead of drm kobj for notification.(Rodrigo) - Cleanup (Badal) v5 - Add tile id and gt id as additional info provided by uevent. - Provide code documentation for the uevent. (Rodrigo) Cc: Aravind Iddamsetty Cc: Tejas Upadhyay Cc: Rodrigo Vivi Reviewed-by: Badal Nilawar Signed-off-by: Himal Prasad Ghimiray Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_gt.c | 19 +++++++++++++++++++ include/uapi/drm/xe_drm.h | 10 ++++++++++ 2 files changed, 29 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index bb7794cf2c1a..82b987404070 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -8,6 +8,7 @@ #include #include +#include #include "regs/xe_gt_regs.h" #include "xe_bb.h" @@ -499,6 +500,20 @@ static int do_gt_restart(struct xe_gt *gt) return 0; } +static void xe_uevent_gt_reset_failure(struct pci_dev *pdev, u8 tile_id, u8 gt_id) +{ + char *reset_event[4]; + + reset_event[0] = XE_RESET_FAILED_UEVENT "=NEEDS_RESET"; + reset_event[1] = kasprintf(GFP_KERNEL, "TILE_ID=%d", tile_id); + reset_event[2] = kasprintf(GFP_KERNEL, "GT_ID=%d", gt_id); + reset_event[3] = NULL; + kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, reset_event); + + kfree(reset_event[1]); + kfree(reset_event[2]); +} + static int gt_reset(struct xe_gt *gt) { int err; @@ -549,6 +564,10 @@ err_msg: xe_device_mem_access_put(gt_to_xe(gt)); xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err)); + /* Notify userspace about gt reset failure */ + xe_uevent_gt_reset_failure(to_pci_dev(gt_to_xe(gt)->drm.dev), + gt_to_tile(gt)->id, gt->info.id); + return err; } diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 259de80376b4..3d09e9e9267b 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -16,6 +16,16 @@ extern "C" { * subject to backwards-compatibility constraints. */ +/** + * DOC: uevent generated by xe on it's pci node. + * + * XE_RESET_FAILED_UEVENT - Event is generated when attempt to reset gt + * fails. The value supplied with the event is always "NEEDS_RESET". + * Additional information supplied is tile id and gt id of the gt unit for + * which reset has failed. + */ +#define XE_RESET_FAILED_UEVENT "DEVICE_STATUS" + /** * struct xe_user_extension - Base class for defining a chain of extensions * -- cgit v1.2.3 From 9b9529ce379a08e68d65231497dd6bad94281902 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Mon, 31 Jul 2023 17:30:02 +0200 Subject: drm/xe: Rename engine to exec_queue Engine was inappropriately used to refer to execution queues and it also created some confusion with hardware engines. Where it applies the exec_queue variable name is changed to q and comments are also updated. Link: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/162 Signed-off-by: Francois Dugast Reviewed-by: Rodrigo Vivi Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/tests/xe_migrate.c | 18 +- drivers/gpu/drm/xe/xe_bb.c | 26 +- drivers/gpu/drm/xe/xe_bb.h | 8 +- drivers/gpu/drm/xe/xe_devcoredump.c | 38 +- drivers/gpu/drm/xe/xe_devcoredump.h | 6 +- drivers/gpu/drm/xe/xe_devcoredump_types.h | 2 +- drivers/gpu/drm/xe/xe_device.c | 60 +- drivers/gpu/drm/xe/xe_device.h | 8 +- drivers/gpu/drm/xe/xe_device_types.h | 4 +- drivers/gpu/drm/xe/xe_engine_types.h | 209 ----- drivers/gpu/drm/xe/xe_exec.c | 60 +- drivers/gpu/drm/xe/xe_exec_queue.c | 526 ++++++------- drivers/gpu/drm/xe/xe_exec_queue.h | 64 +- drivers/gpu/drm/xe/xe_exec_queue_types.h | 209 +++++ drivers/gpu/drm/xe/xe_execlist.c | 142 ++-- drivers/gpu/drm/xe/xe_execlist_types.h | 14 +- drivers/gpu/drm/xe/xe_gt.c | 70 +- drivers/gpu/drm/xe/xe_gt_types.h | 6 +- drivers/gpu/drm/xe/xe_guc_ads.c | 2 +- drivers/gpu/drm/xe/xe_guc_ct.c | 10 +- drivers/gpu/drm/xe/xe_guc_engine_types.h | 54 -- drivers/gpu/drm/xe/xe_guc_exec_queue_types.h | 54 ++ drivers/gpu/drm/xe/xe_guc_fwif.h | 6 +- drivers/gpu/drm/xe/xe_guc_submit.c | 1088 +++++++++++++------------- drivers/gpu/drm/xe/xe_guc_submit.h | 20 +- drivers/gpu/drm/xe/xe_guc_submit_types.h | 20 +- drivers/gpu/drm/xe/xe_guc_types.h | 4 +- drivers/gpu/drm/xe/xe_lrc.c | 10 +- drivers/gpu/drm/xe/xe_lrc.h | 4 +- drivers/gpu/drm/xe/xe_migrate.c | 64 +- drivers/gpu/drm/xe/xe_migrate.h | 6 +- drivers/gpu/drm/xe/xe_mocs.h | 2 +- drivers/gpu/drm/xe/xe_preempt_fence.c | 30 +- drivers/gpu/drm/xe/xe_preempt_fence.h | 4 +- drivers/gpu/drm/xe/xe_preempt_fence_types.h | 7 +- drivers/gpu/drm/xe/xe_pt.c | 18 +- drivers/gpu/drm/xe/xe_pt.h | 6 +- drivers/gpu/drm/xe/xe_query.c | 2 +- drivers/gpu/drm/xe/xe_ring_ops.c | 38 +- drivers/gpu/drm/xe/xe_sched_job.c | 74 +- drivers/gpu/drm/xe/xe_sched_job.h | 4 +- drivers/gpu/drm/xe/xe_sched_job_types.h | 6 +- drivers/gpu/drm/xe/xe_trace.h | 140 ++-- drivers/gpu/drm/xe/xe_vm.c | 192 ++--- drivers/gpu/drm/xe/xe_vm.h | 4 +- drivers/gpu/drm/xe/xe_vm_types.h | 16 +- include/uapi/drm/xe_drm.h | 86 +- 47 files changed, 1720 insertions(+), 1721 deletions(-) delete mode 100644 drivers/gpu/drm/xe/xe_engine_types.h create mode 100644 drivers/gpu/drm/xe/xe_exec_queue_types.h delete mode 100644 drivers/gpu/drm/xe/xe_guc_engine_types.h create mode 100644 drivers/gpu/drm/xe/xe_guc_exec_queue_types.h (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/tests/xe_migrate.c b/drivers/gpu/drm/xe/tests/xe_migrate.c index 9e9b228fe315..5c8d5e78d9bc 100644 --- a/drivers/gpu/drm/xe/tests/xe_migrate.c +++ b/drivers/gpu/drm/xe/tests/xe_migrate.c @@ -38,7 +38,7 @@ static int run_sanity_job(struct xe_migrate *m, struct xe_device *xe, struct kunit *test) { u64 batch_base = xe_migrate_batch_base(m, xe->info.supports_usm); - struct xe_sched_job *job = xe_bb_create_migration_job(m->eng, bb, + struct xe_sched_job *job = xe_bb_create_migration_job(m->q, bb, batch_base, second_idx); struct dma_fence *fence; @@ -215,7 +215,7 @@ static void test_pt_update(struct xe_migrate *m, struct xe_bo *pt, xe_map_memset(xe, &pt->vmap, 0, (u8)expected, pt->size); then = ktime_get(); - fence = xe_migrate_update_pgtables(m, NULL, NULL, m->eng, &update, 1, + fence = xe_migrate_update_pgtables(m, NULL, NULL, m->q, &update, 1, NULL, 0, &pt_update); now = ktime_get(); if (sanity_fence_failed(xe, fence, "Migration pagetable update", test)) @@ -257,7 +257,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) return; } - big = xe_bo_create_pin_map(xe, tile, m->eng->vm, SZ_4M, + big = xe_bo_create_pin_map(xe, tile, m->q->vm, SZ_4M, ttm_bo_type_kernel, XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_PINNED_BIT); @@ -266,7 +266,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) goto vunmap; } - pt = xe_bo_create_pin_map(xe, tile, m->eng->vm, XE_PAGE_SIZE, + pt = xe_bo_create_pin_map(xe, tile, m->q->vm, XE_PAGE_SIZE, ttm_bo_type_kernel, XE_BO_CREATE_VRAM_IF_DGFX(tile) | XE_BO_CREATE_PINNED_BIT); @@ -276,7 +276,7 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) goto free_big; } - tiny = xe_bo_create_pin_map(xe, tile, m->eng->vm, + tiny = xe_bo_create_pin_map(xe, tile, m->q->vm, 2 * SZ_4K, ttm_bo_type_kernel, XE_BO_CREATE_VRAM_IF_DGFX(tile) | @@ -295,14 +295,14 @@ static void xe_migrate_sanity_test(struct xe_migrate *m, struct kunit *test) } kunit_info(test, "Starting tests, top level PT addr: %lx, special pagetable base addr: %lx\n", - (unsigned long)xe_bo_main_addr(m->eng->vm->pt_root[id]->bo, XE_PAGE_SIZE), + (unsigned long)xe_bo_main_addr(m->q->vm->pt_root[id]->bo, XE_PAGE_SIZE), (unsigned long)xe_bo_main_addr(m->pt_bo, XE_PAGE_SIZE)); /* First part of the test, are we updating our pagetable bo with a new entry? */ xe_map_wr(xe, &bo->vmap, XE_PAGE_SIZE * (NUM_KERNEL_PDE - 1), u64, 0xdeaddeadbeefbeef); expected = xe_pte_encode(pt, 0, XE_CACHE_WB, 0); - if (m->eng->vm->flags & XE_VM_FLAG_64K) + if (m->q->vm->flags & XE_VM_FLAG_64K) expected |= XE_PTE_PS64; if (xe_bo_is_vram(pt)) xe_res_first(pt->ttm.resource, 0, pt->size, &src_it); @@ -399,11 +399,11 @@ static int migrate_test_run_device(struct xe_device *xe) struct ww_acquire_ctx ww; kunit_info(test, "Testing tile id %d.\n", id); - xe_vm_lock(m->eng->vm, &ww, 0, true); + xe_vm_lock(m->q->vm, &ww, 0, true); xe_device_mem_access_get(xe); xe_migrate_sanity_test(m, test); xe_device_mem_access_put(xe); - xe_vm_unlock(m->eng->vm, &ww); + xe_vm_unlock(m->q->vm, &ww); } return 0; diff --git a/drivers/gpu/drm/xe/xe_bb.c b/drivers/gpu/drm/xe/xe_bb.c index b15a7cb7db4c..38f4ce83a207 100644 --- a/drivers/gpu/drm/xe/xe_bb.c +++ b/drivers/gpu/drm/xe/xe_bb.c @@ -7,7 +7,7 @@ #include "regs/xe_gpu_commands.h" #include "xe_device.h" -#include "xe_engine_types.h" +#include "xe_exec_queue_types.h" #include "xe_gt.h" #include "xe_hw_fence.h" #include "xe_sa.h" @@ -60,30 +60,30 @@ err: } static struct xe_sched_job * -__xe_bb_create_job(struct xe_engine *kernel_eng, struct xe_bb *bb, u64 *addr) +__xe_bb_create_job(struct xe_exec_queue *q, struct xe_bb *bb, u64 *addr) { u32 size = drm_suballoc_size(bb->bo); bb->cs[bb->len++] = MI_BATCH_BUFFER_END; - WARN_ON(bb->len * 4 + bb_prefetch(kernel_eng->gt) > size); + WARN_ON(bb->len * 4 + bb_prefetch(q->gt) > size); xe_sa_bo_flush_write(bb->bo); - return xe_sched_job_create(kernel_eng, addr); + return xe_sched_job_create(q, addr); } -struct xe_sched_job *xe_bb_create_wa_job(struct xe_engine *wa_eng, +struct xe_sched_job *xe_bb_create_wa_job(struct xe_exec_queue *q, struct xe_bb *bb, u64 batch_base_ofs) { u64 addr = batch_base_ofs + drm_suballoc_soffset(bb->bo); - XE_WARN_ON(!(wa_eng->vm->flags & XE_VM_FLAG_MIGRATION)); + XE_WARN_ON(!(q->vm->flags & XE_VM_FLAG_MIGRATION)); - return __xe_bb_create_job(wa_eng, bb, &addr); + return __xe_bb_create_job(q, bb, &addr); } -struct xe_sched_job *xe_bb_create_migration_job(struct xe_engine *kernel_eng, +struct xe_sched_job *xe_bb_create_migration_job(struct xe_exec_queue *q, struct xe_bb *bb, u64 batch_base_ofs, u32 second_idx) @@ -95,18 +95,18 @@ struct xe_sched_job *xe_bb_create_migration_job(struct xe_engine *kernel_eng, }; XE_WARN_ON(second_idx > bb->len); - XE_WARN_ON(!(kernel_eng->vm->flags & XE_VM_FLAG_MIGRATION)); + XE_WARN_ON(!(q->vm->flags & XE_VM_FLAG_MIGRATION)); - return __xe_bb_create_job(kernel_eng, bb, addr); + return __xe_bb_create_job(q, bb, addr); } -struct xe_sched_job *xe_bb_create_job(struct xe_engine *kernel_eng, +struct xe_sched_job *xe_bb_create_job(struct xe_exec_queue *q, struct xe_bb *bb) { u64 addr = xe_sa_bo_gpu_addr(bb->bo); - XE_WARN_ON(kernel_eng->vm && kernel_eng->vm->flags & XE_VM_FLAG_MIGRATION); - return __xe_bb_create_job(kernel_eng, bb, &addr); + XE_WARN_ON(q->vm && q->vm->flags & XE_VM_FLAG_MIGRATION); + return __xe_bb_create_job(q, bb, &addr); } void xe_bb_free(struct xe_bb *bb, struct dma_fence *fence) diff --git a/drivers/gpu/drm/xe/xe_bb.h b/drivers/gpu/drm/xe/xe_bb.h index 0cc9260c9634..c5ae0770bab5 100644 --- a/drivers/gpu/drm/xe/xe_bb.h +++ b/drivers/gpu/drm/xe/xe_bb.h @@ -11,16 +11,16 @@ struct dma_fence; struct xe_gt; -struct xe_engine; +struct xe_exec_queue; struct xe_sched_job; struct xe_bb *xe_bb_new(struct xe_gt *gt, u32 size, bool usm); -struct xe_sched_job *xe_bb_create_job(struct xe_engine *kernel_eng, +struct xe_sched_job *xe_bb_create_job(struct xe_exec_queue *q, struct xe_bb *bb); -struct xe_sched_job *xe_bb_create_migration_job(struct xe_engine *kernel_eng, +struct xe_sched_job *xe_bb_create_migration_job(struct xe_exec_queue *q, struct xe_bb *bb, u64 batch_ofs, u32 second_idx); -struct xe_sched_job *xe_bb_create_wa_job(struct xe_engine *wa_eng, +struct xe_sched_job *xe_bb_create_wa_job(struct xe_exec_queue *q, struct xe_bb *bb, u64 batch_ofs); void xe_bb_free(struct xe_bb *bb, struct dma_fence *fence); diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c index 61ff97ea7659..68abc0b195be 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.c +++ b/drivers/gpu/drm/xe/xe_devcoredump.c @@ -53,9 +53,9 @@ static struct xe_device *coredump_to_xe(const struct xe_devcoredump *coredump) return container_of(coredump, struct xe_device, devcoredump); } -static struct xe_guc *engine_to_guc(struct xe_engine *e) +static struct xe_guc *exec_queue_to_guc(struct xe_exec_queue *q) { - return &e->gt->uc.guc; + return &q->gt->uc.guc; } static ssize_t xe_devcoredump_read(char *buffer, loff_t offset, @@ -91,7 +91,7 @@ static ssize_t xe_devcoredump_read(char *buffer, loff_t offset, drm_printf(&p, "\n**** GuC CT ****\n"); xe_guc_ct_snapshot_print(coredump->snapshot.ct, &p); - xe_guc_engine_snapshot_print(coredump->snapshot.ge, &p); + xe_guc_exec_queue_snapshot_print(coredump->snapshot.ge, &p); drm_printf(&p, "\n**** HW Engines ****\n"); for (i = 0; i < XE_NUM_HW_ENGINES; i++) @@ -112,7 +112,7 @@ static void xe_devcoredump_free(void *data) return; xe_guc_ct_snapshot_free(coredump->snapshot.ct); - xe_guc_engine_snapshot_free(coredump->snapshot.ge); + xe_guc_exec_queue_snapshot_free(coredump->snapshot.ge); for (i = 0; i < XE_NUM_HW_ENGINES; i++) if (coredump->snapshot.hwe[i]) xe_hw_engine_snapshot_free(coredump->snapshot.hwe[i]); @@ -123,14 +123,14 @@ static void xe_devcoredump_free(void *data) } static void devcoredump_snapshot(struct xe_devcoredump *coredump, - struct xe_engine *e) + struct xe_exec_queue *q) { struct xe_devcoredump_snapshot *ss = &coredump->snapshot; - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); struct xe_hw_engine *hwe; enum xe_hw_engine_id id; - u32 adj_logical_mask = e->logical_mask; - u32 width_mask = (0x1 << e->width) - 1; + u32 adj_logical_mask = q->logical_mask; + u32 width_mask = (0x1 << q->width) - 1; int i; bool cookie; @@ -138,22 +138,22 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump, ss->boot_time = ktime_get_boottime(); cookie = dma_fence_begin_signalling(); - for (i = 0; e->width > 1 && i < XE_HW_ENGINE_MAX_INSTANCE;) { + for (i = 0; q->width > 1 && i < XE_HW_ENGINE_MAX_INSTANCE;) { if (adj_logical_mask & BIT(i)) { adj_logical_mask |= width_mask << i; - i += e->width; + i += q->width; } else { ++i; } } - xe_force_wake_get(gt_to_fw(e->gt), XE_FORCEWAKE_ALL); + xe_force_wake_get(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); coredump->snapshot.ct = xe_guc_ct_snapshot_capture(&guc->ct, true); - coredump->snapshot.ge = xe_guc_engine_snapshot_capture(e); + coredump->snapshot.ge = xe_guc_exec_queue_snapshot_capture(q); - for_each_hw_engine(hwe, e->gt, id) { - if (hwe->class != e->hwe->class || + for_each_hw_engine(hwe, q->gt, id) { + if (hwe->class != q->hwe->class || !(BIT(hwe->logical_instance) & adj_logical_mask)) { coredump->snapshot.hwe[id] = NULL; continue; @@ -161,21 +161,21 @@ static void devcoredump_snapshot(struct xe_devcoredump *coredump, coredump->snapshot.hwe[id] = xe_hw_engine_snapshot_capture(hwe); } - xe_force_wake_put(gt_to_fw(e->gt), XE_FORCEWAKE_ALL); + xe_force_wake_put(gt_to_fw(q->gt), XE_FORCEWAKE_ALL); dma_fence_end_signalling(cookie); } /** * xe_devcoredump - Take the required snapshots and initialize coredump device. - * @e: The faulty xe_engine, where the issue was detected. + * @q: The faulty xe_exec_queue, where the issue was detected. * * This function should be called at the crash time within the serialized * gt_reset. It is skipped if we still have the core dump device available * with the information of the 'first' snapshot. */ -void xe_devcoredump(struct xe_engine *e) +void xe_devcoredump(struct xe_exec_queue *q) { - struct xe_device *xe = gt_to_xe(e->gt); + struct xe_device *xe = gt_to_xe(q->gt); struct xe_devcoredump *coredump = &xe->devcoredump; if (coredump->captured) { @@ -184,7 +184,7 @@ void xe_devcoredump(struct xe_engine *e) } coredump->captured = true; - devcoredump_snapshot(coredump, e); + devcoredump_snapshot(coredump, q); drm_info(&xe->drm, "Xe device coredump has been created\n"); drm_info(&xe->drm, "Check your /sys/class/drm/card%d/device/devcoredump/data\n", diff --git a/drivers/gpu/drm/xe/xe_devcoredump.h b/drivers/gpu/drm/xe/xe_devcoredump.h index 854882129227..6ac218a5c194 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.h +++ b/drivers/gpu/drm/xe/xe_devcoredump.h @@ -7,12 +7,12 @@ #define _XE_DEVCOREDUMP_H_ struct xe_device; -struct xe_engine; +struct xe_exec_queue; #ifdef CONFIG_DEV_COREDUMP -void xe_devcoredump(struct xe_engine *e); +void xe_devcoredump(struct xe_exec_queue *q); #else -static inline void xe_devcoredump(struct xe_engine *e) +static inline void xe_devcoredump(struct xe_exec_queue *q) { } #endif diff --git a/drivers/gpu/drm/xe/xe_devcoredump_types.h b/drivers/gpu/drm/xe/xe_devcoredump_types.h index c0d711eb6ab3..7fdad9c3d3dd 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump_types.h +++ b/drivers/gpu/drm/xe/xe_devcoredump_types.h @@ -30,7 +30,7 @@ struct xe_devcoredump_snapshot { /** @ct: GuC CT snapshot */ struct xe_guc_ct_snapshot *ct; /** @ge: Guc Engine snapshot */ - struct xe_guc_submit_engine_snapshot *ge; + struct xe_guc_submit_exec_queue_snapshot *ge; /** @hwe: HW Engine snapshot array */ struct xe_hw_engine_snapshot *hwe[XE_NUM_HW_ENGINES]; }; diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index a8ab86379ed6..df1953759c67 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -53,33 +53,33 @@ static int xe_file_open(struct drm_device *dev, struct drm_file *file) mutex_init(&xef->vm.lock); xa_init_flags(&xef->vm.xa, XA_FLAGS_ALLOC1); - mutex_init(&xef->engine.lock); - xa_init_flags(&xef->engine.xa, XA_FLAGS_ALLOC1); + mutex_init(&xef->exec_queue.lock); + xa_init_flags(&xef->exec_queue.xa, XA_FLAGS_ALLOC1); file->driver_priv = xef; return 0; } -static void device_kill_persistent_engines(struct xe_device *xe, - struct xe_file *xef); +static void device_kill_persistent_exec_queues(struct xe_device *xe, + struct xe_file *xef); static void xe_file_close(struct drm_device *dev, struct drm_file *file) { struct xe_device *xe = to_xe_device(dev); struct xe_file *xef = file->driver_priv; struct xe_vm *vm; - struct xe_engine *e; + struct xe_exec_queue *q; unsigned long idx; - mutex_lock(&xef->engine.lock); - xa_for_each(&xef->engine.xa, idx, e) { - xe_engine_kill(e); - xe_engine_put(e); + mutex_lock(&xef->exec_queue.lock); + xa_for_each(&xef->exec_queue.xa, idx, q) { + xe_exec_queue_kill(q); + xe_exec_queue_put(q); } - mutex_unlock(&xef->engine.lock); - xa_destroy(&xef->engine.xa); - mutex_destroy(&xef->engine.lock); - device_kill_persistent_engines(xe, xef); + mutex_unlock(&xef->exec_queue.lock); + xa_destroy(&xef->exec_queue.xa); + mutex_destroy(&xef->exec_queue.lock); + device_kill_persistent_exec_queues(xe, xef); mutex_lock(&xef->vm.lock); xa_for_each(&xef->vm.xa, idx, vm) @@ -99,15 +99,15 @@ static const struct drm_ioctl_desc xe_ioctls[] = { DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(XE_ENGINE_CREATE, xe_engine_create_ioctl, + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(XE_ENGINE_GET_PROPERTY, xe_engine_get_property_ioctl, + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(XE_ENGINE_DESTROY, xe_engine_destroy_ioctl, + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_MMIO, xe_mmio_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(XE_ENGINE_SET_PROPERTY, xe_engine_set_property_ioctl, + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, DRM_RENDER_ALLOW), @@ -324,33 +324,33 @@ void xe_device_shutdown(struct xe_device *xe) { } -void xe_device_add_persistent_engines(struct xe_device *xe, struct xe_engine *e) +void xe_device_add_persistent_exec_queues(struct xe_device *xe, struct xe_exec_queue *q) { mutex_lock(&xe->persistent_engines.lock); - list_add_tail(&e->persistent.link, &xe->persistent_engines.list); + list_add_tail(&q->persistent.link, &xe->persistent_engines.list); mutex_unlock(&xe->persistent_engines.lock); } -void xe_device_remove_persistent_engines(struct xe_device *xe, - struct xe_engine *e) +void xe_device_remove_persistent_exec_queues(struct xe_device *xe, + struct xe_exec_queue *q) { mutex_lock(&xe->persistent_engines.lock); - if (!list_empty(&e->persistent.link)) - list_del(&e->persistent.link); + if (!list_empty(&q->persistent.link)) + list_del(&q->persistent.link); mutex_unlock(&xe->persistent_engines.lock); } -static void device_kill_persistent_engines(struct xe_device *xe, - struct xe_file *xef) +static void device_kill_persistent_exec_queues(struct xe_device *xe, + struct xe_file *xef) { - struct xe_engine *e, *next; + struct xe_exec_queue *q, *next; mutex_lock(&xe->persistent_engines.lock); - list_for_each_entry_safe(e, next, &xe->persistent_engines.list, + list_for_each_entry_safe(q, next, &xe->persistent_engines.list, persistent.link) - if (e->persistent.xef == xef) { - xe_engine_kill(e); - list_del_init(&e->persistent.link); + if (q->persistent.xef == xef) { + xe_exec_queue_kill(q); + list_del_init(&q->persistent.link); } mutex_unlock(&xe->persistent_engines.lock); } diff --git a/drivers/gpu/drm/xe/xe_device.h b/drivers/gpu/drm/xe/xe_device.h index 61a5cf1f7300..71582094834c 100644 --- a/drivers/gpu/drm/xe/xe_device.h +++ b/drivers/gpu/drm/xe/xe_device.h @@ -6,7 +6,7 @@ #ifndef _XE_DEVICE_H_ #define _XE_DEVICE_H_ -struct xe_engine; +struct xe_exec_queue; struct xe_file; #include @@ -41,9 +41,9 @@ int xe_device_probe(struct xe_device *xe); void xe_device_remove(struct xe_device *xe); void xe_device_shutdown(struct xe_device *xe); -void xe_device_add_persistent_engines(struct xe_device *xe, struct xe_engine *e); -void xe_device_remove_persistent_engines(struct xe_device *xe, - struct xe_engine *e); +void xe_device_add_persistent_exec_queues(struct xe_device *xe, struct xe_exec_queue *q); +void xe_device_remove_persistent_exec_queues(struct xe_device *xe, + struct xe_exec_queue *q); void xe_device_wmb(struct xe_device *xe); diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index c521ffaf3871..128e0a953692 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -377,13 +377,13 @@ struct xe_file { struct mutex lock; } vm; - /** @engine: Submission engine state for file */ + /** @exec_queue: Submission exec queue state for file */ struct { /** @xe: xarray to store engines */ struct xarray xa; /** @lock: protects file engine state */ struct mutex lock; - } engine; + } exec_queue; }; #endif diff --git a/drivers/gpu/drm/xe/xe_engine_types.h b/drivers/gpu/drm/xe/xe_engine_types.h deleted file mode 100644 index f1d531735f6d..000000000000 --- a/drivers/gpu/drm/xe/xe_engine_types.h +++ /dev/null @@ -1,209 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2022 Intel Corporation - */ - -#ifndef _XE_ENGINE_TYPES_H_ -#define _XE_ENGINE_TYPES_H_ - -#include - -#include - -#include "xe_gpu_scheduler_types.h" -#include "xe_hw_engine_types.h" -#include "xe_hw_fence_types.h" -#include "xe_lrc_types.h" - -struct xe_execlist_engine; -struct xe_gt; -struct xe_guc_engine; -struct xe_hw_engine; -struct xe_vm; - -enum xe_engine_priority { - XE_ENGINE_PRIORITY_UNSET = -2, /* For execlist usage only */ - XE_ENGINE_PRIORITY_LOW = 0, - XE_ENGINE_PRIORITY_NORMAL, - XE_ENGINE_PRIORITY_HIGH, - XE_ENGINE_PRIORITY_KERNEL, - - XE_ENGINE_PRIORITY_COUNT -}; - -/** - * struct xe_engine - Submission engine - * - * Contains all state necessary for submissions. Can either be a user object or - * a kernel object. - */ -struct xe_engine { - /** @gt: graphics tile this engine can submit to */ - struct xe_gt *gt; - /** - * @hwe: A hardware of the same class. May (physical engine) or may not - * (virtual engine) be where jobs actual engine up running. Should never - * really be used for submissions. - */ - struct xe_hw_engine *hwe; - /** @refcount: ref count of this engine */ - struct kref refcount; - /** @vm: VM (address space) for this engine */ - struct xe_vm *vm; - /** @class: class of this engine */ - enum xe_engine_class class; - /** @priority: priority of this exec queue */ - enum xe_engine_priority priority; - /** - * @logical_mask: logical mask of where job submitted to engine can run - */ - u32 logical_mask; - /** @name: name of this engine */ - char name[MAX_FENCE_NAME_LEN]; - /** @width: width (number BB submitted per exec) of this engine */ - u16 width; - /** @fence_irq: fence IRQ used to signal job completion */ - struct xe_hw_fence_irq *fence_irq; - -#define ENGINE_FLAG_BANNED BIT(0) -#define ENGINE_FLAG_KERNEL BIT(1) -#define ENGINE_FLAG_PERSISTENT BIT(2) -#define ENGINE_FLAG_COMPUTE_MODE BIT(3) -/* Caller needs to hold rpm ref when creating engine with ENGINE_FLAG_VM */ -#define ENGINE_FLAG_VM BIT(4) -#define ENGINE_FLAG_BIND_ENGINE_CHILD BIT(5) -#define ENGINE_FLAG_WA BIT(6) - - /** - * @flags: flags for this engine, should statically setup aside from ban - * bit - */ - unsigned long flags; - - union { - /** @multi_gt_list: list head for VM bind engines if multi-GT */ - struct list_head multi_gt_list; - /** @multi_gt_link: link for VM bind engines if multi-GT */ - struct list_head multi_gt_link; - }; - - union { - /** @execlist: execlist backend specific state for engine */ - struct xe_execlist_engine *execlist; - /** @guc: GuC backend specific state for engine */ - struct xe_guc_engine *guc; - }; - - /** - * @persistent: persistent engine state - */ - struct { - /** @xef: file which this engine belongs to */ - struct xe_file *xef; - /** @link: link in list of persistent engines */ - struct list_head link; - } persistent; - - union { - /** - * @parallel: parallel submission state - */ - struct { - /** @composite_fence_ctx: context composite fence */ - u64 composite_fence_ctx; - /** @composite_fence_seqno: seqno for composite fence */ - u32 composite_fence_seqno; - } parallel; - /** - * @bind: bind submission state - */ - struct { - /** @fence_ctx: context bind fence */ - u64 fence_ctx; - /** @fence_seqno: seqno for bind fence */ - u32 fence_seqno; - } bind; - }; - - /** @sched_props: scheduling properties */ - struct { - /** @timeslice_us: timeslice period in micro-seconds */ - u32 timeslice_us; - /** @preempt_timeout_us: preemption timeout in micro-seconds */ - u32 preempt_timeout_us; - } sched_props; - - /** @compute: compute engine state */ - struct { - /** @pfence: preemption fence */ - struct dma_fence *pfence; - /** @context: preemption fence context */ - u64 context; - /** @seqno: preemption fence seqno */ - u32 seqno; - /** @link: link into VM's list of engines */ - struct list_head link; - /** @lock: preemption fences lock */ - spinlock_t lock; - } compute; - - /** @usm: unified shared memory state */ - struct { - /** @acc_trigger: access counter trigger */ - u32 acc_trigger; - /** @acc_notify: access counter notify */ - u32 acc_notify; - /** @acc_granularity: access counter granularity */ - u32 acc_granularity; - } usm; - - /** @ops: submission backend engine operations */ - const struct xe_engine_ops *ops; - - /** @ring_ops: ring operations for this engine */ - const struct xe_ring_ops *ring_ops; - /** @entity: DRM sched entity for this engine (1 to 1 relationship) */ - struct drm_sched_entity *entity; - /** @lrc: logical ring context for this engine */ - struct xe_lrc lrc[]; -}; - -/** - * struct xe_engine_ops - Submission backend engine operations - */ -struct xe_engine_ops { - /** @init: Initialize engine for submission backend */ - int (*init)(struct xe_engine *e); - /** @kill: Kill inflight submissions for backend */ - void (*kill)(struct xe_engine *e); - /** @fini: Fini engine for submission backend */ - void (*fini)(struct xe_engine *e); - /** @set_priority: Set priority for engine */ - int (*set_priority)(struct xe_engine *e, - enum xe_engine_priority priority); - /** @set_timeslice: Set timeslice for engine */ - int (*set_timeslice)(struct xe_engine *e, u32 timeslice_us); - /** @set_preempt_timeout: Set preemption timeout for engine */ - int (*set_preempt_timeout)(struct xe_engine *e, u32 preempt_timeout_us); - /** @set_job_timeout: Set job timeout for engine */ - int (*set_job_timeout)(struct xe_engine *e, u32 job_timeout_ms); - /** - * @suspend: Suspend engine from executing, allowed to be called - * multiple times in a row before resume with the caveat that - * suspend_wait returns before calling suspend again. - */ - int (*suspend)(struct xe_engine *e); - /** - * @suspend_wait: Wait for an engine to suspend executing, should be - * call after suspend. - */ - void (*suspend_wait)(struct xe_engine *e); - /** - * @resume: Resume engine execution, engine must be in a suspended - * state and dma fence returned from most recent suspend call must be - * signalled when this function is called. - */ - void (*resume)(struct xe_engine *e); -}; - -#endif diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index a043c649249b..629d81a789e7 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -95,19 +95,19 @@ #define XE_EXEC_BIND_RETRY_TIMEOUT_MS 1000 -static int xe_exec_begin(struct xe_engine *e, struct ww_acquire_ctx *ww, +static int xe_exec_begin(struct xe_exec_queue *q, struct ww_acquire_ctx *ww, struct ttm_validate_buffer tv_onstack[], struct ttm_validate_buffer **tv, struct list_head *objs) { - struct xe_vm *vm = e->vm; + struct xe_vm *vm = q->vm; struct xe_vma *vma; LIST_HEAD(dups); ktime_t end = 0; int err = 0; *tv = NULL; - if (xe_vm_no_dma_fences(e->vm)) + if (xe_vm_no_dma_fences(q->vm)) return 0; retry: @@ -153,14 +153,14 @@ retry: return err; } -static void xe_exec_end(struct xe_engine *e, +static void xe_exec_end(struct xe_exec_queue *q, struct ttm_validate_buffer *tv_onstack, struct ttm_validate_buffer *tv, struct ww_acquire_ctx *ww, struct list_head *objs) { - if (!xe_vm_no_dma_fences(e->vm)) - xe_vm_unlock_dma_resv(e->vm, tv_onstack, tv, ww, objs); + if (!xe_vm_no_dma_fences(q->vm)) + xe_vm_unlock_dma_resv(q->vm, tv_onstack, tv, ww, objs); } int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) @@ -170,7 +170,7 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) struct drm_xe_exec *args = data; struct drm_xe_sync __user *syncs_user = u64_to_user_ptr(args->syncs); u64 __user *addresses_user = u64_to_user_ptr(args->address); - struct xe_engine *engine; + struct xe_exec_queue *q; struct xe_sync_entry *syncs = NULL; u64 addresses[XE_HW_ENGINE_MAX_INSTANCE]; struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV]; @@ -189,30 +189,30 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) return -EINVAL; - engine = xe_engine_lookup(xef, args->engine_id); - if (XE_IOCTL_DBG(xe, !engine)) + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) return -ENOENT; - if (XE_IOCTL_DBG(xe, engine->flags & ENGINE_FLAG_VM)) + if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) return -EINVAL; - if (XE_IOCTL_DBG(xe, engine->width != args->num_batch_buffer)) + if (XE_IOCTL_DBG(xe, q->width != args->num_batch_buffer)) return -EINVAL; - if (XE_IOCTL_DBG(xe, engine->flags & ENGINE_FLAG_BANNED)) { + if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_BANNED)) { err = -ECANCELED; - goto err_engine; + goto err_exec_queue; } if (args->num_syncs) { syncs = kcalloc(args->num_syncs, sizeof(*syncs), GFP_KERNEL); if (!syncs) { err = -ENOMEM; - goto err_engine; + goto err_exec_queue; } } - vm = engine->vm; + vm = q->vm; for (i = 0; i < args->num_syncs; i++) { err = xe_sync_entry_parse(xe, xef, &syncs[num_syncs++], @@ -222,9 +222,9 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto err_syncs; } - if (xe_engine_is_parallel(engine)) { + if (xe_exec_queue_is_parallel(q)) { err = __copy_from_user(addresses, addresses_user, sizeof(u64) * - engine->width); + q->width); if (err) { err = -EFAULT; goto err_syncs; @@ -294,26 +294,26 @@ retry: goto err_unlock_list; } - err = xe_exec_begin(engine, &ww, tv_onstack, &tv, &objs); + err = xe_exec_begin(q, &ww, tv_onstack, &tv, &objs); if (err) goto err_unlock_list; - if (xe_vm_is_closed_or_banned(engine->vm)) { + if (xe_vm_is_closed_or_banned(q->vm)) { drm_warn(&xe->drm, "Trying to schedule after vm is closed or banned\n"); err = -ECANCELED; - goto err_engine_end; + goto err_exec_queue_end; } - if (xe_engine_is_lr(engine) && xe_engine_ring_full(engine)) { + if (xe_exec_queue_is_lr(q) && xe_exec_queue_ring_full(q)) { err = -EWOULDBLOCK; - goto err_engine_end; + goto err_exec_queue_end; } - job = xe_sched_job_create(engine, xe_engine_is_parallel(engine) ? + job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ? addresses : &args->address); if (IS_ERR(job)) { err = PTR_ERR(job); - goto err_engine_end; + goto err_exec_queue_end; } /* @@ -395,8 +395,8 @@ retry: xe_sync_entry_signal(&syncs[i], job, &job->drm.s_fence->finished); - if (xe_engine_is_lr(engine)) - engine->ring_ops->emit_job(job); + if (xe_exec_queue_is_lr(q)) + q->ring_ops->emit_job(job); xe_sched_job_push(job); xe_vm_reactivate_rebind(vm); @@ -412,8 +412,8 @@ err_repin: err_put_job: if (err) xe_sched_job_put(job); -err_engine_end: - xe_exec_end(engine, tv_onstack, tv, &ww, &objs); +err_exec_queue_end: + xe_exec_end(q, tv_onstack, tv, &ww, &objs); err_unlock_list: if (write_locked) up_write(&vm->lock); @@ -425,8 +425,8 @@ err_syncs: for (i = 0; i < num_syncs; i++) xe_sync_entry_cleanup(&syncs[i]); kfree(syncs); -err_engine: - xe_engine_put(engine); +err_exec_queue: + xe_exec_queue_put(q); return err; } diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index f1cfc4b604d4..1371829b9e35 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -22,57 +22,57 @@ #include "xe_trace.h" #include "xe_vm.h" -static struct xe_engine *__xe_engine_create(struct xe_device *xe, - struct xe_vm *vm, - u32 logical_mask, - u16 width, struct xe_hw_engine *hwe, - u32 flags) +static struct xe_exec_queue *__xe_exec_queue_create(struct xe_device *xe, + struct xe_vm *vm, + u32 logical_mask, + u16 width, struct xe_hw_engine *hwe, + u32 flags) { - struct xe_engine *e; + struct xe_exec_queue *q; struct xe_gt *gt = hwe->gt; int err; int i; - e = kzalloc(sizeof(*e) + sizeof(struct xe_lrc) * width, GFP_KERNEL); - if (!e) + q = kzalloc(sizeof(*q) + sizeof(struct xe_lrc) * width, GFP_KERNEL); + if (!q) return ERR_PTR(-ENOMEM); - kref_init(&e->refcount); - e->flags = flags; - e->hwe = hwe; - e->gt = gt; + kref_init(&q->refcount); + q->flags = flags; + q->hwe = hwe; + q->gt = gt; if (vm) - e->vm = xe_vm_get(vm); - e->class = hwe->class; - e->width = width; - e->logical_mask = logical_mask; - e->fence_irq = >->fence_irq[hwe->class]; - e->ring_ops = gt->ring_ops[hwe->class]; - e->ops = gt->engine_ops; - INIT_LIST_HEAD(&e->persistent.link); - INIT_LIST_HEAD(&e->compute.link); - INIT_LIST_HEAD(&e->multi_gt_link); + q->vm = xe_vm_get(vm); + q->class = hwe->class; + q->width = width; + q->logical_mask = logical_mask; + q->fence_irq = >->fence_irq[hwe->class]; + q->ring_ops = gt->ring_ops[hwe->class]; + q->ops = gt->exec_queue_ops; + INIT_LIST_HEAD(&q->persistent.link); + INIT_LIST_HEAD(&q->compute.link); + INIT_LIST_HEAD(&q->multi_gt_link); /* FIXME: Wire up to configurable default value */ - e->sched_props.timeslice_us = 1 * 1000; - e->sched_props.preempt_timeout_us = 640 * 1000; + q->sched_props.timeslice_us = 1 * 1000; + q->sched_props.preempt_timeout_us = 640 * 1000; - if (xe_engine_is_parallel(e)) { - e->parallel.composite_fence_ctx = dma_fence_context_alloc(1); - e->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO; + if (xe_exec_queue_is_parallel(q)) { + q->parallel.composite_fence_ctx = dma_fence_context_alloc(1); + q->parallel.composite_fence_seqno = XE_FENCE_INITIAL_SEQNO; } - if (e->flags & ENGINE_FLAG_VM) { - e->bind.fence_ctx = dma_fence_context_alloc(1); - e->bind.fence_seqno = XE_FENCE_INITIAL_SEQNO; + if (q->flags & EXEC_QUEUE_FLAG_VM) { + q->bind.fence_ctx = dma_fence_context_alloc(1); + q->bind.fence_seqno = XE_FENCE_INITIAL_SEQNO; } for (i = 0; i < width; ++i) { - err = xe_lrc_init(e->lrc + i, hwe, e, vm, SZ_16K); + err = xe_lrc_init(q->lrc + i, hwe, q, vm, SZ_16K); if (err) goto err_lrc; } - err = e->ops->init(e); + err = q->ops->init(q); if (err) goto err_lrc; @@ -84,24 +84,24 @@ static struct xe_engine *__xe_engine_create(struct xe_device *xe, * can perform GuC CT actions when needed. Caller is expected to * have already grabbed the rpm ref outside any sensitive locks. */ - if (e->flags & ENGINE_FLAG_VM) + if (q->flags & EXEC_QUEUE_FLAG_VM) drm_WARN_ON(&xe->drm, !xe_device_mem_access_get_if_ongoing(xe)); - return e; + return q; err_lrc: for (i = i - 1; i >= 0; --i) - xe_lrc_finish(e->lrc + i); - kfree(e); + xe_lrc_finish(q->lrc + i); + kfree(q); return ERR_PTR(err); } -struct xe_engine *xe_engine_create(struct xe_device *xe, struct xe_vm *vm, - u32 logical_mask, u16 width, - struct xe_hw_engine *hwe, u32 flags) +struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, + u32 logical_mask, u16 width, + struct xe_hw_engine *hwe, u32 flags) { struct ww_acquire_ctx ww; - struct xe_engine *e; + struct xe_exec_queue *q; int err; if (vm) { @@ -109,16 +109,16 @@ struct xe_engine *xe_engine_create(struct xe_device *xe, struct xe_vm *vm, if (err) return ERR_PTR(err); } - e = __xe_engine_create(xe, vm, logical_mask, width, hwe, flags); + q = __xe_exec_queue_create(xe, vm, logical_mask, width, hwe, flags); if (vm) xe_vm_unlock(vm, &ww); - return e; + return q; } -struct xe_engine *xe_engine_create_class(struct xe_device *xe, struct xe_gt *gt, - struct xe_vm *vm, - enum xe_engine_class class, u32 flags) +struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, + enum xe_engine_class class, u32 flags) { struct xe_hw_engine *hwe, *hwe0 = NULL; enum xe_hw_engine_id id; @@ -138,102 +138,102 @@ struct xe_engine *xe_engine_create_class(struct xe_device *xe, struct xe_gt *gt, if (!logical_mask) return ERR_PTR(-ENODEV); - return xe_engine_create(xe, vm, logical_mask, 1, hwe0, flags); + return xe_exec_queue_create(xe, vm, logical_mask, 1, hwe0, flags); } -void xe_engine_destroy(struct kref *ref) +void xe_exec_queue_destroy(struct kref *ref) { - struct xe_engine *e = container_of(ref, struct xe_engine, refcount); - struct xe_engine *engine, *next; + struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); + struct xe_exec_queue *eq, *next; - if (!(e->flags & ENGINE_FLAG_BIND_ENGINE_CHILD)) { - list_for_each_entry_safe(engine, next, &e->multi_gt_list, + if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { + list_for_each_entry_safe(eq, next, &q->multi_gt_list, multi_gt_link) - xe_engine_put(engine); + xe_exec_queue_put(eq); } - e->ops->fini(e); + q->ops->fini(q); } -void xe_engine_fini(struct xe_engine *e) +void xe_exec_queue_fini(struct xe_exec_queue *q) { int i; - for (i = 0; i < e->width; ++i) - xe_lrc_finish(e->lrc + i); - if (e->vm) - xe_vm_put(e->vm); - if (e->flags & ENGINE_FLAG_VM) - xe_device_mem_access_put(gt_to_xe(e->gt)); + for (i = 0; i < q->width; ++i) + xe_lrc_finish(q->lrc + i); + if (q->vm) + xe_vm_put(q->vm); + if (q->flags & EXEC_QUEUE_FLAG_VM) + xe_device_mem_access_put(gt_to_xe(q->gt)); - kfree(e); + kfree(q); } -struct xe_engine *xe_engine_lookup(struct xe_file *xef, u32 id) +struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id) { - struct xe_engine *e; + struct xe_exec_queue *q; - mutex_lock(&xef->engine.lock); - e = xa_load(&xef->engine.xa, id); - if (e) - xe_engine_get(e); - mutex_unlock(&xef->engine.lock); + mutex_lock(&xef->exec_queue.lock); + q = xa_load(&xef->exec_queue.xa, id); + if (q) + xe_exec_queue_get(q); + mutex_unlock(&xef->exec_queue.lock); - return e; + return q; } -enum xe_engine_priority -xe_engine_device_get_max_priority(struct xe_device *xe) +enum xe_exec_queue_priority +xe_exec_queue_device_get_max_priority(struct xe_device *xe) { - return capable(CAP_SYS_NICE) ? XE_ENGINE_PRIORITY_HIGH : - XE_ENGINE_PRIORITY_NORMAL; + return capable(CAP_SYS_NICE) ? XE_EXEC_QUEUE_PRIORITY_HIGH : + XE_EXEC_QUEUE_PRIORITY_NORMAL; } -static int engine_set_priority(struct xe_device *xe, struct xe_engine *e, - u64 value, bool create) +static int exec_queue_set_priority(struct xe_device *xe, struct xe_exec_queue *q, + u64 value, bool create) { - if (XE_IOCTL_DBG(xe, value > XE_ENGINE_PRIORITY_HIGH)) + if (XE_IOCTL_DBG(xe, value > XE_EXEC_QUEUE_PRIORITY_HIGH)) return -EINVAL; - if (XE_IOCTL_DBG(xe, value > xe_engine_device_get_max_priority(xe))) + if (XE_IOCTL_DBG(xe, value > xe_exec_queue_device_get_max_priority(xe))) return -EPERM; - return e->ops->set_priority(e, value); + return q->ops->set_priority(q, value); } -static int engine_set_timeslice(struct xe_device *xe, struct xe_engine *e, - u64 value, bool create) +static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue *q, + u64 value, bool create) { if (!capable(CAP_SYS_NICE)) return -EPERM; - return e->ops->set_timeslice(e, value); + return q->ops->set_timeslice(q, value); } -static int engine_set_preemption_timeout(struct xe_device *xe, - struct xe_engine *e, u64 value, - bool create) +static int exec_queue_set_preemption_timeout(struct xe_device *xe, + struct xe_exec_queue *q, u64 value, + bool create) { if (!capable(CAP_SYS_NICE)) return -EPERM; - return e->ops->set_preempt_timeout(e, value); + return q->ops->set_preempt_timeout(q, value); } -static int engine_set_compute_mode(struct xe_device *xe, struct xe_engine *e, - u64 value, bool create) +static int exec_queue_set_compute_mode(struct xe_device *xe, struct xe_exec_queue *q, + u64 value, bool create) { if (XE_IOCTL_DBG(xe, !create)) return -EINVAL; - if (XE_IOCTL_DBG(xe, e->flags & ENGINE_FLAG_COMPUTE_MODE)) + if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_COMPUTE_MODE)) return -EINVAL; - if (XE_IOCTL_DBG(xe, e->flags & ENGINE_FLAG_VM)) + if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_VM)) return -EINVAL; if (value) { - struct xe_vm *vm = e->vm; + struct xe_vm *vm = q->vm; int err; if (XE_IOCTL_DBG(xe, xe_vm_in_fault_mode(vm))) @@ -242,42 +242,42 @@ static int engine_set_compute_mode(struct xe_device *xe, struct xe_engine *e, if (XE_IOCTL_DBG(xe, !xe_vm_in_compute_mode(vm))) return -EOPNOTSUPP; - if (XE_IOCTL_DBG(xe, e->width != 1)) + if (XE_IOCTL_DBG(xe, q->width != 1)) return -EINVAL; - e->compute.context = dma_fence_context_alloc(1); - spin_lock_init(&e->compute.lock); + q->compute.context = dma_fence_context_alloc(1); + spin_lock_init(&q->compute.lock); - err = xe_vm_add_compute_engine(vm, e); + err = xe_vm_add_compute_exec_queue(vm, q); if (XE_IOCTL_DBG(xe, err)) return err; - e->flags |= ENGINE_FLAG_COMPUTE_MODE; - e->flags &= ~ENGINE_FLAG_PERSISTENT; + q->flags |= EXEC_QUEUE_FLAG_COMPUTE_MODE; + q->flags &= ~EXEC_QUEUE_FLAG_PERSISTENT; } return 0; } -static int engine_set_persistence(struct xe_device *xe, struct xe_engine *e, - u64 value, bool create) +static int exec_queue_set_persistence(struct xe_device *xe, struct xe_exec_queue *q, + u64 value, bool create) { if (XE_IOCTL_DBG(xe, !create)) return -EINVAL; - if (XE_IOCTL_DBG(xe, e->flags & ENGINE_FLAG_COMPUTE_MODE)) + if (XE_IOCTL_DBG(xe, q->flags & EXEC_QUEUE_FLAG_COMPUTE_MODE)) return -EINVAL; if (value) - e->flags |= ENGINE_FLAG_PERSISTENT; + q->flags |= EXEC_QUEUE_FLAG_PERSISTENT; else - e->flags &= ~ENGINE_FLAG_PERSISTENT; + q->flags &= ~EXEC_QUEUE_FLAG_PERSISTENT; return 0; } -static int engine_set_job_timeout(struct xe_device *xe, struct xe_engine *e, - u64 value, bool create) +static int exec_queue_set_job_timeout(struct xe_device *xe, struct xe_exec_queue *q, + u64 value, bool create) { if (XE_IOCTL_DBG(xe, !create)) return -EINVAL; @@ -285,11 +285,11 @@ static int engine_set_job_timeout(struct xe_device *xe, struct xe_engine *e, if (!capable(CAP_SYS_NICE)) return -EPERM; - return e->ops->set_job_timeout(e, value); + return q->ops->set_job_timeout(q, value); } -static int engine_set_acc_trigger(struct xe_device *xe, struct xe_engine *e, - u64 value, bool create) +static int exec_queue_set_acc_trigger(struct xe_device *xe, struct xe_exec_queue *q, + u64 value, bool create) { if (XE_IOCTL_DBG(xe, !create)) return -EINVAL; @@ -297,13 +297,13 @@ static int engine_set_acc_trigger(struct xe_device *xe, struct xe_engine *e, if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) return -EINVAL; - e->usm.acc_trigger = value; + q->usm.acc_trigger = value; return 0; } -static int engine_set_acc_notify(struct xe_device *xe, struct xe_engine *e, - u64 value, bool create) +static int exec_queue_set_acc_notify(struct xe_device *xe, struct xe_exec_queue *q, + u64 value, bool create) { if (XE_IOCTL_DBG(xe, !create)) return -EINVAL; @@ -311,13 +311,13 @@ static int engine_set_acc_notify(struct xe_device *xe, struct xe_engine *e, if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) return -EINVAL; - e->usm.acc_notify = value; + q->usm.acc_notify = value; return 0; } -static int engine_set_acc_granularity(struct xe_device *xe, struct xe_engine *e, - u64 value, bool create) +static int exec_queue_set_acc_granularity(struct xe_device *xe, struct xe_exec_queue *q, + u64 value, bool create) { if (XE_IOCTL_DBG(xe, !create)) return -EINVAL; @@ -325,34 +325,34 @@ static int engine_set_acc_granularity(struct xe_device *xe, struct xe_engine *e, if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) return -EINVAL; - e->usm.acc_granularity = value; + q->usm.acc_granularity = value; return 0; } -typedef int (*xe_engine_set_property_fn)(struct xe_device *xe, - struct xe_engine *e, - u64 value, bool create); - -static const xe_engine_set_property_fn engine_set_property_funcs[] = { - [XE_ENGINE_SET_PROPERTY_PRIORITY] = engine_set_priority, - [XE_ENGINE_SET_PROPERTY_TIMESLICE] = engine_set_timeslice, - [XE_ENGINE_SET_PROPERTY_PREEMPTION_TIMEOUT] = engine_set_preemption_timeout, - [XE_ENGINE_SET_PROPERTY_COMPUTE_MODE] = engine_set_compute_mode, - [XE_ENGINE_SET_PROPERTY_PERSISTENCE] = engine_set_persistence, - [XE_ENGINE_SET_PROPERTY_JOB_TIMEOUT] = engine_set_job_timeout, - [XE_ENGINE_SET_PROPERTY_ACC_TRIGGER] = engine_set_acc_trigger, - [XE_ENGINE_SET_PROPERTY_ACC_NOTIFY] = engine_set_acc_notify, - [XE_ENGINE_SET_PROPERTY_ACC_GRANULARITY] = engine_set_acc_granularity, +typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, + struct xe_exec_queue *q, + u64 value, bool create); + +static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { + [XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, + [XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, + [XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT] = exec_queue_set_preemption_timeout, + [XE_EXEC_QUEUE_SET_PROPERTY_COMPUTE_MODE] = exec_queue_set_compute_mode, + [XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE] = exec_queue_set_persistence, + [XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT] = exec_queue_set_job_timeout, + [XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER] = exec_queue_set_acc_trigger, + [XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY] = exec_queue_set_acc_notify, + [XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY] = exec_queue_set_acc_granularity, }; -static int engine_user_ext_set_property(struct xe_device *xe, - struct xe_engine *e, - u64 extension, - bool create) +static int exec_queue_user_ext_set_property(struct xe_device *xe, + struct xe_exec_queue *q, + u64 extension, + bool create) { u64 __user *address = u64_to_user_ptr(extension); - struct drm_xe_ext_engine_set_property ext; + struct drm_xe_ext_exec_queue_set_property ext; int err; u32 idx; @@ -361,26 +361,26 @@ static int engine_user_ext_set_property(struct xe_device *xe, return -EFAULT; if (XE_IOCTL_DBG(xe, ext.property >= - ARRAY_SIZE(engine_set_property_funcs)) || + ARRAY_SIZE(exec_queue_set_property_funcs)) || XE_IOCTL_DBG(xe, ext.pad)) return -EINVAL; - idx = array_index_nospec(ext.property, ARRAY_SIZE(engine_set_property_funcs)); - return engine_set_property_funcs[idx](xe, e, ext.value, create); + idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); + return exec_queue_set_property_funcs[idx](xe, q, ext.value, create); } -typedef int (*xe_engine_user_extension_fn)(struct xe_device *xe, - struct xe_engine *e, - u64 extension, - bool create); +typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, + struct xe_exec_queue *q, + u64 extension, + bool create); -static const xe_engine_set_property_fn engine_user_extension_funcs[] = { - [XE_ENGINE_EXTENSION_SET_PROPERTY] = engine_user_ext_set_property, +static const xe_exec_queue_set_property_fn exec_queue_user_extension_funcs[] = { + [XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, }; #define MAX_USER_EXTENSIONS 16 -static int engine_user_extensions(struct xe_device *xe, struct xe_engine *e, - u64 extensions, int ext_number, bool create) +static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue *q, + u64 extensions, int ext_number, bool create) { u64 __user *address = u64_to_user_ptr(extensions); struct xe_user_extension ext; @@ -396,17 +396,17 @@ static int engine_user_extensions(struct xe_device *xe, struct xe_engine *e, if (XE_IOCTL_DBG(xe, ext.pad) || XE_IOCTL_DBG(xe, ext.name >= - ARRAY_SIZE(engine_user_extension_funcs))) + ARRAY_SIZE(exec_queue_user_extension_funcs))) return -EINVAL; idx = array_index_nospec(ext.name, - ARRAY_SIZE(engine_user_extension_funcs)); - err = engine_user_extension_funcs[idx](xe, e, extensions, create); + ARRAY_SIZE(exec_queue_user_extension_funcs)); + err = exec_queue_user_extension_funcs[idx](xe, q, extensions, create); if (XE_IOCTL_DBG(xe, err)) return err; if (ext.next_extension) - return engine_user_extensions(xe, e, ext.next_extension, + return exec_queue_user_extensions(xe, q, ext.next_extension, ++ext_number, create); return 0; @@ -440,9 +440,9 @@ find_hw_engine(struct xe_device *xe, eci.engine_instance, true); } -static u32 bind_engine_logical_mask(struct xe_device *xe, struct xe_gt *gt, - struct drm_xe_engine_class_instance *eci, - u16 width, u16 num_placements) +static u32 bind_exec_queue_logical_mask(struct xe_device *xe, struct xe_gt *gt, + struct drm_xe_engine_class_instance *eci, + u16 width, u16 num_placements) { struct xe_hw_engine *hwe; enum xe_hw_engine_id id; @@ -520,19 +520,19 @@ static u32 calc_validate_logical_mask(struct xe_device *xe, struct xe_gt *gt, return return_mask; } -int xe_engine_create_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) +int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) { struct xe_device *xe = to_xe_device(dev); struct xe_file *xef = to_xe_file(file); - struct drm_xe_engine_create *args = data; + struct drm_xe_exec_queue_create *args = data; struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; struct drm_xe_engine_class_instance __user *user_eci = u64_to_user_ptr(args->instances); struct xe_hw_engine *hwe; struct xe_vm *vm, *migrate_vm; struct xe_gt *gt; - struct xe_engine *e = NULL; + struct xe_exec_queue *q = NULL; u32 logical_mask; u32 id; u32 len; @@ -557,15 +557,15 @@ int xe_engine_create_ioctl(struct drm_device *dev, void *data, if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { for_each_gt(gt, xe, id) { - struct xe_engine *new; + struct xe_exec_queue *new; if (xe_gt_is_media_type(gt)) continue; eci[0].gt_id = gt->info.id; - logical_mask = bind_engine_logical_mask(xe, gt, eci, - args->width, - args->num_placements); + logical_mask = bind_exec_queue_logical_mask(xe, gt, eci, + args->width, + args->num_placements); if (XE_IOCTL_DBG(xe, !logical_mask)) return -EINVAL; @@ -577,28 +577,28 @@ int xe_engine_create_ioctl(struct drm_device *dev, void *data, xe_device_mem_access_get(xe); migrate_vm = xe_migrate_get_vm(gt_to_tile(gt)->migrate); - new = xe_engine_create(xe, migrate_vm, logical_mask, - args->width, hwe, - ENGINE_FLAG_PERSISTENT | - ENGINE_FLAG_VM | - (id ? - ENGINE_FLAG_BIND_ENGINE_CHILD : - 0)); + new = xe_exec_queue_create(xe, migrate_vm, logical_mask, + args->width, hwe, + EXEC_QUEUE_FLAG_PERSISTENT | + EXEC_QUEUE_FLAG_VM | + (id ? + EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : + 0)); xe_device_mem_access_put(xe); /* now held by engine */ xe_vm_put(migrate_vm); if (IS_ERR(new)) { err = PTR_ERR(new); - if (e) - goto put_engine; + if (q) + goto put_exec_queue; return err; } if (id == 0) - e = new; + q = new; else list_add_tail(&new->multi_gt_list, - &e->multi_gt_link); + &q->multi_gt_link); } } else { gt = xe_device_get_gt(xe, eci[0].gt_id); @@ -628,223 +628,223 @@ int xe_engine_create_ioctl(struct drm_device *dev, void *data, return -ENOENT; } - e = xe_engine_create(xe, vm, logical_mask, - args->width, hwe, - xe_vm_no_dma_fences(vm) ? 0 : - ENGINE_FLAG_PERSISTENT); + q = xe_exec_queue_create(xe, vm, logical_mask, + args->width, hwe, + xe_vm_no_dma_fences(vm) ? 0 : + EXEC_QUEUE_FLAG_PERSISTENT); up_read(&vm->lock); xe_vm_put(vm); - if (IS_ERR(e)) - return PTR_ERR(e); + if (IS_ERR(q)) + return PTR_ERR(q); } if (args->extensions) { - err = engine_user_extensions(xe, e, args->extensions, 0, true); + err = exec_queue_user_extensions(xe, q, args->extensions, 0, true); if (XE_IOCTL_DBG(xe, err)) - goto put_engine; + goto put_exec_queue; } - if (XE_IOCTL_DBG(xe, e->vm && xe_vm_in_compute_mode(e->vm) != - !!(e->flags & ENGINE_FLAG_COMPUTE_MODE))) { + if (XE_IOCTL_DBG(xe, q->vm && xe_vm_in_compute_mode(q->vm) != + !!(q->flags & EXEC_QUEUE_FLAG_COMPUTE_MODE))) { err = -EOPNOTSUPP; - goto put_engine; + goto put_exec_queue; } - e->persistent.xef = xef; + q->persistent.xef = xef; - mutex_lock(&xef->engine.lock); - err = xa_alloc(&xef->engine.xa, &id, e, xa_limit_32b, GFP_KERNEL); - mutex_unlock(&xef->engine.lock); + mutex_lock(&xef->exec_queue.lock); + err = xa_alloc(&xef->exec_queue.xa, &id, q, xa_limit_32b, GFP_KERNEL); + mutex_unlock(&xef->exec_queue.lock); if (err) - goto put_engine; + goto put_exec_queue; - args->engine_id = id; + args->exec_queue_id = id; return 0; -put_engine: - xe_engine_kill(e); - xe_engine_put(e); +put_exec_queue: + xe_exec_queue_kill(q); + xe_exec_queue_put(q); return err; } -int xe_engine_get_property_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) +int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) { struct xe_device *xe = to_xe_device(dev); struct xe_file *xef = to_xe_file(file); - struct drm_xe_engine_get_property *args = data; - struct xe_engine *e; + struct drm_xe_exec_queue_get_property *args = data; + struct xe_exec_queue *q; int ret; if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) return -EINVAL; - e = xe_engine_lookup(xef, args->engine_id); - if (XE_IOCTL_DBG(xe, !e)) + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) return -ENOENT; switch (args->property) { - case XE_ENGINE_GET_PROPERTY_BAN: - args->value = !!(e->flags & ENGINE_FLAG_BANNED); + case XE_EXEC_QUEUE_GET_PROPERTY_BAN: + args->value = !!(q->flags & EXEC_QUEUE_FLAG_BANNED); ret = 0; break; default: ret = -EINVAL; } - xe_engine_put(e); + xe_exec_queue_put(q); return ret; } -static void engine_kill_compute(struct xe_engine *e) +static void exec_queue_kill_compute(struct xe_exec_queue *q) { - if (!xe_vm_in_compute_mode(e->vm)) + if (!xe_vm_in_compute_mode(q->vm)) return; - down_write(&e->vm->lock); - list_del(&e->compute.link); - --e->vm->preempt.num_engines; - if (e->compute.pfence) { - dma_fence_enable_sw_signaling(e->compute.pfence); - dma_fence_put(e->compute.pfence); - e->compute.pfence = NULL; + down_write(&q->vm->lock); + list_del(&q->compute.link); + --q->vm->preempt.num_exec_queues; + if (q->compute.pfence) { + dma_fence_enable_sw_signaling(q->compute.pfence); + dma_fence_put(q->compute.pfence); + q->compute.pfence = NULL; } - up_write(&e->vm->lock); + up_write(&q->vm->lock); } /** - * xe_engine_is_lr() - Whether an engine is long-running - * @e: The engine + * xe_exec_queue_is_lr() - Whether an exec_queue is long-running + * @q: The exec_queue * - * Return: True if the engine is long-running, false otherwise. + * Return: True if the exec_queue is long-running, false otherwise. */ -bool xe_engine_is_lr(struct xe_engine *e) +bool xe_exec_queue_is_lr(struct xe_exec_queue *q) { - return e->vm && xe_vm_no_dma_fences(e->vm) && - !(e->flags & ENGINE_FLAG_VM); + return q->vm && xe_vm_no_dma_fences(q->vm) && + !(q->flags & EXEC_QUEUE_FLAG_VM); } -static s32 xe_engine_num_job_inflight(struct xe_engine *e) +static s32 xe_exec_queue_num_job_inflight(struct xe_exec_queue *q) { - return e->lrc->fence_ctx.next_seqno - xe_lrc_seqno(e->lrc) - 1; + return q->lrc->fence_ctx.next_seqno - xe_lrc_seqno(q->lrc) - 1; } /** - * xe_engine_ring_full() - Whether an engine's ring is full - * @e: The engine + * xe_exec_queue_ring_full() - Whether an exec_queue's ring is full + * @q: The exec_queue * - * Return: True if the engine's ring is full, false otherwise. + * Return: True if the exec_queue's ring is full, false otherwise. */ -bool xe_engine_ring_full(struct xe_engine *e) +bool xe_exec_queue_ring_full(struct xe_exec_queue *q) { - struct xe_lrc *lrc = e->lrc; + struct xe_lrc *lrc = q->lrc; s32 max_job = lrc->ring.size / MAX_JOB_SIZE_BYTES; - return xe_engine_num_job_inflight(e) >= max_job; + return xe_exec_queue_num_job_inflight(q) >= max_job; } /** - * xe_engine_is_idle() - Whether an engine is idle. - * @engine: The engine + * xe_exec_queue_is_idle() - Whether an exec_queue is idle. + * @q: The exec_queue * * FIXME: Need to determine what to use as the short-lived - * timeline lock for the engines, so that the return value + * timeline lock for the exec_queues, so that the return value * of this function becomes more than just an advisory * snapshot in time. The timeline lock must protect the - * seqno from racing submissions on the same engine. + * seqno from racing submissions on the same exec_queue. * Typically vm->resv, but user-created timeline locks use the migrate vm * and never grabs the migrate vm->resv so we have a race there. * - * Return: True if the engine is idle, false otherwise. + * Return: True if the exec_queue is idle, false otherwise. */ -bool xe_engine_is_idle(struct xe_engine *engine) +bool xe_exec_queue_is_idle(struct xe_exec_queue *q) { - if (XE_WARN_ON(xe_engine_is_parallel(engine))) + if (XE_WARN_ON(xe_exec_queue_is_parallel(q))) return false; - return xe_lrc_seqno(&engine->lrc[0]) == - engine->lrc[0].fence_ctx.next_seqno - 1; + return xe_lrc_seqno(&q->lrc[0]) == + q->lrc[0].fence_ctx.next_seqno - 1; } -void xe_engine_kill(struct xe_engine *e) +void xe_exec_queue_kill(struct xe_exec_queue *q) { - struct xe_engine *engine = e, *next; + struct xe_exec_queue *eq = q, *next; - list_for_each_entry_safe(engine, next, &engine->multi_gt_list, + list_for_each_entry_safe(eq, next, &eq->multi_gt_list, multi_gt_link) { - e->ops->kill(engine); - engine_kill_compute(engine); + q->ops->kill(eq); + exec_queue_kill_compute(eq); } - e->ops->kill(e); - engine_kill_compute(e); + q->ops->kill(q); + exec_queue_kill_compute(q); } -int xe_engine_destroy_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) +int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) { struct xe_device *xe = to_xe_device(dev); struct xe_file *xef = to_xe_file(file); - struct drm_xe_engine_destroy *args = data; - struct xe_engine *e; + struct drm_xe_exec_queue_destroy *args = data; + struct xe_exec_queue *q; if (XE_IOCTL_DBG(xe, args->pad) || XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) return -EINVAL; - mutex_lock(&xef->engine.lock); - e = xa_erase(&xef->engine.xa, args->engine_id); - mutex_unlock(&xef->engine.lock); - if (XE_IOCTL_DBG(xe, !e)) + mutex_lock(&xef->exec_queue.lock); + q = xa_erase(&xef->exec_queue.xa, args->exec_queue_id); + mutex_unlock(&xef->exec_queue.lock); + if (XE_IOCTL_DBG(xe, !q)) return -ENOENT; - if (!(e->flags & ENGINE_FLAG_PERSISTENT)) - xe_engine_kill(e); + if (!(q->flags & EXEC_QUEUE_FLAG_PERSISTENT)) + xe_exec_queue_kill(q); else - xe_device_add_persistent_engines(xe, e); + xe_device_add_persistent_exec_queues(xe, q); - trace_xe_engine_close(e); - xe_engine_put(e); + trace_xe_exec_queue_close(q); + xe_exec_queue_put(q); return 0; } -int xe_engine_set_property_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) +int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file) { struct xe_device *xe = to_xe_device(dev); struct xe_file *xef = to_xe_file(file); - struct drm_xe_engine_set_property *args = data; - struct xe_engine *e; + struct drm_xe_exec_queue_set_property *args = data; + struct xe_exec_queue *q; int ret; u32 idx; if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) return -EINVAL; - e = xe_engine_lookup(xef, args->engine_id); - if (XE_IOCTL_DBG(xe, !e)) + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) return -ENOENT; if (XE_IOCTL_DBG(xe, args->property >= - ARRAY_SIZE(engine_set_property_funcs))) { + ARRAY_SIZE(exec_queue_set_property_funcs))) { ret = -EINVAL; goto out; } idx = array_index_nospec(args->property, - ARRAY_SIZE(engine_set_property_funcs)); - ret = engine_set_property_funcs[idx](xe, e, args->value, false); + ARRAY_SIZE(exec_queue_set_property_funcs)); + ret = exec_queue_set_property_funcs[idx](xe, q, args->value, false); if (XE_IOCTL_DBG(xe, ret)) goto out; if (args->extensions) - ret = engine_user_extensions(xe, e, args->extensions, 0, - false); + ret = exec_queue_user_extensions(xe, q, args->extensions, 0, + false); out: - xe_engine_put(e); + xe_exec_queue_put(q); return ret; } diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 3017e4fe308d..94a6abee38a6 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -3,10 +3,10 @@ * Copyright © 2021 Intel Corporation */ -#ifndef _XE_ENGINE_H_ -#define _XE_ENGINE_H_ +#ifndef _XE_EXEC_QUEUE_H_ +#define _XE_EXEC_QUEUE_H_ -#include "xe_engine_types.h" +#include "xe_exec_queue_types.h" #include "xe_vm_types.h" struct drm_device; @@ -14,50 +14,50 @@ struct drm_file; struct xe_device; struct xe_file; -struct xe_engine *xe_engine_create(struct xe_device *xe, struct xe_vm *vm, - u32 logical_mask, u16 width, - struct xe_hw_engine *hw_engine, u32 flags); -struct xe_engine *xe_engine_create_class(struct xe_device *xe, struct xe_gt *gt, - struct xe_vm *vm, - enum xe_engine_class class, u32 flags); +struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *vm, + u32 logical_mask, u16 width, + struct xe_hw_engine *hw_engine, u32 flags); +struct xe_exec_queue *xe_exec_queue_create_class(struct xe_device *xe, struct xe_gt *gt, + struct xe_vm *vm, + enum xe_engine_class class, u32 flags); -void xe_engine_fini(struct xe_engine *e); -void xe_engine_destroy(struct kref *ref); +void xe_exec_queue_fini(struct xe_exec_queue *q); +void xe_exec_queue_destroy(struct kref *ref); -struct xe_engine *xe_engine_lookup(struct xe_file *xef, u32 id); +struct xe_exec_queue *xe_exec_queue_lookup(struct xe_file *xef, u32 id); -static inline struct xe_engine *xe_engine_get(struct xe_engine *engine) +static inline struct xe_exec_queue *xe_exec_queue_get(struct xe_exec_queue *q) { - kref_get(&engine->refcount); - return engine; + kref_get(&q->refcount); + return q; } -static inline void xe_engine_put(struct xe_engine *engine) +static inline void xe_exec_queue_put(struct xe_exec_queue *q) { - kref_put(&engine->refcount, xe_engine_destroy); + kref_put(&q->refcount, xe_exec_queue_destroy); } -static inline bool xe_engine_is_parallel(struct xe_engine *engine) +static inline bool xe_exec_queue_is_parallel(struct xe_exec_queue *q) { - return engine->width > 1; + return q->width > 1; } -bool xe_engine_is_lr(struct xe_engine *e); +bool xe_exec_queue_is_lr(struct xe_exec_queue *q); -bool xe_engine_ring_full(struct xe_engine *e); +bool xe_exec_queue_ring_full(struct xe_exec_queue *q); -bool xe_engine_is_idle(struct xe_engine *engine); +bool xe_exec_queue_is_idle(struct xe_exec_queue *q); -void xe_engine_kill(struct xe_engine *e); +void xe_exec_queue_kill(struct xe_exec_queue *q); -int xe_engine_create_ioctl(struct drm_device *dev, void *data, - struct drm_file *file); -int xe_engine_destroy_ioctl(struct drm_device *dev, void *data, - struct drm_file *file); -int xe_engine_set_property_ioctl(struct drm_device *dev, void *data, - struct drm_file *file); -int xe_engine_get_property_ioctl(struct drm_device *dev, void *data, - struct drm_file *file); -enum xe_engine_priority xe_engine_device_get_max_priority(struct xe_device *xe); +int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, + struct drm_file *file); +enum xe_exec_queue_priority xe_exec_queue_device_get_max_priority(struct xe_device *xe); #endif diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h new file mode 100644 index 000000000000..4506289b8b7b --- /dev/null +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -0,0 +1,209 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_EXEC_QUEUE_TYPES_H_ +#define _XE_EXEC_QUEUE_TYPES_H_ + +#include + +#include + +#include "xe_gpu_scheduler_types.h" +#include "xe_hw_engine_types.h" +#include "xe_hw_fence_types.h" +#include "xe_lrc_types.h" + +struct xe_execlist_exec_queue; +struct xe_gt; +struct xe_guc_exec_queue; +struct xe_hw_engine; +struct xe_vm; + +enum xe_exec_queue_priority { + XE_EXEC_QUEUE_PRIORITY_UNSET = -2, /* For execlist usage only */ + XE_EXEC_QUEUE_PRIORITY_LOW = 0, + XE_EXEC_QUEUE_PRIORITY_NORMAL, + XE_EXEC_QUEUE_PRIORITY_HIGH, + XE_EXEC_QUEUE_PRIORITY_KERNEL, + + XE_EXEC_QUEUE_PRIORITY_COUNT +}; + +/** + * struct xe_exec_queue - Execution queue + * + * Contains all state necessary for submissions. Can either be a user object or + * a kernel object. + */ +struct xe_exec_queue { + /** @gt: graphics tile this exec queue can submit to */ + struct xe_gt *gt; + /** + * @hwe: A hardware of the same class. May (physical engine) or may not + * (virtual engine) be where jobs actual engine up running. Should never + * really be used for submissions. + */ + struct xe_hw_engine *hwe; + /** @refcount: ref count of this exec queue */ + struct kref refcount; + /** @vm: VM (address space) for this exec queue */ + struct xe_vm *vm; + /** @class: class of this exec queue */ + enum xe_engine_class class; + /** @priority: priority of this exec queue */ + enum xe_exec_queue_priority priority; + /** + * @logical_mask: logical mask of where job submitted to exec queue can run + */ + u32 logical_mask; + /** @name: name of this exec queue */ + char name[MAX_FENCE_NAME_LEN]; + /** @width: width (number BB submitted per exec) of this exec queue */ + u16 width; + /** @fence_irq: fence IRQ used to signal job completion */ + struct xe_hw_fence_irq *fence_irq; + +#define EXEC_QUEUE_FLAG_BANNED BIT(0) +#define EXEC_QUEUE_FLAG_KERNEL BIT(1) +#define EXEC_QUEUE_FLAG_PERSISTENT BIT(2) +#define EXEC_QUEUE_FLAG_COMPUTE_MODE BIT(3) +/* Caller needs to hold rpm ref when creating engine with EXEC_QUEUE_FLAG_VM */ +#define EXEC_QUEUE_FLAG_VM BIT(4) +#define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD BIT(5) +#define EXEC_QUEUE_FLAG_WA BIT(6) + + /** + * @flags: flags for this exec queue, should statically setup aside from ban + * bit + */ + unsigned long flags; + + union { + /** @multi_gt_list: list head for VM bind engines if multi-GT */ + struct list_head multi_gt_list; + /** @multi_gt_link: link for VM bind engines if multi-GT */ + struct list_head multi_gt_link; + }; + + union { + /** @execlist: execlist backend specific state for exec queue */ + struct xe_execlist_exec_queue *execlist; + /** @guc: GuC backend specific state for exec queue */ + struct xe_guc_exec_queue *guc; + }; + + /** + * @persistent: persistent exec queue state + */ + struct { + /** @xef: file which this exec queue belongs to */ + struct xe_file *xef; + /** @link: link in list of persistent exec queues */ + struct list_head link; + } persistent; + + union { + /** + * @parallel: parallel submission state + */ + struct { + /** @composite_fence_ctx: context composite fence */ + u64 composite_fence_ctx; + /** @composite_fence_seqno: seqno for composite fence */ + u32 composite_fence_seqno; + } parallel; + /** + * @bind: bind submission state + */ + struct { + /** @fence_ctx: context bind fence */ + u64 fence_ctx; + /** @fence_seqno: seqno for bind fence */ + u32 fence_seqno; + } bind; + }; + + /** @sched_props: scheduling properties */ + struct { + /** @timeslice_us: timeslice period in micro-seconds */ + u32 timeslice_us; + /** @preempt_timeout_us: preemption timeout in micro-seconds */ + u32 preempt_timeout_us; + } sched_props; + + /** @compute: compute exec queue state */ + struct { + /** @pfence: preemption fence */ + struct dma_fence *pfence; + /** @context: preemption fence context */ + u64 context; + /** @seqno: preemption fence seqno */ + u32 seqno; + /** @link: link into VM's list of exec queues */ + struct list_head link; + /** @lock: preemption fences lock */ + spinlock_t lock; + } compute; + + /** @usm: unified shared memory state */ + struct { + /** @acc_trigger: access counter trigger */ + u32 acc_trigger; + /** @acc_notify: access counter notify */ + u32 acc_notify; + /** @acc_granularity: access counter granularity */ + u32 acc_granularity; + } usm; + + /** @ops: submission backend exec queue operations */ + const struct xe_exec_queue_ops *ops; + + /** @ring_ops: ring operations for this exec queue */ + const struct xe_ring_ops *ring_ops; + /** @entity: DRM sched entity for this exec queue (1 to 1 relationship) */ + struct drm_sched_entity *entity; + /** @lrc: logical ring context for this exec queue */ + struct xe_lrc lrc[]; +}; + +/** + * struct xe_exec_queue_ops - Submission backend exec queue operations + */ +struct xe_exec_queue_ops { + /** @init: Initialize exec queue for submission backend */ + int (*init)(struct xe_exec_queue *q); + /** @kill: Kill inflight submissions for backend */ + void (*kill)(struct xe_exec_queue *q); + /** @fini: Fini exec queue for submission backend */ + void (*fini)(struct xe_exec_queue *q); + /** @set_priority: Set priority for exec queue */ + int (*set_priority)(struct xe_exec_queue *q, + enum xe_exec_queue_priority priority); + /** @set_timeslice: Set timeslice for exec queue */ + int (*set_timeslice)(struct xe_exec_queue *q, u32 timeslice_us); + /** @set_preempt_timeout: Set preemption timeout for exec queue */ + int (*set_preempt_timeout)(struct xe_exec_queue *q, u32 preempt_timeout_us); + /** @set_job_timeout: Set job timeout for exec queue */ + int (*set_job_timeout)(struct xe_exec_queue *q, u32 job_timeout_ms); + /** + * @suspend: Suspend exec queue from executing, allowed to be called + * multiple times in a row before resume with the caveat that + * suspend_wait returns before calling suspend again. + */ + int (*suspend)(struct xe_exec_queue *q); + /** + * @suspend_wait: Wait for an exec queue to suspend executing, should be + * call after suspend. + */ + void (*suspend_wait)(struct xe_exec_queue *q); + /** + * @resume: Resume exec queue execution, exec queue must be in a suspended + * state and dma fence returned from most recent suspend call must be + * signalled when this function is called. + */ + void (*resume)(struct xe_exec_queue *q); +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c index 5b6748e1a37f..3b8be55fe19c 100644 --- a/drivers/gpu/drm/xe/xe_execlist.c +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -91,7 +91,7 @@ static void __start_lrc(struct xe_hw_engine *hwe, struct xe_lrc *lrc, } static void __xe_execlist_port_start(struct xe_execlist_port *port, - struct xe_execlist_engine *exl) + struct xe_execlist_exec_queue *exl) { struct xe_device *xe = gt_to_xe(port->hwe->gt); int max_ctx = FIELD_MAX(GEN11_SW_CTX_ID); @@ -109,7 +109,7 @@ static void __xe_execlist_port_start(struct xe_execlist_port *port, port->last_ctx_id = 1; } - __start_lrc(port->hwe, exl->engine->lrc, port->last_ctx_id); + __start_lrc(port->hwe, exl->q->lrc, port->last_ctx_id); port->running_exl = exl; exl->has_run = true; } @@ -128,16 +128,16 @@ static void __xe_execlist_port_idle(struct xe_execlist_port *port) port->running_exl = NULL; } -static bool xe_execlist_is_idle(struct xe_execlist_engine *exl) +static bool xe_execlist_is_idle(struct xe_execlist_exec_queue *exl) { - struct xe_lrc *lrc = exl->engine->lrc; + struct xe_lrc *lrc = exl->q->lrc; return lrc->ring.tail == lrc->ring.old_tail; } static void __xe_execlist_port_start_next_active(struct xe_execlist_port *port) { - struct xe_execlist_engine *exl = NULL; + struct xe_execlist_exec_queue *exl = NULL; int i; xe_execlist_port_assert_held(port); @@ -145,12 +145,12 @@ static void __xe_execlist_port_start_next_active(struct xe_execlist_port *port) for (i = ARRAY_SIZE(port->active) - 1; i >= 0; i--) { while (!list_empty(&port->active[i])) { exl = list_first_entry(&port->active[i], - struct xe_execlist_engine, + struct xe_execlist_exec_queue, active_link); list_del(&exl->active_link); if (xe_execlist_is_idle(exl)) { - exl->active_priority = XE_ENGINE_PRIORITY_UNSET; + exl->active_priority = XE_EXEC_QUEUE_PRIORITY_UNSET; continue; } @@ -198,7 +198,7 @@ static void xe_execlist_port_irq_handler(struct xe_hw_engine *hwe, } static void xe_execlist_port_wake_locked(struct xe_execlist_port *port, - enum xe_engine_priority priority) + enum xe_exec_queue_priority priority) { xe_execlist_port_assert_held(port); @@ -208,25 +208,25 @@ static void xe_execlist_port_wake_locked(struct xe_execlist_port *port, __xe_execlist_port_start_next_active(port); } -static void xe_execlist_make_active(struct xe_execlist_engine *exl) +static void xe_execlist_make_active(struct xe_execlist_exec_queue *exl) { struct xe_execlist_port *port = exl->port; - enum xe_engine_priority priority = exl->active_priority; + enum xe_exec_queue_priority priority = exl->active_priority; - XE_WARN_ON(priority == XE_ENGINE_PRIORITY_UNSET); + XE_WARN_ON(priority == XE_EXEC_QUEUE_PRIORITY_UNSET); XE_WARN_ON(priority < 0); XE_WARN_ON(priority >= ARRAY_SIZE(exl->port->active)); spin_lock_irq(&port->lock); if (exl->active_priority != priority && - exl->active_priority != XE_ENGINE_PRIORITY_UNSET) { + exl->active_priority != XE_EXEC_QUEUE_PRIORITY_UNSET) { /* Priority changed, move it to the right list */ list_del(&exl->active_link); - exl->active_priority = XE_ENGINE_PRIORITY_UNSET; + exl->active_priority = XE_EXEC_QUEUE_PRIORITY_UNSET; } - if (exl->active_priority == XE_ENGINE_PRIORITY_UNSET) { + if (exl->active_priority == XE_EXEC_QUEUE_PRIORITY_UNSET) { exl->active_priority = priority; list_add_tail(&exl->active_link, &port->active[priority]); } @@ -293,10 +293,10 @@ static struct dma_fence * execlist_run_job(struct drm_sched_job *drm_job) { struct xe_sched_job *job = to_xe_sched_job(drm_job); - struct xe_engine *e = job->engine; - struct xe_execlist_engine *exl = job->engine->execlist; + struct xe_exec_queue *q = job->q; + struct xe_execlist_exec_queue *exl = job->q->execlist; - e->ring_ops->emit_job(job); + q->ring_ops->emit_job(job); xe_execlist_make_active(exl); return dma_fence_get(job->fence); @@ -314,11 +314,11 @@ static const struct drm_sched_backend_ops drm_sched_ops = { .free_job = execlist_job_free, }; -static int execlist_engine_init(struct xe_engine *e) +static int execlist_exec_queue_init(struct xe_exec_queue *q) { struct drm_gpu_scheduler *sched; - struct xe_execlist_engine *exl; - struct xe_device *xe = gt_to_xe(e->gt); + struct xe_execlist_exec_queue *exl; + struct xe_device *xe = gt_to_xe(q->gt); int err; XE_WARN_ON(xe_device_guc_submission_enabled(xe)); @@ -329,13 +329,13 @@ static int execlist_engine_init(struct xe_engine *e) if (!exl) return -ENOMEM; - exl->engine = e; + exl->q = q; err = drm_sched_init(&exl->sched, &drm_sched_ops, NULL, 1, - e->lrc[0].ring.size / MAX_JOB_SIZE_BYTES, + q->lrc[0].ring.size / MAX_JOB_SIZE_BYTES, XE_SCHED_HANG_LIMIT, XE_SCHED_JOB_TIMEOUT, - NULL, NULL, e->hwe->name, - gt_to_xe(e->gt)->drm.dev); + NULL, NULL, q->hwe->name, + gt_to_xe(q->gt)->drm.dev); if (err) goto err_free; @@ -344,30 +344,30 @@ static int execlist_engine_init(struct xe_engine *e) if (err) goto err_sched; - exl->port = e->hwe->exl_port; + exl->port = q->hwe->exl_port; exl->has_run = false; - exl->active_priority = XE_ENGINE_PRIORITY_UNSET; - e->execlist = exl; - e->entity = &exl->entity; + exl->active_priority = XE_EXEC_QUEUE_PRIORITY_UNSET; + q->execlist = exl; + q->entity = &exl->entity; - switch (e->class) { + switch (q->class) { case XE_ENGINE_CLASS_RENDER: - sprintf(e->name, "rcs%d", ffs(e->logical_mask) - 1); + sprintf(q->name, "rcs%d", ffs(q->logical_mask) - 1); break; case XE_ENGINE_CLASS_VIDEO_DECODE: - sprintf(e->name, "vcs%d", ffs(e->logical_mask) - 1); + sprintf(q->name, "vcs%d", ffs(q->logical_mask) - 1); break; case XE_ENGINE_CLASS_VIDEO_ENHANCE: - sprintf(e->name, "vecs%d", ffs(e->logical_mask) - 1); + sprintf(q->name, "vecs%d", ffs(q->logical_mask) - 1); break; case XE_ENGINE_CLASS_COPY: - sprintf(e->name, "bcs%d", ffs(e->logical_mask) - 1); + sprintf(q->name, "bcs%d", ffs(q->logical_mask) - 1); break; case XE_ENGINE_CLASS_COMPUTE: - sprintf(e->name, "ccs%d", ffs(e->logical_mask) - 1); + sprintf(q->name, "ccs%d", ffs(q->logical_mask) - 1); break; default: - XE_WARN_ON(e->class); + XE_WARN_ON(q->class); } return 0; @@ -379,96 +379,96 @@ err_free: return err; } -static void execlist_engine_fini_async(struct work_struct *w) +static void execlist_exec_queue_fini_async(struct work_struct *w) { - struct xe_execlist_engine *ee = - container_of(w, struct xe_execlist_engine, fini_async); - struct xe_engine *e = ee->engine; - struct xe_execlist_engine *exl = e->execlist; + struct xe_execlist_exec_queue *ee = + container_of(w, struct xe_execlist_exec_queue, fini_async); + struct xe_exec_queue *q = ee->q; + struct xe_execlist_exec_queue *exl = q->execlist; unsigned long flags; - XE_WARN_ON(xe_device_guc_submission_enabled(gt_to_xe(e->gt))); + XE_WARN_ON(xe_device_guc_submission_enabled(gt_to_xe(q->gt))); spin_lock_irqsave(&exl->port->lock, flags); - if (WARN_ON(exl->active_priority != XE_ENGINE_PRIORITY_UNSET)) + if (WARN_ON(exl->active_priority != XE_EXEC_QUEUE_PRIORITY_UNSET)) list_del(&exl->active_link); spin_unlock_irqrestore(&exl->port->lock, flags); - if (e->flags & ENGINE_FLAG_PERSISTENT) - xe_device_remove_persistent_engines(gt_to_xe(e->gt), e); + if (q->flags & EXEC_QUEUE_FLAG_PERSISTENT) + xe_device_remove_persistent_exec_queues(gt_to_xe(q->gt), q); drm_sched_entity_fini(&exl->entity); drm_sched_fini(&exl->sched); kfree(exl); - xe_engine_fini(e); + xe_exec_queue_fini(q); } -static void execlist_engine_kill(struct xe_engine *e) +static void execlist_exec_queue_kill(struct xe_exec_queue *q) { /* NIY */ } -static void execlist_engine_fini(struct xe_engine *e) +static void execlist_exec_queue_fini(struct xe_exec_queue *q) { - INIT_WORK(&e->execlist->fini_async, execlist_engine_fini_async); - queue_work(system_unbound_wq, &e->execlist->fini_async); + INIT_WORK(&q->execlist->fini_async, execlist_exec_queue_fini_async); + queue_work(system_unbound_wq, &q->execlist->fini_async); } -static int execlist_engine_set_priority(struct xe_engine *e, - enum xe_engine_priority priority) +static int execlist_exec_queue_set_priority(struct xe_exec_queue *q, + enum xe_exec_queue_priority priority) { /* NIY */ return 0; } -static int execlist_engine_set_timeslice(struct xe_engine *e, u32 timeslice_us) +static int execlist_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) { /* NIY */ return 0; } -static int execlist_engine_set_preempt_timeout(struct xe_engine *e, - u32 preempt_timeout_us) +static int execlist_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, + u32 preempt_timeout_us) { /* NIY */ return 0; } -static int execlist_engine_set_job_timeout(struct xe_engine *e, - u32 job_timeout_ms) +static int execlist_exec_queue_set_job_timeout(struct xe_exec_queue *q, + u32 job_timeout_ms) { /* NIY */ return 0; } -static int execlist_engine_suspend(struct xe_engine *e) +static int execlist_exec_queue_suspend(struct xe_exec_queue *q) { /* NIY */ return 0; } -static void execlist_engine_suspend_wait(struct xe_engine *e) +static void execlist_exec_queue_suspend_wait(struct xe_exec_queue *q) { /* NIY */ } -static void execlist_engine_resume(struct xe_engine *e) +static void execlist_exec_queue_resume(struct xe_exec_queue *q) { /* NIY */ } -static const struct xe_engine_ops execlist_engine_ops = { - .init = execlist_engine_init, - .kill = execlist_engine_kill, - .fini = execlist_engine_fini, - .set_priority = execlist_engine_set_priority, - .set_timeslice = execlist_engine_set_timeslice, - .set_preempt_timeout = execlist_engine_set_preempt_timeout, - .set_job_timeout = execlist_engine_set_job_timeout, - .suspend = execlist_engine_suspend, - .suspend_wait = execlist_engine_suspend_wait, - .resume = execlist_engine_resume, +static const struct xe_exec_queue_ops execlist_exec_queue_ops = { + .init = execlist_exec_queue_init, + .kill = execlist_exec_queue_kill, + .fini = execlist_exec_queue_fini, + .set_priority = execlist_exec_queue_set_priority, + .set_timeslice = execlist_exec_queue_set_timeslice, + .set_preempt_timeout = execlist_exec_queue_set_preempt_timeout, + .set_job_timeout = execlist_exec_queue_set_job_timeout, + .suspend = execlist_exec_queue_suspend, + .suspend_wait = execlist_exec_queue_suspend_wait, + .resume = execlist_exec_queue_resume, }; int xe_execlist_init(struct xe_gt *gt) @@ -477,7 +477,7 @@ int xe_execlist_init(struct xe_gt *gt) if (xe_device_guc_submission_enabled(gt_to_xe(gt))) return 0; - gt->engine_ops = &execlist_engine_ops; + gt->exec_queue_ops = &execlist_exec_queue_ops; return 0; } diff --git a/drivers/gpu/drm/xe/xe_execlist_types.h b/drivers/gpu/drm/xe/xe_execlist_types.h index 9b1239b47292..f94bbf4c53e4 100644 --- a/drivers/gpu/drm/xe/xe_execlist_types.h +++ b/drivers/gpu/drm/xe/xe_execlist_types.h @@ -10,27 +10,27 @@ #include #include -#include "xe_engine_types.h" +#include "xe_exec_queue_types.h" struct xe_hw_engine; -struct xe_execlist_engine; +struct xe_execlist_exec_queue; struct xe_execlist_port { struct xe_hw_engine *hwe; spinlock_t lock; - struct list_head active[XE_ENGINE_PRIORITY_COUNT]; + struct list_head active[XE_EXEC_QUEUE_PRIORITY_COUNT]; u32 last_ctx_id; - struct xe_execlist_engine *running_exl; + struct xe_execlist_exec_queue *running_exl; struct timer_list irq_fail; }; -struct xe_execlist_engine { - struct xe_engine *engine; +struct xe_execlist_exec_queue { + struct xe_exec_queue *q; struct drm_gpu_scheduler sched; @@ -42,7 +42,7 @@ struct xe_execlist_engine { struct work_struct fini_async; - enum xe_engine_priority active_priority; + enum xe_exec_queue_priority active_priority; struct list_head active_link; }; diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 543b085723c5..3077faa1e792 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -26,7 +26,7 @@ #include "xe_gt_sysfs.h" #include "xe_gt_tlb_invalidation.h" #include "xe_gt_topology.h" -#include "xe_guc_engine_types.h" +#include "xe_guc_exec_queue_types.h" #include "xe_hw_fence.h" #include "xe_irq.h" #include "xe_lrc.h" @@ -81,7 +81,7 @@ static void gt_fini(struct drm_device *drm, void *arg) static void gt_reset_worker(struct work_struct *w); -static int emit_nop_job(struct xe_gt *gt, struct xe_engine *e) +static int emit_nop_job(struct xe_gt *gt, struct xe_exec_queue *q) { struct xe_sched_job *job; struct xe_bb *bb; @@ -94,7 +94,7 @@ static int emit_nop_job(struct xe_gt *gt, struct xe_engine *e) return PTR_ERR(bb); batch_ofs = xe_bo_ggtt_addr(gt_to_tile(gt)->mem.kernel_bb_pool->bo); - job = xe_bb_create_wa_job(e, bb, batch_ofs); + job = xe_bb_create_wa_job(q, bb, batch_ofs); if (IS_ERR(job)) { xe_bb_free(bb, NULL); return PTR_ERR(job); @@ -115,9 +115,9 @@ static int emit_nop_job(struct xe_gt *gt, struct xe_engine *e) return 0; } -static int emit_wa_job(struct xe_gt *gt, struct xe_engine *e) +static int emit_wa_job(struct xe_gt *gt, struct xe_exec_queue *q) { - struct xe_reg_sr *sr = &e->hwe->reg_lrc; + struct xe_reg_sr *sr = &q->hwe->reg_lrc; struct xe_reg_sr_entry *entry; unsigned long reg; struct xe_sched_job *job; @@ -143,7 +143,7 @@ static int emit_wa_job(struct xe_gt *gt, struct xe_engine *e) } batch_ofs = xe_bo_ggtt_addr(gt_to_tile(gt)->mem.kernel_bb_pool->bo); - job = xe_bb_create_wa_job(e, bb, batch_ofs); + job = xe_bb_create_wa_job(q, bb, batch_ofs); if (IS_ERR(job)) { xe_bb_free(bb, NULL); return PTR_ERR(job); @@ -173,7 +173,7 @@ int xe_gt_record_default_lrcs(struct xe_gt *gt) int err = 0; for_each_hw_engine(hwe, gt, id) { - struct xe_engine *e, *nop_e; + struct xe_exec_queue *q, *nop_q; struct xe_vm *vm; void *default_lrc; @@ -192,58 +192,58 @@ int xe_gt_record_default_lrcs(struct xe_gt *gt) return -ENOMEM; vm = xe_migrate_get_vm(tile->migrate); - e = xe_engine_create(xe, vm, BIT(hwe->logical_instance), 1, - hwe, ENGINE_FLAG_WA); - if (IS_ERR(e)) { - err = PTR_ERR(e); - xe_gt_err(gt, "hwe %s: xe_engine_create failed (%pe)\n", - hwe->name, e); + q = xe_exec_queue_create(xe, vm, BIT(hwe->logical_instance), 1, + hwe, EXEC_QUEUE_FLAG_WA); + if (IS_ERR(q)) { + err = PTR_ERR(q); + xe_gt_err(gt, "hwe %s: xe_exec_queue_create failed (%pe)\n", + hwe->name, q); goto put_vm; } /* Prime golden LRC with known good state */ - err = emit_wa_job(gt, e); + err = emit_wa_job(gt, q); if (err) { xe_gt_err(gt, "hwe %s: emit_wa_job failed (%pe) guc_id=%u\n", - hwe->name, ERR_PTR(err), e->guc->id); - goto put_engine; + hwe->name, ERR_PTR(err), q->guc->id); + goto put_exec_queue; } - nop_e = xe_engine_create(xe, vm, BIT(hwe->logical_instance), - 1, hwe, ENGINE_FLAG_WA); - if (IS_ERR(nop_e)) { - err = PTR_ERR(nop_e); - xe_gt_err(gt, "hwe %s: nop xe_engine_create failed (%pe)\n", - hwe->name, nop_e); - goto put_engine; + nop_q = xe_exec_queue_create(xe, vm, BIT(hwe->logical_instance), + 1, hwe, EXEC_QUEUE_FLAG_WA); + if (IS_ERR(nop_q)) { + err = PTR_ERR(nop_q); + xe_gt_err(gt, "hwe %s: nop xe_exec_queue_create failed (%pe)\n", + hwe->name, nop_q); + goto put_exec_queue; } /* Switch to different LRC */ - err = emit_nop_job(gt, nop_e); + err = emit_nop_job(gt, nop_q); if (err) { xe_gt_err(gt, "hwe %s: nop emit_nop_job failed (%pe) guc_id=%u\n", - hwe->name, ERR_PTR(err), nop_e->guc->id); - goto put_nop_e; + hwe->name, ERR_PTR(err), nop_q->guc->id); + goto put_nop_q; } /* Reload golden LRC to record the effect of any indirect W/A */ - err = emit_nop_job(gt, e); + err = emit_nop_job(gt, q); if (err) { xe_gt_err(gt, "hwe %s: emit_nop_job failed (%pe) guc_id=%u\n", - hwe->name, ERR_PTR(err), e->guc->id); - goto put_nop_e; + hwe->name, ERR_PTR(err), q->guc->id); + goto put_nop_q; } xe_map_memcpy_from(xe, default_lrc, - &e->lrc[0].bo->vmap, - xe_lrc_pphwsp_offset(&e->lrc[0]), + &q->lrc[0].bo->vmap, + xe_lrc_pphwsp_offset(&q->lrc[0]), xe_lrc_size(xe, hwe->class)); gt->default_lrc[hwe->class] = default_lrc; -put_nop_e: - xe_engine_put(nop_e); -put_engine: - xe_engine_put(e); +put_nop_q: + xe_exec_queue_put(nop_q); +put_exec_queue: + xe_exec_queue_put(q); put_vm: xe_vm_put(vm); if (err) diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index 78a9fe9f0bd3..c326932e53d7 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -14,7 +14,7 @@ #include "xe_sa_types.h" #include "xe_uc_types.h" -struct xe_engine_ops; +struct xe_exec_queue_ops; struct xe_migrate; struct xe_ring_ops; @@ -269,8 +269,8 @@ struct xe_gt { /** @gtidle: idle properties of GT */ struct xe_gt_idle gtidle; - /** @engine_ops: submission backend engine operations */ - const struct xe_engine_ops *engine_ops; + /** @exec_queue_ops: submission backend exec queue operations */ + const struct xe_exec_queue_ops *exec_queue_ops; /** * @ring_ops: ring operations for this hw engine (1 per engine class) diff --git a/drivers/gpu/drm/xe/xe_guc_ads.c b/drivers/gpu/drm/xe/xe_guc_ads.c index a7da29be2e51..7d1244df959d 100644 --- a/drivers/gpu/drm/xe/xe_guc_ads.c +++ b/drivers/gpu/drm/xe/xe_guc_ads.c @@ -495,7 +495,7 @@ static void guc_mmio_reg_state_init(struct xe_guc_ads *ads) u8 gc; /* - * 1. Write all MMIO entries for this engine to the table. No + * 1. Write all MMIO entries for this exec queue to the table. No * need to worry about fused-off engines and when there are * entries in the regset: the reg_state_list has been zero'ed * by xe_guc_ads_populate() diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index fb1d63ffaee4..59136b6a7c6f 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -888,11 +888,11 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) ret = xe_guc_deregister_done_handler(guc, payload, adj_len); break; case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION: - ret = xe_guc_engine_reset_handler(guc, payload, adj_len); + ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len); break; case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION: - ret = xe_guc_engine_reset_failure_handler(guc, payload, - adj_len); + ret = xe_guc_exec_queue_reset_failure_handler(guc, payload, + adj_len); break; case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE: /* Selftest only at the moment */ @@ -902,8 +902,8 @@ static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len) /* FIXME: Handle this */ break; case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR: - ret = xe_guc_engine_memory_cat_error_handler(guc, payload, - adj_len); + ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload, + adj_len); break; case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC: ret = xe_guc_pagefault_handler(guc, payload, adj_len); diff --git a/drivers/gpu/drm/xe/xe_guc_engine_types.h b/drivers/gpu/drm/xe/xe_guc_engine_types.h deleted file mode 100644 index 5565412fe7f1..000000000000 --- a/drivers/gpu/drm/xe/xe_guc_engine_types.h +++ /dev/null @@ -1,54 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2022 Intel Corporation - */ - -#ifndef _XE_GUC_ENGINE_TYPES_H_ -#define _XE_GUC_ENGINE_TYPES_H_ - -#include -#include - -#include "xe_gpu_scheduler_types.h" - -struct dma_fence; -struct xe_engine; - -/** - * struct xe_guc_engine - GuC specific state for an xe_engine - */ -struct xe_guc_engine { - /** @engine: Backpointer to parent xe_engine */ - struct xe_engine *engine; - /** @sched: GPU scheduler for this xe_engine */ - struct xe_gpu_scheduler sched; - /** @entity: Scheduler entity for this xe_engine */ - struct xe_sched_entity entity; - /** - * @static_msgs: Static messages for this xe_engine, used when a message - * needs to sent through the GPU scheduler but memory allocations are - * not allowed. - */ -#define MAX_STATIC_MSG_TYPE 3 - struct xe_sched_msg static_msgs[MAX_STATIC_MSG_TYPE]; - /** @lr_tdr: long running TDR worker */ - struct work_struct lr_tdr; - /** @fini_async: do final fini async from this worker */ - struct work_struct fini_async; - /** @resume_time: time of last resume */ - u64 resume_time; - /** @state: GuC specific state for this xe_engine */ - atomic_t state; - /** @wqi_head: work queue item tail */ - u32 wqi_head; - /** @wqi_tail: work queue item tail */ - u32 wqi_tail; - /** @id: GuC id for this xe_engine */ - u16 id; - /** @suspend_wait: wait queue used to wait on pending suspends */ - wait_queue_head_t suspend_wait; - /** @suspend_pending: a suspend of the engine is pending */ - bool suspend_pending; -}; - -#endif diff --git a/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h new file mode 100644 index 000000000000..4c39f01e4f52 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_guc_exec_queue_types.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2022 Intel Corporation + */ + +#ifndef _XE_GUC_ENGINE_TYPES_H_ +#define _XE_GUC_ENGINE_TYPES_H_ + +#include +#include + +#include "xe_gpu_scheduler_types.h" + +struct dma_fence; +struct xe_exec_queue; + +/** + * struct xe_guc_exec_queue - GuC specific state for an xe_exec_queue + */ +struct xe_guc_exec_queue { + /** @q: Backpointer to parent xe_exec_queue */ + struct xe_exec_queue *q; + /** @sched: GPU scheduler for this xe_exec_queue */ + struct xe_gpu_scheduler sched; + /** @entity: Scheduler entity for this xe_exec_queue */ + struct xe_sched_entity entity; + /** + * @static_msgs: Static messages for this xe_exec_queue, used when + * a message needs to sent through the GPU scheduler but memory + * allocations are not allowed. + */ +#define MAX_STATIC_MSG_TYPE 3 + struct xe_sched_msg static_msgs[MAX_STATIC_MSG_TYPE]; + /** @lr_tdr: long running TDR worker */ + struct work_struct lr_tdr; + /** @fini_async: do final fini async from this worker */ + struct work_struct fini_async; + /** @resume_time: time of last resume */ + u64 resume_time; + /** @state: GuC specific state for this xe_exec_queue */ + atomic_t state; + /** @wqi_head: work queue item tail */ + u32 wqi_head; + /** @wqi_tail: work queue item tail */ + u32 wqi_tail; + /** @id: GuC id for this exec_queue */ + u16 id; + /** @suspend_wait: wait queue used to wait on pending suspends */ + wait_queue_head_t suspend_wait; + /** @suspend_pending: a suspend of the exec_queue is pending */ + bool suspend_pending; +}; + +#endif diff --git a/drivers/gpu/drm/xe/xe_guc_fwif.h b/drivers/gpu/drm/xe/xe_guc_fwif.h index 7515d7fbb723..4216a6d9e478 100644 --- a/drivers/gpu/drm/xe/xe_guc_fwif.h +++ b/drivers/gpu/drm/xe/xe_guc_fwif.h @@ -69,13 +69,13 @@ struct guc_klv_generic_dw_t { } __packed; /* Format of the UPDATE_CONTEXT_POLICIES H2G data packet */ -struct guc_update_engine_policy_header { +struct guc_update_exec_queue_policy_header { u32 action; u32 guc_id; } __packed; -struct guc_update_engine_policy { - struct guc_update_engine_policy_header header; +struct guc_update_exec_queue_policy { + struct guc_update_exec_queue_policy_header header; struct guc_klv_generic_dw_t klv[GUC_CONTEXT_POLICIES_KLV_NUM_IDS]; } __packed; diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index 5198e91eeefb..42454c12efb3 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -22,7 +22,7 @@ #include "xe_gt.h" #include "xe_guc.h" #include "xe_guc_ct.h" -#include "xe_guc_engine_types.h" +#include "xe_guc_exec_queue_types.h" #include "xe_guc_submit_types.h" #include "xe_hw_engine.h" #include "xe_hw_fence.h" @@ -48,9 +48,9 @@ guc_to_xe(struct xe_guc *guc) } static struct xe_guc * -engine_to_guc(struct xe_engine *e) +exec_queue_to_guc(struct xe_exec_queue *q) { - return &e->gt->uc.guc; + return &q->gt->uc.guc; } /* @@ -58,140 +58,140 @@ engine_to_guc(struct xe_engine *e) * as the same time (e.g. a suspend can be happning at the same time as schedule * engine done being processed). */ -#define ENGINE_STATE_REGISTERED (1 << 0) +#define EXEC_QUEUE_STATE_REGISTERED (1 << 0) #define ENGINE_STATE_ENABLED (1 << 1) -#define ENGINE_STATE_PENDING_ENABLE (1 << 2) -#define ENGINE_STATE_PENDING_DISABLE (1 << 3) -#define ENGINE_STATE_DESTROYED (1 << 4) +#define EXEC_QUEUE_STATE_PENDING_ENABLE (1 << 2) +#define EXEC_QUEUE_STATE_PENDING_DISABLE (1 << 3) +#define EXEC_QUEUE_STATE_DESTROYED (1 << 4) #define ENGINE_STATE_SUSPENDED (1 << 5) -#define ENGINE_STATE_RESET (1 << 6) +#define EXEC_QUEUE_STATE_RESET (1 << 6) #define ENGINE_STATE_KILLED (1 << 7) -static bool engine_registered(struct xe_engine *e) +static bool exec_queue_registered(struct xe_exec_queue *q) { - return atomic_read(&e->guc->state) & ENGINE_STATE_REGISTERED; + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_REGISTERED; } -static void set_engine_registered(struct xe_engine *e) +static void set_exec_queue_registered(struct xe_exec_queue *q) { - atomic_or(ENGINE_STATE_REGISTERED, &e->guc->state); + atomic_or(EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); } -static void clear_engine_registered(struct xe_engine *e) +static void clear_exec_queue_registered(struct xe_exec_queue *q) { - atomic_and(~ENGINE_STATE_REGISTERED, &e->guc->state); + atomic_and(~EXEC_QUEUE_STATE_REGISTERED, &q->guc->state); } -static bool engine_enabled(struct xe_engine *e) +static bool exec_queue_enabled(struct xe_exec_queue *q) { - return atomic_read(&e->guc->state) & ENGINE_STATE_ENABLED; + return atomic_read(&q->guc->state) & ENGINE_STATE_ENABLED; } -static void set_engine_enabled(struct xe_engine *e) +static void set_exec_queue_enabled(struct xe_exec_queue *q) { - atomic_or(ENGINE_STATE_ENABLED, &e->guc->state); + atomic_or(ENGINE_STATE_ENABLED, &q->guc->state); } -static void clear_engine_enabled(struct xe_engine *e) +static void clear_exec_queue_enabled(struct xe_exec_queue *q) { - atomic_and(~ENGINE_STATE_ENABLED, &e->guc->state); + atomic_and(~ENGINE_STATE_ENABLED, &q->guc->state); } -static bool engine_pending_enable(struct xe_engine *e) +static bool exec_queue_pending_enable(struct xe_exec_queue *q) { - return atomic_read(&e->guc->state) & ENGINE_STATE_PENDING_ENABLE; + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_ENABLE; } -static void set_engine_pending_enable(struct xe_engine *e) +static void set_exec_queue_pending_enable(struct xe_exec_queue *q) { - atomic_or(ENGINE_STATE_PENDING_ENABLE, &e->guc->state); + atomic_or(EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); } -static void clear_engine_pending_enable(struct xe_engine *e) +static void clear_exec_queue_pending_enable(struct xe_exec_queue *q) { - atomic_and(~ENGINE_STATE_PENDING_ENABLE, &e->guc->state); + atomic_and(~EXEC_QUEUE_STATE_PENDING_ENABLE, &q->guc->state); } -static bool engine_pending_disable(struct xe_engine *e) +static bool exec_queue_pending_disable(struct xe_exec_queue *q) { - return atomic_read(&e->guc->state) & ENGINE_STATE_PENDING_DISABLE; + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_PENDING_DISABLE; } -static void set_engine_pending_disable(struct xe_engine *e) +static void set_exec_queue_pending_disable(struct xe_exec_queue *q) { - atomic_or(ENGINE_STATE_PENDING_DISABLE, &e->guc->state); + atomic_or(EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); } -static void clear_engine_pending_disable(struct xe_engine *e) +static void clear_exec_queue_pending_disable(struct xe_exec_queue *q) { - atomic_and(~ENGINE_STATE_PENDING_DISABLE, &e->guc->state); + atomic_and(~EXEC_QUEUE_STATE_PENDING_DISABLE, &q->guc->state); } -static bool engine_destroyed(struct xe_engine *e) +static bool exec_queue_destroyed(struct xe_exec_queue *q) { - return atomic_read(&e->guc->state) & ENGINE_STATE_DESTROYED; + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_DESTROYED; } -static void set_engine_destroyed(struct xe_engine *e) +static void set_exec_queue_destroyed(struct xe_exec_queue *q) { - atomic_or(ENGINE_STATE_DESTROYED, &e->guc->state); + atomic_or(EXEC_QUEUE_STATE_DESTROYED, &q->guc->state); } -static bool engine_banned(struct xe_engine *e) +static bool exec_queue_banned(struct xe_exec_queue *q) { - return (e->flags & ENGINE_FLAG_BANNED); + return (q->flags & EXEC_QUEUE_FLAG_BANNED); } -static void set_engine_banned(struct xe_engine *e) +static void set_exec_queue_banned(struct xe_exec_queue *q) { - e->flags |= ENGINE_FLAG_BANNED; + q->flags |= EXEC_QUEUE_FLAG_BANNED; } -static bool engine_suspended(struct xe_engine *e) +static bool exec_queue_suspended(struct xe_exec_queue *q) { - return atomic_read(&e->guc->state) & ENGINE_STATE_SUSPENDED; + return atomic_read(&q->guc->state) & ENGINE_STATE_SUSPENDED; } -static void set_engine_suspended(struct xe_engine *e) +static void set_exec_queue_suspended(struct xe_exec_queue *q) { - atomic_or(ENGINE_STATE_SUSPENDED, &e->guc->state); + atomic_or(ENGINE_STATE_SUSPENDED, &q->guc->state); } -static void clear_engine_suspended(struct xe_engine *e) +static void clear_exec_queue_suspended(struct xe_exec_queue *q) { - atomic_and(~ENGINE_STATE_SUSPENDED, &e->guc->state); + atomic_and(~ENGINE_STATE_SUSPENDED, &q->guc->state); } -static bool engine_reset(struct xe_engine *e) +static bool exec_queue_reset(struct xe_exec_queue *q) { - return atomic_read(&e->guc->state) & ENGINE_STATE_RESET; + return atomic_read(&q->guc->state) & EXEC_QUEUE_STATE_RESET; } -static void set_engine_reset(struct xe_engine *e) +static void set_exec_queue_reset(struct xe_exec_queue *q) { - atomic_or(ENGINE_STATE_RESET, &e->guc->state); + atomic_or(EXEC_QUEUE_STATE_RESET, &q->guc->state); } -static bool engine_killed(struct xe_engine *e) +static bool exec_queue_killed(struct xe_exec_queue *q) { - return atomic_read(&e->guc->state) & ENGINE_STATE_KILLED; + return atomic_read(&q->guc->state) & ENGINE_STATE_KILLED; } -static void set_engine_killed(struct xe_engine *e) +static void set_exec_queue_killed(struct xe_exec_queue *q) { - atomic_or(ENGINE_STATE_KILLED, &e->guc->state); + atomic_or(ENGINE_STATE_KILLED, &q->guc->state); } -static bool engine_killed_or_banned(struct xe_engine *e) +static bool exec_queue_killed_or_banned(struct xe_exec_queue *q) { - return engine_killed(e) || engine_banned(e); + return exec_queue_killed(q) || exec_queue_banned(q); } static void guc_submit_fini(struct drm_device *drm, void *arg) { struct xe_guc *guc = arg; - xa_destroy(&guc->submission_state.engine_lookup); + xa_destroy(&guc->submission_state.exec_queue_lookup); ida_destroy(&guc->submission_state.guc_ids); bitmap_free(guc->submission_state.guc_ids_bitmap); } @@ -201,7 +201,7 @@ static void guc_submit_fini(struct drm_device *drm, void *arg) #define GUC_ID_NUMBER_SLRC (GUC_ID_MAX - GUC_ID_NUMBER_MLRC) #define GUC_ID_START_MLRC GUC_ID_NUMBER_SLRC -static const struct xe_engine_ops guc_engine_ops; +static const struct xe_exec_queue_ops guc_exec_queue_ops; static void primelockdep(struct xe_guc *guc) { @@ -228,10 +228,10 @@ int xe_guc_submit_init(struct xe_guc *guc) if (!guc->submission_state.guc_ids_bitmap) return -ENOMEM; - gt->engine_ops = &guc_engine_ops; + gt->exec_queue_ops = &guc_exec_queue_ops; mutex_init(&guc->submission_state.lock); - xa_init(&guc->submission_state.engine_lookup); + xa_init(&guc->submission_state.exec_queue_lookup); ida_init(&guc->submission_state.guc_ids); spin_lock_init(&guc->submission_state.suspend.lock); @@ -246,7 +246,7 @@ int xe_guc_submit_init(struct xe_guc *guc) return 0; } -static int alloc_guc_id(struct xe_guc *guc, struct xe_engine *e) +static int alloc_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) { int ret; void *ptr; @@ -260,11 +260,11 @@ static int alloc_guc_id(struct xe_guc *guc, struct xe_engine *e) */ lockdep_assert_held(&guc->submission_state.lock); - if (xe_engine_is_parallel(e)) { + if (xe_exec_queue_is_parallel(q)) { void *bitmap = guc->submission_state.guc_ids_bitmap; ret = bitmap_find_free_region(bitmap, GUC_ID_NUMBER_MLRC, - order_base_2(e->width)); + order_base_2(q->width)); } else { ret = ida_simple_get(&guc->submission_state.guc_ids, 0, GUC_ID_NUMBER_SLRC, GFP_NOWAIT); @@ -272,12 +272,12 @@ static int alloc_guc_id(struct xe_guc *guc, struct xe_engine *e) if (ret < 0) return ret; - e->guc->id = ret; - if (xe_engine_is_parallel(e)) - e->guc->id += GUC_ID_START_MLRC; + q->guc->id = ret; + if (xe_exec_queue_is_parallel(q)) + q->guc->id += GUC_ID_START_MLRC; - ptr = xa_store(&guc->submission_state.engine_lookup, - e->guc->id, e, GFP_NOWAIT); + ptr = xa_store(&guc->submission_state.exec_queue_lookup, + q->guc->id, q, GFP_NOWAIT); if (IS_ERR(ptr)) { ret = PTR_ERR(ptr); goto err_release; @@ -286,29 +286,29 @@ static int alloc_guc_id(struct xe_guc *guc, struct xe_engine *e) return 0; err_release: - ida_simple_remove(&guc->submission_state.guc_ids, e->guc->id); + ida_simple_remove(&guc->submission_state.guc_ids, q->guc->id); return ret; } -static void release_guc_id(struct xe_guc *guc, struct xe_engine *e) +static void release_guc_id(struct xe_guc *guc, struct xe_exec_queue *q) { mutex_lock(&guc->submission_state.lock); - xa_erase(&guc->submission_state.engine_lookup, e->guc->id); - if (xe_engine_is_parallel(e)) + xa_erase(&guc->submission_state.exec_queue_lookup, q->guc->id); + if (xe_exec_queue_is_parallel(q)) bitmap_release_region(guc->submission_state.guc_ids_bitmap, - e->guc->id - GUC_ID_START_MLRC, - order_base_2(e->width)); + q->guc->id - GUC_ID_START_MLRC, + order_base_2(q->width)); else - ida_simple_remove(&guc->submission_state.guc_ids, e->guc->id); + ida_simple_remove(&guc->submission_state.guc_ids, q->guc->id); mutex_unlock(&guc->submission_state.lock); } -struct engine_policy { +struct exec_queue_policy { u32 count; - struct guc_update_engine_policy h2g; + struct guc_update_exec_queue_policy h2g; }; -static u32 __guc_engine_policy_action_size(struct engine_policy *policy) +static u32 __guc_exec_queue_policy_action_size(struct exec_queue_policy *policy) { size_t bytes = sizeof(policy->h2g.header) + (sizeof(policy->h2g.klv[0]) * policy->count); @@ -316,8 +316,8 @@ static u32 __guc_engine_policy_action_size(struct engine_policy *policy) return bytes / sizeof(u32); } -static void __guc_engine_policy_start_klv(struct engine_policy *policy, - u16 guc_id) +static void __guc_exec_queue_policy_start_klv(struct exec_queue_policy *policy, + u16 guc_id) { policy->h2g.header.action = XE_GUC_ACTION_HOST2GUC_UPDATE_CONTEXT_POLICIES; @@ -325,8 +325,8 @@ static void __guc_engine_policy_start_klv(struct engine_policy *policy, policy->count = 0; } -#define MAKE_ENGINE_POLICY_ADD(func, id) \ -static void __guc_engine_policy_add_##func(struct engine_policy *policy, \ +#define MAKE_EXEC_QUEUE_POLICY_ADD(func, id) \ +static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, \ u32 data) \ { \ XE_WARN_ON(policy->count >= GUC_CONTEXT_POLICIES_KLV_NUM_IDS); \ @@ -339,45 +339,45 @@ static void __guc_engine_policy_add_##func(struct engine_policy *policy, \ policy->count++; \ } -MAKE_ENGINE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) -MAKE_ENGINE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) -MAKE_ENGINE_POLICY_ADD(priority, SCHEDULING_PRIORITY) -#undef MAKE_ENGINE_POLICY_ADD +MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) +MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) +MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) +#undef MAKE_EXEC_QUEUE_POLICY_ADD -static const int xe_engine_prio_to_guc[] = { - [XE_ENGINE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, - [XE_ENGINE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, - [XE_ENGINE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, - [XE_ENGINE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, +static const int xe_exec_queue_prio_to_guc[] = { + [XE_EXEC_QUEUE_PRIORITY_LOW] = GUC_CLIENT_PRIORITY_NORMAL, + [XE_EXEC_QUEUE_PRIORITY_NORMAL] = GUC_CLIENT_PRIORITY_KMD_NORMAL, + [XE_EXEC_QUEUE_PRIORITY_HIGH] = GUC_CLIENT_PRIORITY_HIGH, + [XE_EXEC_QUEUE_PRIORITY_KERNEL] = GUC_CLIENT_PRIORITY_KMD_HIGH, }; -static void init_policies(struct xe_guc *guc, struct xe_engine *e) +static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) { - struct engine_policy policy; - enum xe_engine_priority prio = e->priority; - u32 timeslice_us = e->sched_props.timeslice_us; - u32 preempt_timeout_us = e->sched_props.preempt_timeout_us; + struct exec_queue_policy policy; + enum xe_exec_queue_priority prio = q->priority; + u32 timeslice_us = q->sched_props.timeslice_us; + u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; - XE_WARN_ON(!engine_registered(e)); + XE_WARN_ON(!exec_queue_registered(q)); - __guc_engine_policy_start_klv(&policy, e->guc->id); - __guc_engine_policy_add_priority(&policy, xe_engine_prio_to_guc[prio]); - __guc_engine_policy_add_execution_quantum(&policy, timeslice_us); - __guc_engine_policy_add_preemption_timeout(&policy, preempt_timeout_us); + __guc_exec_queue_policy_start_klv(&policy, q->guc->id); + __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); + __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); + __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, - __guc_engine_policy_action_size(&policy), 0, 0); + __guc_exec_queue_policy_action_size(&policy), 0, 0); } -static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_engine *e) +static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_exec_queue *q) { - struct engine_policy policy; + struct exec_queue_policy policy; - __guc_engine_policy_start_klv(&policy, e->guc->id); - __guc_engine_policy_add_preemption_timeout(&policy, 1); + __guc_exec_queue_policy_start_klv(&policy, q->guc->id); + __guc_exec_queue_policy_add_preemption_timeout(&policy, 1); xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, - __guc_engine_policy_action_size(&policy), 0, 0); + __guc_exec_queue_policy_action_size(&policy), 0, 0); } #define parallel_read(xe_, map_, field_) \ @@ -388,7 +388,7 @@ static void set_min_preemption_timeout(struct xe_guc *guc, struct xe_engine *e) field_, val_) static void __register_mlrc_engine(struct xe_guc *guc, - struct xe_engine *e, + struct xe_exec_queue *q, struct guc_ctxt_registration_info *info) { #define MAX_MLRC_REG_SIZE (13 + XE_HW_ENGINE_MAX_INSTANCE * 2) @@ -396,7 +396,7 @@ static void __register_mlrc_engine(struct xe_guc *guc, int len = 0; int i; - XE_WARN_ON(!xe_engine_is_parallel(e)); + XE_WARN_ON(!xe_exec_queue_is_parallel(q)); action[len++] = XE_GUC_ACTION_REGISTER_CONTEXT_MULTI_LRC; action[len++] = info->flags; @@ -408,12 +408,12 @@ static void __register_mlrc_engine(struct xe_guc *guc, action[len++] = info->wq_base_lo; action[len++] = info->wq_base_hi; action[len++] = info->wq_size; - action[len++] = e->width; + action[len++] = q->width; action[len++] = info->hwlrca_lo; action[len++] = info->hwlrca_hi; - for (i = 1; i < e->width; ++i) { - struct xe_lrc *lrc = e->lrc + i; + for (i = 1; i < q->width; ++i) { + struct xe_lrc *lrc = q->lrc + i; action[len++] = lower_32_bits(xe_lrc_descriptor(lrc)); action[len++] = upper_32_bits(xe_lrc_descriptor(lrc)); @@ -446,24 +446,24 @@ static void __register_engine(struct xe_guc *guc, xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); } -static void register_engine(struct xe_engine *e) +static void register_engine(struct xe_exec_queue *q) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); - struct xe_lrc *lrc = e->lrc; + struct xe_lrc *lrc = q->lrc; struct guc_ctxt_registration_info info; - XE_WARN_ON(engine_registered(e)); + XE_WARN_ON(exec_queue_registered(q)); memset(&info, 0, sizeof(info)); - info.context_idx = e->guc->id; - info.engine_class = xe_engine_class_to_guc_class(e->class); - info.engine_submit_mask = e->logical_mask; + info.context_idx = q->guc->id; + info.engine_class = xe_engine_class_to_guc_class(q->class); + info.engine_submit_mask = q->logical_mask; info.hwlrca_lo = lower_32_bits(xe_lrc_descriptor(lrc)); info.hwlrca_hi = upper_32_bits(xe_lrc_descriptor(lrc)); info.flags = CONTEXT_REGISTRATION_FLAG_KMD; - if (xe_engine_is_parallel(e)) { + if (xe_exec_queue_is_parallel(q)) { u32 ggtt_addr = xe_lrc_parallel_ggtt_addr(lrc); struct iosys_map map = xe_lrc_parallel_map(lrc); @@ -477,8 +477,8 @@ static void register_engine(struct xe_engine *e) offsetof(struct guc_submit_parallel_scratch, wq[0])); info.wq_size = WQ_SIZE; - e->guc->wqi_head = 0; - e->guc->wqi_tail = 0; + q->guc->wqi_head = 0; + q->guc->wqi_tail = 0; xe_map_memset(xe, &map, 0, 0, PARALLEL_SCRATCH_SIZE - WQ_SIZE); parallel_write(xe, map, wq_desc.wq_status, WQ_STATUS_ACTIVE); } @@ -488,38 +488,38 @@ static void register_engine(struct xe_engine *e) * the GuC as jobs signal immediately and can't destroy an engine if the * GuC has a reference to it. */ - if (xe_engine_is_lr(e)) - xe_engine_get(e); + if (xe_exec_queue_is_lr(q)) + xe_exec_queue_get(q); - set_engine_registered(e); - trace_xe_engine_register(e); - if (xe_engine_is_parallel(e)) - __register_mlrc_engine(guc, e, &info); + set_exec_queue_registered(q); + trace_xe_exec_queue_register(q); + if (xe_exec_queue_is_parallel(q)) + __register_mlrc_engine(guc, q, &info); else __register_engine(guc, &info); - init_policies(guc, e); + init_policies(guc, q); } -static u32 wq_space_until_wrap(struct xe_engine *e) +static u32 wq_space_until_wrap(struct xe_exec_queue *q) { - return (WQ_SIZE - e->guc->wqi_tail); + return (WQ_SIZE - q->guc->wqi_tail); } -static int wq_wait_for_space(struct xe_engine *e, u32 wqi_size) +static int wq_wait_for_space(struct xe_exec_queue *q, u32 wqi_size) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(e->lrc); + struct iosys_map map = xe_lrc_parallel_map(q->lrc); unsigned int sleep_period_ms = 1; #define AVAILABLE_SPACE \ - CIRC_SPACE(e->guc->wqi_tail, e->guc->wqi_head, WQ_SIZE) + CIRC_SPACE(q->guc->wqi_tail, q->guc->wqi_head, WQ_SIZE) if (wqi_size > AVAILABLE_SPACE) { try_again: - e->guc->wqi_head = parallel_read(xe, map, wq_desc.head); + q->guc->wqi_head = parallel_read(xe, map, wq_desc.head); if (wqi_size > AVAILABLE_SPACE) { if (sleep_period_ms == 1024) { - xe_gt_reset_async(e->gt); + xe_gt_reset_async(q->gt); return -ENODEV; } @@ -533,52 +533,52 @@ try_again: return 0; } -static int wq_noop_append(struct xe_engine *e) +static int wq_noop_append(struct xe_exec_queue *q) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(e->lrc); - u32 len_dw = wq_space_until_wrap(e) / sizeof(u32) - 1; + struct iosys_map map = xe_lrc_parallel_map(q->lrc); + u32 len_dw = wq_space_until_wrap(q) / sizeof(u32) - 1; - if (wq_wait_for_space(e, wq_space_until_wrap(e))) + if (wq_wait_for_space(q, wq_space_until_wrap(q))) return -ENODEV; XE_WARN_ON(!FIELD_FIT(WQ_LEN_MASK, len_dw)); - parallel_write(xe, map, wq[e->guc->wqi_tail / sizeof(u32)], + parallel_write(xe, map, wq[q->guc->wqi_tail / sizeof(u32)], FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | FIELD_PREP(WQ_LEN_MASK, len_dw)); - e->guc->wqi_tail = 0; + q->guc->wqi_tail = 0; return 0; } -static void wq_item_append(struct xe_engine *e) +static void wq_item_append(struct xe_exec_queue *q) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(e->lrc); + struct iosys_map map = xe_lrc_parallel_map(q->lrc); #define WQ_HEADER_SIZE 4 /* Includes 1 LRC address too */ u32 wqi[XE_HW_ENGINE_MAX_INSTANCE + (WQ_HEADER_SIZE - 1)]; - u32 wqi_size = (e->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); + u32 wqi_size = (q->width + (WQ_HEADER_SIZE - 1)) * sizeof(u32); u32 len_dw = (wqi_size / sizeof(u32)) - 1; int i = 0, j; - if (wqi_size > wq_space_until_wrap(e)) { - if (wq_noop_append(e)) + if (wqi_size > wq_space_until_wrap(q)) { + if (wq_noop_append(q)) return; } - if (wq_wait_for_space(e, wqi_size)) + if (wq_wait_for_space(q, wqi_size)) return; wqi[i++] = FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_MULTI_LRC) | FIELD_PREP(WQ_LEN_MASK, len_dw); - wqi[i++] = xe_lrc_descriptor(e->lrc); - wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, e->guc->id) | - FIELD_PREP(WQ_RING_TAIL_MASK, e->lrc->ring.tail / sizeof(u64)); + wqi[i++] = xe_lrc_descriptor(q->lrc); + wqi[i++] = FIELD_PREP(WQ_GUC_ID_MASK, q->guc->id) | + FIELD_PREP(WQ_RING_TAIL_MASK, q->lrc->ring.tail / sizeof(u64)); wqi[i++] = 0; - for (j = 1; j < e->width; ++j) { - struct xe_lrc *lrc = e->lrc + j; + for (j = 1; j < q->width; ++j) { + struct xe_lrc *lrc = q->lrc + j; wqi[i++] = lrc->ring.tail / sizeof(u64); } @@ -586,55 +586,55 @@ static void wq_item_append(struct xe_engine *e) XE_WARN_ON(i != wqi_size / sizeof(u32)); iosys_map_incr(&map, offsetof(struct guc_submit_parallel_scratch, - wq[e->guc->wqi_tail / sizeof(u32)])); + wq[q->guc->wqi_tail / sizeof(u32)])); xe_map_memcpy_to(xe, &map, 0, wqi, wqi_size); - e->guc->wqi_tail += wqi_size; - XE_WARN_ON(e->guc->wqi_tail > WQ_SIZE); + q->guc->wqi_tail += wqi_size; + XE_WARN_ON(q->guc->wqi_tail > WQ_SIZE); xe_device_wmb(xe); - map = xe_lrc_parallel_map(e->lrc); - parallel_write(xe, map, wq_desc.tail, e->guc->wqi_tail); + map = xe_lrc_parallel_map(q->lrc); + parallel_write(xe, map, wq_desc.tail, q->guc->wqi_tail); } #define RESUME_PENDING ~0x0ull -static void submit_engine(struct xe_engine *e) +static void submit_exec_queue(struct xe_exec_queue *q) { - struct xe_guc *guc = engine_to_guc(e); - struct xe_lrc *lrc = e->lrc; + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_lrc *lrc = q->lrc; u32 action[3]; u32 g2h_len = 0; u32 num_g2h = 0; int len = 0; bool extra_submit = false; - XE_WARN_ON(!engine_registered(e)); + XE_WARN_ON(!exec_queue_registered(q)); - if (xe_engine_is_parallel(e)) - wq_item_append(e); + if (xe_exec_queue_is_parallel(q)) + wq_item_append(q); else xe_lrc_write_ctx_reg(lrc, CTX_RING_TAIL, lrc->ring.tail); - if (engine_suspended(e) && !xe_engine_is_parallel(e)) + if (exec_queue_suspended(q) && !xe_exec_queue_is_parallel(q)) return; - if (!engine_enabled(e) && !engine_suspended(e)) { + if (!exec_queue_enabled(q) && !exec_queue_suspended(q)) { action[len++] = XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET; - action[len++] = e->guc->id; + action[len++] = q->guc->id; action[len++] = GUC_CONTEXT_ENABLE; g2h_len = G2H_LEN_DW_SCHED_CONTEXT_MODE_SET; num_g2h = 1; - if (xe_engine_is_parallel(e)) + if (xe_exec_queue_is_parallel(q)) extra_submit = true; - e->guc->resume_time = RESUME_PENDING; - set_engine_pending_enable(e); - set_engine_enabled(e); - trace_xe_engine_scheduling_enable(e); + q->guc->resume_time = RESUME_PENDING; + set_exec_queue_pending_enable(q); + set_exec_queue_enabled(q); + trace_xe_exec_queue_scheduling_enable(q); } else { action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; - action[len++] = e->guc->id; - trace_xe_engine_submit(e); + action[len++] = q->guc->id; + trace_xe_exec_queue_submit(q); } xe_guc_ct_send(&guc->ct, action, len, g2h_len, num_g2h); @@ -642,31 +642,31 @@ static void submit_engine(struct xe_engine *e) if (extra_submit) { len = 0; action[len++] = XE_GUC_ACTION_SCHED_CONTEXT; - action[len++] = e->guc->id; - trace_xe_engine_submit(e); + action[len++] = q->guc->id; + trace_xe_exec_queue_submit(q); xe_guc_ct_send(&guc->ct, action, len, 0, 0); } } static struct dma_fence * -guc_engine_run_job(struct drm_sched_job *drm_job) +guc_exec_queue_run_job(struct drm_sched_job *drm_job) { struct xe_sched_job *job = to_xe_sched_job(drm_job); - struct xe_engine *e = job->engine; - bool lr = xe_engine_is_lr(e); + struct xe_exec_queue *q = job->q; + bool lr = xe_exec_queue_is_lr(q); - XE_WARN_ON((engine_destroyed(e) || engine_pending_disable(e)) && - !engine_banned(e) && !engine_suspended(e)); + XE_WARN_ON((exec_queue_destroyed(q) || exec_queue_pending_disable(q)) && + !exec_queue_banned(q) && !exec_queue_suspended(q)); trace_xe_sched_job_run(job); - if (!engine_killed_or_banned(e) && !xe_sched_job_is_error(job)) { - if (!engine_registered(e)) - register_engine(e); + if (!exec_queue_killed_or_banned(q) && !xe_sched_job_is_error(job)) { + if (!exec_queue_registered(q)) + register_engine(q); if (!lr) /* LR jobs are emitted in the exec IOCTL */ - e->ring_ops->emit_job(job); - submit_engine(e); + q->ring_ops->emit_job(job); + submit_exec_queue(q); } if (lr) { @@ -679,7 +679,7 @@ guc_engine_run_job(struct drm_sched_job *drm_job) } } -static void guc_engine_free_job(struct drm_sched_job *drm_job) +static void guc_exec_queue_free_job(struct drm_sched_job *drm_job) { struct xe_sched_job *job = to_xe_sched_job(drm_job); @@ -692,37 +692,37 @@ static int guc_read_stopped(struct xe_guc *guc) return atomic_read(&guc->submission_state.stopped); } -#define MAKE_SCHED_CONTEXT_ACTION(e, enable_disable) \ +#define MAKE_SCHED_CONTEXT_ACTION(q, enable_disable) \ u32 action[] = { \ XE_GUC_ACTION_SCHED_CONTEXT_MODE_SET, \ - e->guc->id, \ + q->guc->id, \ GUC_CONTEXT_##enable_disable, \ } static void disable_scheduling_deregister(struct xe_guc *guc, - struct xe_engine *e) + struct xe_exec_queue *q) { - MAKE_SCHED_CONTEXT_ACTION(e, DISABLE); + MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); int ret; - set_min_preemption_timeout(guc, e); + set_min_preemption_timeout(guc, q); smp_rmb(); - ret = wait_event_timeout(guc->ct.wq, !engine_pending_enable(e) || + ret = wait_event_timeout(guc->ct.wq, !exec_queue_pending_enable(q) || guc_read_stopped(guc), HZ * 5); if (!ret) { - struct xe_gpu_scheduler *sched = &e->guc->sched; + struct xe_gpu_scheduler *sched = &q->guc->sched; XE_WARN_ON("Pending enable failed to respond"); xe_sched_submission_start(sched); - xe_gt_reset_async(e->gt); + xe_gt_reset_async(q->gt); xe_sched_tdr_queue_imm(sched); return; } - clear_engine_enabled(e); - set_engine_pending_disable(e); - set_engine_destroyed(e); - trace_xe_engine_scheduling_disable(e); + clear_exec_queue_enabled(q); + set_exec_queue_pending_disable(q); + set_exec_queue_destroyed(q); + trace_xe_exec_queue_scheduling_disable(q); /* * Reserve space for both G2H here as the 2nd G2H is sent from a G2H @@ -733,27 +733,27 @@ static void disable_scheduling_deregister(struct xe_guc *guc, G2H_LEN_DW_DEREGISTER_CONTEXT, 2); } -static void guc_engine_print(struct xe_engine *e, struct drm_printer *p); +static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p); #if IS_ENABLED(CONFIG_DRM_XE_SIMPLE_ERROR_CAPTURE) -static void simple_error_capture(struct xe_engine *e) +static void simple_error_capture(struct xe_exec_queue *q) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); struct drm_printer p = drm_err_printer(""); struct xe_hw_engine *hwe; enum xe_hw_engine_id id; - u32 adj_logical_mask = e->logical_mask; - u32 width_mask = (0x1 << e->width) - 1; + u32 adj_logical_mask = q->logical_mask; + u32 width_mask = (0x1 << q->width) - 1; int i; bool cookie; - if (e->vm && !e->vm->error_capture.capture_once) { - e->vm->error_capture.capture_once = true; + if (q->vm && !q->vm->error_capture.capture_once) { + q->vm->error_capture.capture_once = true; cookie = dma_fence_begin_signalling(); - for (i = 0; e->width > 1 && i < XE_HW_ENGINE_MAX_INSTANCE;) { + for (i = 0; q->width > 1 && i < XE_HW_ENGINE_MAX_INSTANCE;) { if (adj_logical_mask & BIT(i)) { adj_logical_mask |= width_mask << i; - i += e->width; + i += q->width; } else { ++i; } @@ -761,66 +761,66 @@ static void simple_error_capture(struct xe_engine *e) xe_force_wake_get(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL); xe_guc_ct_print(&guc->ct, &p, true); - guc_engine_print(e, &p); + guc_exec_queue_print(q, &p); for_each_hw_engine(hwe, guc_to_gt(guc), id) { - if (hwe->class != e->hwe->class || + if (hwe->class != q->hwe->class || !(BIT(hwe->logical_instance) & adj_logical_mask)) continue; xe_hw_engine_print(hwe, &p); } - xe_analyze_vm(&p, e->vm, e->gt->info.id); + xe_analyze_vm(&p, q->vm, q->gt->info.id); xe_force_wake_put(gt_to_fw(guc_to_gt(guc)), XE_FORCEWAKE_ALL); dma_fence_end_signalling(cookie); } } #else -static void simple_error_capture(struct xe_engine *e) +static void simple_error_capture(struct xe_exec_queue *q) { } #endif -static void xe_guc_engine_trigger_cleanup(struct xe_engine *e) +static void xe_guc_exec_queue_trigger_cleanup(struct xe_exec_queue *q) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); - if (xe_engine_is_lr(e)) - queue_work(guc_to_gt(guc)->ordered_wq, &e->guc->lr_tdr); + if (xe_exec_queue_is_lr(q)) + queue_work(guc_to_gt(guc)->ordered_wq, &q->guc->lr_tdr); else - xe_sched_tdr_queue_imm(&e->guc->sched); + xe_sched_tdr_queue_imm(&q->guc->sched); } -static void xe_guc_engine_lr_cleanup(struct work_struct *w) +static void xe_guc_exec_queue_lr_cleanup(struct work_struct *w) { - struct xe_guc_engine *ge = - container_of(w, struct xe_guc_engine, lr_tdr); - struct xe_engine *e = ge->engine; + struct xe_guc_exec_queue *ge = + container_of(w, struct xe_guc_exec_queue, lr_tdr); + struct xe_exec_queue *q = ge->q; struct xe_gpu_scheduler *sched = &ge->sched; - XE_WARN_ON(!xe_engine_is_lr(e)); - trace_xe_engine_lr_cleanup(e); + XE_WARN_ON(!xe_exec_queue_is_lr(q)); + trace_xe_exec_queue_lr_cleanup(q); /* Kill the run_job / process_msg entry points */ xe_sched_submission_stop(sched); /* Engine state now stable, disable scheduling / deregister if needed */ - if (engine_registered(e)) { - struct xe_guc *guc = engine_to_guc(e); + if (exec_queue_registered(q)) { + struct xe_guc *guc = exec_queue_to_guc(q); int ret; - set_engine_banned(e); - disable_scheduling_deregister(guc, e); + set_exec_queue_banned(q); + disable_scheduling_deregister(guc, q); /* * Must wait for scheduling to be disabled before signalling * any fences, if GT broken the GT reset code should signal us. */ ret = wait_event_timeout(guc->ct.wq, - !engine_pending_disable(e) || + !exec_queue_pending_disable(q) || guc_read_stopped(guc), HZ * 5); if (!ret) { XE_WARN_ON("Schedule disable failed to respond"); xe_sched_submission_start(sched); - xe_gt_reset_async(e->gt); + xe_gt_reset_async(q->gt); return; } } @@ -829,27 +829,27 @@ static void xe_guc_engine_lr_cleanup(struct work_struct *w) } static enum drm_gpu_sched_stat -guc_engine_timedout_job(struct drm_sched_job *drm_job) +guc_exec_queue_timedout_job(struct drm_sched_job *drm_job) { struct xe_sched_job *job = to_xe_sched_job(drm_job); struct xe_sched_job *tmp_job; - struct xe_engine *e = job->engine; - struct xe_gpu_scheduler *sched = &e->guc->sched; - struct xe_device *xe = guc_to_xe(engine_to_guc(e)); + struct xe_exec_queue *q = job->q; + struct xe_gpu_scheduler *sched = &q->guc->sched; + struct xe_device *xe = guc_to_xe(exec_queue_to_guc(q)); int err = -ETIME; int i = 0; if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &job->fence->flags)) { - XE_WARN_ON(e->flags & ENGINE_FLAG_KERNEL); - XE_WARN_ON(e->flags & ENGINE_FLAG_VM && !engine_killed(e)); + XE_WARN_ON(q->flags & EXEC_QUEUE_FLAG_KERNEL); + XE_WARN_ON(q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q)); drm_notice(&xe->drm, "Timedout job: seqno=%u, guc_id=%d, flags=0x%lx", - xe_sched_job_seqno(job), e->guc->id, e->flags); - simple_error_capture(e); - xe_devcoredump(e); + xe_sched_job_seqno(job), q->guc->id, q->flags); + simple_error_capture(q); + xe_devcoredump(q); } else { drm_dbg(&xe->drm, "Timedout signaled job: seqno=%u, guc_id=%d, flags=0x%lx", - xe_sched_job_seqno(job), e->guc->id, e->flags); + xe_sched_job_seqno(job), q->guc->id, q->flags); } trace_xe_sched_job_timedout(job); @@ -860,26 +860,26 @@ guc_engine_timedout_job(struct drm_sched_job *drm_job) * Kernel jobs should never fail, nor should VM jobs if they do * somethings has gone wrong and the GT needs a reset */ - if (e->flags & ENGINE_FLAG_KERNEL || - (e->flags & ENGINE_FLAG_VM && !engine_killed(e))) { + if (q->flags & EXEC_QUEUE_FLAG_KERNEL || + (q->flags & EXEC_QUEUE_FLAG_VM && !exec_queue_killed(q))) { if (!xe_sched_invalidate_job(job, 2)) { xe_sched_add_pending_job(sched, job); xe_sched_submission_start(sched); - xe_gt_reset_async(e->gt); + xe_gt_reset_async(q->gt); goto out; } } /* Engine state now stable, disable scheduling if needed */ - if (engine_enabled(e)) { - struct xe_guc *guc = engine_to_guc(e); + if (exec_queue_enabled(q)) { + struct xe_guc *guc = exec_queue_to_guc(q); int ret; - if (engine_reset(e)) + if (exec_queue_reset(q)) err = -EIO; - set_engine_banned(e); - xe_engine_get(e); - disable_scheduling_deregister(guc, e); + set_exec_queue_banned(q); + xe_exec_queue_get(q); + disable_scheduling_deregister(guc, q); /* * Must wait for scheduling to be disabled before signalling @@ -891,20 +891,20 @@ guc_engine_timedout_job(struct drm_sched_job *drm_job) */ smp_rmb(); ret = wait_event_timeout(guc->ct.wq, - !engine_pending_disable(e) || + !exec_queue_pending_disable(q) || guc_read_stopped(guc), HZ * 5); if (!ret) { XE_WARN_ON("Schedule disable failed to respond"); xe_sched_add_pending_job(sched, job); xe_sched_submission_start(sched); - xe_gt_reset_async(e->gt); + xe_gt_reset_async(q->gt); xe_sched_tdr_queue_imm(sched); goto out; } } /* Stop fence signaling */ - xe_hw_fence_irq_stop(e->fence_irq); + xe_hw_fence_irq_stop(q->fence_irq); /* * Fence state now stable, stop / start scheduler which cleans up any @@ -912,7 +912,7 @@ guc_engine_timedout_job(struct drm_sched_job *drm_job) */ xe_sched_add_pending_job(sched, job); xe_sched_submission_start(sched); - xe_guc_engine_trigger_cleanup(e); + xe_guc_exec_queue_trigger_cleanup(q); /* Mark all outstanding jobs as bad, thus completing them */ spin_lock(&sched->base.job_list_lock); @@ -921,53 +921,53 @@ guc_engine_timedout_job(struct drm_sched_job *drm_job) spin_unlock(&sched->base.job_list_lock); /* Start fence signaling */ - xe_hw_fence_irq_start(e->fence_irq); + xe_hw_fence_irq_start(q->fence_irq); out: return DRM_GPU_SCHED_STAT_NOMINAL; } -static void __guc_engine_fini_async(struct work_struct *w) +static void __guc_exec_queue_fini_async(struct work_struct *w) { - struct xe_guc_engine *ge = - container_of(w, struct xe_guc_engine, fini_async); - struct xe_engine *e = ge->engine; - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc_exec_queue *ge = + container_of(w, struct xe_guc_exec_queue, fini_async); + struct xe_exec_queue *q = ge->q; + struct xe_guc *guc = exec_queue_to_guc(q); - trace_xe_engine_destroy(e); + trace_xe_exec_queue_destroy(q); - if (xe_engine_is_lr(e)) + if (xe_exec_queue_is_lr(q)) cancel_work_sync(&ge->lr_tdr); - if (e->flags & ENGINE_FLAG_PERSISTENT) - xe_device_remove_persistent_engines(gt_to_xe(e->gt), e); - release_guc_id(guc, e); + if (q->flags & EXEC_QUEUE_FLAG_PERSISTENT) + xe_device_remove_persistent_exec_queues(gt_to_xe(q->gt), q); + release_guc_id(guc, q); xe_sched_entity_fini(&ge->entity); xe_sched_fini(&ge->sched); - if (!(e->flags & ENGINE_FLAG_KERNEL)) { + if (!(q->flags & EXEC_QUEUE_FLAG_KERNEL)) { kfree(ge); - xe_engine_fini(e); + xe_exec_queue_fini(q); } } -static void guc_engine_fini_async(struct xe_engine *e) +static void guc_exec_queue_fini_async(struct xe_exec_queue *q) { - bool kernel = e->flags & ENGINE_FLAG_KERNEL; + bool kernel = q->flags & EXEC_QUEUE_FLAG_KERNEL; - INIT_WORK(&e->guc->fini_async, __guc_engine_fini_async); - queue_work(system_wq, &e->guc->fini_async); + INIT_WORK(&q->guc->fini_async, __guc_exec_queue_fini_async); + queue_work(system_wq, &q->guc->fini_async); /* We must block on kernel engines so slabs are empty on driver unload */ if (kernel) { - struct xe_guc_engine *ge = e->guc; + struct xe_guc_exec_queue *ge = q->guc; flush_work(&ge->fini_async); kfree(ge); - xe_engine_fini(e); + xe_exec_queue_fini(q); } } -static void __guc_engine_fini(struct xe_guc *guc, struct xe_engine *e) +static void __guc_exec_queue_fini(struct xe_guc *guc, struct xe_exec_queue *q) { /* * Might be done from within the GPU scheduler, need to do async as we @@ -976,104 +976,104 @@ static void __guc_engine_fini(struct xe_guc *guc, struct xe_engine *e) * this we and don't really care when everything is fini'd, just that it * is. */ - guc_engine_fini_async(e); + guc_exec_queue_fini_async(q); } -static void __guc_engine_process_msg_cleanup(struct xe_sched_msg *msg) +static void __guc_exec_queue_process_msg_cleanup(struct xe_sched_msg *msg) { - struct xe_engine *e = msg->private_data; - struct xe_guc *guc = engine_to_guc(e); + struct xe_exec_queue *q = msg->private_data; + struct xe_guc *guc = exec_queue_to_guc(q); - XE_WARN_ON(e->flags & ENGINE_FLAG_KERNEL); - trace_xe_engine_cleanup_entity(e); + XE_WARN_ON(q->flags & EXEC_QUEUE_FLAG_KERNEL); + trace_xe_exec_queue_cleanup_entity(q); - if (engine_registered(e)) - disable_scheduling_deregister(guc, e); + if (exec_queue_registered(q)) + disable_scheduling_deregister(guc, q); else - __guc_engine_fini(guc, e); + __guc_exec_queue_fini(guc, q); } -static bool guc_engine_allowed_to_change_state(struct xe_engine *e) +static bool guc_exec_queue_allowed_to_change_state(struct xe_exec_queue *q) { - return !engine_killed_or_banned(e) && engine_registered(e); + return !exec_queue_killed_or_banned(q) && exec_queue_registered(q); } -static void __guc_engine_process_msg_set_sched_props(struct xe_sched_msg *msg) +static void __guc_exec_queue_process_msg_set_sched_props(struct xe_sched_msg *msg) { - struct xe_engine *e = msg->private_data; - struct xe_guc *guc = engine_to_guc(e); + struct xe_exec_queue *q = msg->private_data; + struct xe_guc *guc = exec_queue_to_guc(q); - if (guc_engine_allowed_to_change_state(e)) - init_policies(guc, e); + if (guc_exec_queue_allowed_to_change_state(q)) + init_policies(guc, q); kfree(msg); } -static void suspend_fence_signal(struct xe_engine *e) +static void suspend_fence_signal(struct xe_exec_queue *q) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); - XE_WARN_ON(!engine_suspended(e) && !engine_killed(e) && + XE_WARN_ON(!exec_queue_suspended(q) && !exec_queue_killed(q) && !guc_read_stopped(guc)); - XE_WARN_ON(!e->guc->suspend_pending); + XE_WARN_ON(!q->guc->suspend_pending); - e->guc->suspend_pending = false; + q->guc->suspend_pending = false; smp_wmb(); - wake_up(&e->guc->suspend_wait); + wake_up(&q->guc->suspend_wait); } -static void __guc_engine_process_msg_suspend(struct xe_sched_msg *msg) +static void __guc_exec_queue_process_msg_suspend(struct xe_sched_msg *msg) { - struct xe_engine *e = msg->private_data; - struct xe_guc *guc = engine_to_guc(e); + struct xe_exec_queue *q = msg->private_data; + struct xe_guc *guc = exec_queue_to_guc(q); - if (guc_engine_allowed_to_change_state(e) && !engine_suspended(e) && - engine_enabled(e)) { - wait_event(guc->ct.wq, e->guc->resume_time != RESUME_PENDING || + if (guc_exec_queue_allowed_to_change_state(q) && !exec_queue_suspended(q) && + exec_queue_enabled(q)) { + wait_event(guc->ct.wq, q->guc->resume_time != RESUME_PENDING || guc_read_stopped(guc)); if (!guc_read_stopped(guc)) { - MAKE_SCHED_CONTEXT_ACTION(e, DISABLE); + MAKE_SCHED_CONTEXT_ACTION(q, DISABLE); s64 since_resume_ms = ktime_ms_delta(ktime_get(), - e->guc->resume_time); - s64 wait_ms = e->vm->preempt.min_run_period_ms - + q->guc->resume_time); + s64 wait_ms = q->vm->preempt.min_run_period_ms - since_resume_ms; - if (wait_ms > 0 && e->guc->resume_time) + if (wait_ms > 0 && q->guc->resume_time) msleep(wait_ms); - set_engine_suspended(e); - clear_engine_enabled(e); - set_engine_pending_disable(e); - trace_xe_engine_scheduling_disable(e); + set_exec_queue_suspended(q); + clear_exec_queue_enabled(q); + set_exec_queue_pending_disable(q); + trace_xe_exec_queue_scheduling_disable(q); xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); } - } else if (e->guc->suspend_pending) { - set_engine_suspended(e); - suspend_fence_signal(e); + } else if (q->guc->suspend_pending) { + set_exec_queue_suspended(q); + suspend_fence_signal(q); } } -static void __guc_engine_process_msg_resume(struct xe_sched_msg *msg) +static void __guc_exec_queue_process_msg_resume(struct xe_sched_msg *msg) { - struct xe_engine *e = msg->private_data; - struct xe_guc *guc = engine_to_guc(e); + struct xe_exec_queue *q = msg->private_data; + struct xe_guc *guc = exec_queue_to_guc(q); - if (guc_engine_allowed_to_change_state(e)) { - MAKE_SCHED_CONTEXT_ACTION(e, ENABLE); + if (guc_exec_queue_allowed_to_change_state(q)) { + MAKE_SCHED_CONTEXT_ACTION(q, ENABLE); - e->guc->resume_time = RESUME_PENDING; - clear_engine_suspended(e); - set_engine_pending_enable(e); - set_engine_enabled(e); - trace_xe_engine_scheduling_enable(e); + q->guc->resume_time = RESUME_PENDING; + clear_exec_queue_suspended(q); + set_exec_queue_pending_enable(q); + set_exec_queue_enabled(q); + trace_xe_exec_queue_scheduling_enable(q); xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), G2H_LEN_DW_SCHED_CONTEXT_MODE_SET, 1); } else { - clear_engine_suspended(e); + clear_exec_queue_suspended(q); } } @@ -1082,22 +1082,22 @@ static void __guc_engine_process_msg_resume(struct xe_sched_msg *msg) #define SUSPEND 3 #define RESUME 4 -static void guc_engine_process_msg(struct xe_sched_msg *msg) +static void guc_exec_queue_process_msg(struct xe_sched_msg *msg) { trace_xe_sched_msg_recv(msg); switch (msg->opcode) { case CLEANUP: - __guc_engine_process_msg_cleanup(msg); + __guc_exec_queue_process_msg_cleanup(msg); break; case SET_SCHED_PROPS: - __guc_engine_process_msg_set_sched_props(msg); + __guc_exec_queue_process_msg_set_sched_props(msg); break; case SUSPEND: - __guc_engine_process_msg_suspend(msg); + __guc_exec_queue_process_msg_suspend(msg); break; case RESUME: - __guc_engine_process_msg_resume(msg); + __guc_exec_queue_process_msg_resume(msg); break; default: XE_WARN_ON("Unknown message type"); @@ -1105,20 +1105,20 @@ static void guc_engine_process_msg(struct xe_sched_msg *msg) } static const struct drm_sched_backend_ops drm_sched_ops = { - .run_job = guc_engine_run_job, - .free_job = guc_engine_free_job, - .timedout_job = guc_engine_timedout_job, + .run_job = guc_exec_queue_run_job, + .free_job = guc_exec_queue_free_job, + .timedout_job = guc_exec_queue_timedout_job, }; static const struct xe_sched_backend_ops xe_sched_ops = { - .process_msg = guc_engine_process_msg, + .process_msg = guc_exec_queue_process_msg, }; -static int guc_engine_init(struct xe_engine *e) +static int guc_exec_queue_init(struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched; - struct xe_guc *guc = engine_to_guc(e); - struct xe_guc_engine *ge; + struct xe_guc *guc = exec_queue_to_guc(q); + struct xe_guc_exec_queue *ge; long timeout; int err; @@ -1128,15 +1128,15 @@ static int guc_engine_init(struct xe_engine *e) if (!ge) return -ENOMEM; - e->guc = ge; - ge->engine = e; + q->guc = ge; + ge->q = q; init_waitqueue_head(&ge->suspend_wait); - timeout = xe_vm_no_dma_fences(e->vm) ? MAX_SCHEDULE_TIMEOUT : HZ * 5; + timeout = xe_vm_no_dma_fences(q->vm) ? MAX_SCHEDULE_TIMEOUT : HZ * 5; err = xe_sched_init(&ge->sched, &drm_sched_ops, &xe_sched_ops, NULL, - e->lrc[0].ring.size / MAX_JOB_SIZE_BYTES, + q->lrc[0].ring.size / MAX_JOB_SIZE_BYTES, 64, timeout, guc_to_gt(guc)->ordered_wq, NULL, - e->name, gt_to_xe(e->gt)->drm.dev); + q->name, gt_to_xe(q->gt)->drm.dev); if (err) goto err_free; @@ -1144,45 +1144,45 @@ static int guc_engine_init(struct xe_engine *e) err = xe_sched_entity_init(&ge->entity, sched); if (err) goto err_sched; - e->priority = XE_ENGINE_PRIORITY_NORMAL; + q->priority = XE_EXEC_QUEUE_PRIORITY_NORMAL; - if (xe_engine_is_lr(e)) - INIT_WORK(&e->guc->lr_tdr, xe_guc_engine_lr_cleanup); + if (xe_exec_queue_is_lr(q)) + INIT_WORK(&q->guc->lr_tdr, xe_guc_exec_queue_lr_cleanup); mutex_lock(&guc->submission_state.lock); - err = alloc_guc_id(guc, e); + err = alloc_guc_id(guc, q); if (err) goto err_entity; - e->entity = &ge->entity; + q->entity = &ge->entity; if (guc_read_stopped(guc)) xe_sched_stop(sched); mutex_unlock(&guc->submission_state.lock); - switch (e->class) { + switch (q->class) { case XE_ENGINE_CLASS_RENDER: - sprintf(e->name, "rcs%d", e->guc->id); + sprintf(q->name, "rcs%d", q->guc->id); break; case XE_ENGINE_CLASS_VIDEO_DECODE: - sprintf(e->name, "vcs%d", e->guc->id); + sprintf(q->name, "vcs%d", q->guc->id); break; case XE_ENGINE_CLASS_VIDEO_ENHANCE: - sprintf(e->name, "vecs%d", e->guc->id); + sprintf(q->name, "vecs%d", q->guc->id); break; case XE_ENGINE_CLASS_COPY: - sprintf(e->name, "bcs%d", e->guc->id); + sprintf(q->name, "bcs%d", q->guc->id); break; case XE_ENGINE_CLASS_COMPUTE: - sprintf(e->name, "ccs%d", e->guc->id); + sprintf(q->name, "ccs%d", q->guc->id); break; default: - XE_WARN_ON(e->class); + XE_WARN_ON(q->class); } - trace_xe_engine_create(e); + trace_xe_exec_queue_create(q); return 0; @@ -1196,133 +1196,133 @@ err_free: return err; } -static void guc_engine_kill(struct xe_engine *e) +static void guc_exec_queue_kill(struct xe_exec_queue *q) { - trace_xe_engine_kill(e); - set_engine_killed(e); - xe_guc_engine_trigger_cleanup(e); + trace_xe_exec_queue_kill(q); + set_exec_queue_killed(q); + xe_guc_exec_queue_trigger_cleanup(q); } -static void guc_engine_add_msg(struct xe_engine *e, struct xe_sched_msg *msg, - u32 opcode) +static void guc_exec_queue_add_msg(struct xe_exec_queue *q, struct xe_sched_msg *msg, + u32 opcode) { INIT_LIST_HEAD(&msg->link); msg->opcode = opcode; - msg->private_data = e; + msg->private_data = q; trace_xe_sched_msg_add(msg); - xe_sched_add_msg(&e->guc->sched, msg); + xe_sched_add_msg(&q->guc->sched, msg); } #define STATIC_MSG_CLEANUP 0 #define STATIC_MSG_SUSPEND 1 #define STATIC_MSG_RESUME 2 -static void guc_engine_fini(struct xe_engine *e) +static void guc_exec_queue_fini(struct xe_exec_queue *q) { - struct xe_sched_msg *msg = e->guc->static_msgs + STATIC_MSG_CLEANUP; + struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_CLEANUP; - if (!(e->flags & ENGINE_FLAG_KERNEL)) - guc_engine_add_msg(e, msg, CLEANUP); + if (!(q->flags & EXEC_QUEUE_FLAG_KERNEL)) + guc_exec_queue_add_msg(q, msg, CLEANUP); else - __guc_engine_fini(engine_to_guc(e), e); + __guc_exec_queue_fini(exec_queue_to_guc(q), q); } -static int guc_engine_set_priority(struct xe_engine *e, - enum xe_engine_priority priority) +static int guc_exec_queue_set_priority(struct xe_exec_queue *q, + enum xe_exec_queue_priority priority) { struct xe_sched_msg *msg; - if (e->priority == priority || engine_killed_or_banned(e)) + if (q->priority == priority || exec_queue_killed_or_banned(q)) return 0; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) return -ENOMEM; - guc_engine_add_msg(e, msg, SET_SCHED_PROPS); - e->priority = priority; + guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); + q->priority = priority; return 0; } -static int guc_engine_set_timeslice(struct xe_engine *e, u32 timeslice_us) +static int guc_exec_queue_set_timeslice(struct xe_exec_queue *q, u32 timeslice_us) { struct xe_sched_msg *msg; - if (e->sched_props.timeslice_us == timeslice_us || - engine_killed_or_banned(e)) + if (q->sched_props.timeslice_us == timeslice_us || + exec_queue_killed_or_banned(q)) return 0; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) return -ENOMEM; - e->sched_props.timeslice_us = timeslice_us; - guc_engine_add_msg(e, msg, SET_SCHED_PROPS); + q->sched_props.timeslice_us = timeslice_us; + guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); return 0; } -static int guc_engine_set_preempt_timeout(struct xe_engine *e, - u32 preempt_timeout_us) +static int guc_exec_queue_set_preempt_timeout(struct xe_exec_queue *q, + u32 preempt_timeout_us) { struct xe_sched_msg *msg; - if (e->sched_props.preempt_timeout_us == preempt_timeout_us || - engine_killed_or_banned(e)) + if (q->sched_props.preempt_timeout_us == preempt_timeout_us || + exec_queue_killed_or_banned(q)) return 0; msg = kmalloc(sizeof(*msg), GFP_KERNEL); if (!msg) return -ENOMEM; - e->sched_props.preempt_timeout_us = preempt_timeout_us; - guc_engine_add_msg(e, msg, SET_SCHED_PROPS); + q->sched_props.preempt_timeout_us = preempt_timeout_us; + guc_exec_queue_add_msg(q, msg, SET_SCHED_PROPS); return 0; } -static int guc_engine_set_job_timeout(struct xe_engine *e, u32 job_timeout_ms) +static int guc_exec_queue_set_job_timeout(struct xe_exec_queue *q, u32 job_timeout_ms) { - struct xe_gpu_scheduler *sched = &e->guc->sched; + struct xe_gpu_scheduler *sched = &q->guc->sched; - XE_WARN_ON(engine_registered(e)); - XE_WARN_ON(engine_banned(e)); - XE_WARN_ON(engine_killed(e)); + XE_WARN_ON(exec_queue_registered(q)); + XE_WARN_ON(exec_queue_banned(q)); + XE_WARN_ON(exec_queue_killed(q)); sched->base.timeout = job_timeout_ms; return 0; } -static int guc_engine_suspend(struct xe_engine *e) +static int guc_exec_queue_suspend(struct xe_exec_queue *q) { - struct xe_sched_msg *msg = e->guc->static_msgs + STATIC_MSG_SUSPEND; + struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_SUSPEND; - if (engine_killed_or_banned(e) || e->guc->suspend_pending) + if (exec_queue_killed_or_banned(q) || q->guc->suspend_pending) return -EINVAL; - e->guc->suspend_pending = true; - guc_engine_add_msg(e, msg, SUSPEND); + q->guc->suspend_pending = true; + guc_exec_queue_add_msg(q, msg, SUSPEND); return 0; } -static void guc_engine_suspend_wait(struct xe_engine *e) +static void guc_exec_queue_suspend_wait(struct xe_exec_queue *q) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); - wait_event(e->guc->suspend_wait, !e->guc->suspend_pending || + wait_event(q->guc->suspend_wait, !q->guc->suspend_pending || guc_read_stopped(guc)); } -static void guc_engine_resume(struct xe_engine *e) +static void guc_exec_queue_resume(struct xe_exec_queue *q) { - struct xe_sched_msg *msg = e->guc->static_msgs + STATIC_MSG_RESUME; + struct xe_sched_msg *msg = q->guc->static_msgs + STATIC_MSG_RESUME; - XE_WARN_ON(e->guc->suspend_pending); + XE_WARN_ON(q->guc->suspend_pending); - guc_engine_add_msg(e, msg, RESUME); + guc_exec_queue_add_msg(q, msg, RESUME); } /* @@ -1331,49 +1331,49 @@ static void guc_engine_resume(struct xe_engine *e) * really shouldn't do much other than trap into the DRM scheduler which * synchronizes these operations. */ -static const struct xe_engine_ops guc_engine_ops = { - .init = guc_engine_init, - .kill = guc_engine_kill, - .fini = guc_engine_fini, - .set_priority = guc_engine_set_priority, - .set_timeslice = guc_engine_set_timeslice, - .set_preempt_timeout = guc_engine_set_preempt_timeout, - .set_job_timeout = guc_engine_set_job_timeout, - .suspend = guc_engine_suspend, - .suspend_wait = guc_engine_suspend_wait, - .resume = guc_engine_resume, +static const struct xe_exec_queue_ops guc_exec_queue_ops = { + .init = guc_exec_queue_init, + .kill = guc_exec_queue_kill, + .fini = guc_exec_queue_fini, + .set_priority = guc_exec_queue_set_priority, + .set_timeslice = guc_exec_queue_set_timeslice, + .set_preempt_timeout = guc_exec_queue_set_preempt_timeout, + .set_job_timeout = guc_exec_queue_set_job_timeout, + .suspend = guc_exec_queue_suspend, + .suspend_wait = guc_exec_queue_suspend_wait, + .resume = guc_exec_queue_resume, }; -static void guc_engine_stop(struct xe_guc *guc, struct xe_engine *e) +static void guc_exec_queue_stop(struct xe_guc *guc, struct xe_exec_queue *q) { - struct xe_gpu_scheduler *sched = &e->guc->sched; + struct xe_gpu_scheduler *sched = &q->guc->sched; /* Stop scheduling + flush any DRM scheduler operations */ xe_sched_submission_stop(sched); /* Clean up lost G2H + reset engine state */ - if (engine_registered(e)) { - if ((engine_banned(e) && engine_destroyed(e)) || - xe_engine_is_lr(e)) - xe_engine_put(e); - else if (engine_destroyed(e)) - __guc_engine_fini(guc, e); + if (exec_queue_registered(q)) { + if ((exec_queue_banned(q) && exec_queue_destroyed(q)) || + xe_exec_queue_is_lr(q)) + xe_exec_queue_put(q); + else if (exec_queue_destroyed(q)) + __guc_exec_queue_fini(guc, q); } - if (e->guc->suspend_pending) { - set_engine_suspended(e); - suspend_fence_signal(e); + if (q->guc->suspend_pending) { + set_exec_queue_suspended(q); + suspend_fence_signal(q); } - atomic_and(ENGINE_STATE_DESTROYED | ENGINE_STATE_SUSPENDED, - &e->guc->state); - e->guc->resume_time = 0; - trace_xe_engine_stop(e); + atomic_and(EXEC_QUEUE_STATE_DESTROYED | ENGINE_STATE_SUSPENDED, + &q->guc->state); + q->guc->resume_time = 0; + trace_xe_exec_queue_stop(q); /* * Ban any engine (aside from kernel and engines used for VM ops) with a * started but not complete job or if a job has gone through a GT reset * more than twice. */ - if (!(e->flags & (ENGINE_FLAG_KERNEL | ENGINE_FLAG_VM))) { + if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM))) { struct xe_sched_job *job = xe_sched_first_pending_job(sched); if (job) { @@ -1381,8 +1381,8 @@ static void guc_engine_stop(struct xe_guc *guc, struct xe_engine *e) !xe_sched_job_completed(job)) || xe_sched_invalidate_job(job, 2)) { trace_xe_sched_job_ban(job); - xe_sched_tdr_queue_imm(&e->guc->sched); - set_engine_banned(e); + xe_sched_tdr_queue_imm(&q->guc->sched); + set_exec_queue_banned(q); } } } @@ -1413,15 +1413,15 @@ void xe_guc_submit_reset_wait(struct xe_guc *guc) int xe_guc_submit_stop(struct xe_guc *guc) { - struct xe_engine *e; + struct xe_exec_queue *q; unsigned long index; XE_WARN_ON(guc_read_stopped(guc) != 1); mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.engine_lookup, index, e) - guc_engine_stop(guc, e); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + guc_exec_queue_stop(guc, q); mutex_unlock(&guc->submission_state.lock); @@ -1433,16 +1433,16 @@ int xe_guc_submit_stop(struct xe_guc *guc) return 0; } -static void guc_engine_start(struct xe_engine *e) +static void guc_exec_queue_start(struct xe_exec_queue *q) { - struct xe_gpu_scheduler *sched = &e->guc->sched; + struct xe_gpu_scheduler *sched = &q->guc->sched; - if (!engine_killed_or_banned(e)) { + if (!exec_queue_killed_or_banned(q)) { int i; - trace_xe_engine_resubmit(e); - for (i = 0; i < e->width; ++i) - xe_lrc_set_ring_head(e->lrc + i, e->lrc[i].ring.tail); + trace_xe_exec_queue_resubmit(q); + for (i = 0; i < q->width; ++i) + xe_lrc_set_ring_head(q->lrc + i, q->lrc[i].ring.tail); xe_sched_resubmit_jobs(sched); } @@ -1451,15 +1451,15 @@ static void guc_engine_start(struct xe_engine *e) int xe_guc_submit_start(struct xe_guc *guc) { - struct xe_engine *e; + struct xe_exec_queue *q; unsigned long index; XE_WARN_ON(guc_read_stopped(guc) != 1); mutex_lock(&guc->submission_state.lock); atomic_dec(&guc->submission_state.stopped); - xa_for_each(&guc->submission_state.engine_lookup, index, e) - guc_engine_start(e); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + guc_exec_queue_start(q); mutex_unlock(&guc->submission_state.lock); wake_up_all(&guc->ct.wq); @@ -1467,36 +1467,36 @@ int xe_guc_submit_start(struct xe_guc *guc) return 0; } -static struct xe_engine * -g2h_engine_lookup(struct xe_guc *guc, u32 guc_id) +static struct xe_exec_queue * +g2h_exec_queue_lookup(struct xe_guc *guc, u32 guc_id) { struct xe_device *xe = guc_to_xe(guc); - struct xe_engine *e; + struct xe_exec_queue *q; if (unlikely(guc_id >= GUC_ID_MAX)) { drm_err(&xe->drm, "Invalid guc_id %u", guc_id); return NULL; } - e = xa_load(&guc->submission_state.engine_lookup, guc_id); - if (unlikely(!e)) { + q = xa_load(&guc->submission_state.exec_queue_lookup, guc_id); + if (unlikely(!q)) { drm_err(&xe->drm, "Not engine present for guc_id %u", guc_id); return NULL; } - XE_WARN_ON(e->guc->id != guc_id); + XE_WARN_ON(q->guc->id != guc_id); - return e; + return q; } -static void deregister_engine(struct xe_guc *guc, struct xe_engine *e) +static void deregister_exec_queue(struct xe_guc *guc, struct xe_exec_queue *q) { u32 action[] = { XE_GUC_ACTION_DEREGISTER_CONTEXT, - e->guc->id, + q->guc->id, }; - trace_xe_engine_deregister(e); + trace_xe_exec_queue_deregister(q); xe_guc_ct_send_g2h_handler(&guc->ct, action, ARRAY_SIZE(action)); } @@ -1504,7 +1504,7 @@ static void deregister_engine(struct xe_guc *guc, struct xe_engine *e) int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_device *xe = guc_to_xe(guc); - struct xe_engine *e; + struct xe_exec_queue *q; u32 guc_id = msg[0]; if (unlikely(len < 2)) { @@ -1512,34 +1512,34 @@ int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) return -EPROTO; } - e = g2h_engine_lookup(guc, guc_id); - if (unlikely(!e)) + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) return -EPROTO; - if (unlikely(!engine_pending_enable(e) && - !engine_pending_disable(e))) { + if (unlikely(!exec_queue_pending_enable(q) && + !exec_queue_pending_disable(q))) { drm_err(&xe->drm, "Unexpected engine state 0x%04x", - atomic_read(&e->guc->state)); + atomic_read(&q->guc->state)); return -EPROTO; } - trace_xe_engine_scheduling_done(e); + trace_xe_exec_queue_scheduling_done(q); - if (engine_pending_enable(e)) { - e->guc->resume_time = ktime_get(); - clear_engine_pending_enable(e); + if (exec_queue_pending_enable(q)) { + q->guc->resume_time = ktime_get(); + clear_exec_queue_pending_enable(q); smp_wmb(); wake_up_all(&guc->ct.wq); } else { - clear_engine_pending_disable(e); - if (e->guc->suspend_pending) { - suspend_fence_signal(e); + clear_exec_queue_pending_disable(q); + if (q->guc->suspend_pending) { + suspend_fence_signal(q); } else { - if (engine_banned(e)) { + if (exec_queue_banned(q)) { smp_wmb(); wake_up_all(&guc->ct.wq); } - deregister_engine(guc, e); + deregister_exec_queue(guc, q); } } @@ -1549,7 +1549,7 @@ int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len) int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_device *xe = guc_to_xe(guc); - struct xe_engine *e; + struct xe_exec_queue *q; u32 guc_id = msg[0]; if (unlikely(len < 1)) { @@ -1557,33 +1557,33 @@ int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len) return -EPROTO; } - e = g2h_engine_lookup(guc, guc_id); - if (unlikely(!e)) + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) return -EPROTO; - if (!engine_destroyed(e) || engine_pending_disable(e) || - engine_pending_enable(e) || engine_enabled(e)) { + if (!exec_queue_destroyed(q) || exec_queue_pending_disable(q) || + exec_queue_pending_enable(q) || exec_queue_enabled(q)) { drm_err(&xe->drm, "Unexpected engine state 0x%04x", - atomic_read(&e->guc->state)); + atomic_read(&q->guc->state)); return -EPROTO; } - trace_xe_engine_deregister_done(e); + trace_xe_exec_queue_deregister_done(q); - clear_engine_registered(e); + clear_exec_queue_registered(q); - if (engine_banned(e) || xe_engine_is_lr(e)) - xe_engine_put(e); + if (exec_queue_banned(q) || xe_exec_queue_is_lr(q)) + xe_exec_queue_put(q); else - __guc_engine_fini(guc, e); + __guc_exec_queue_fini(guc, q); return 0; } -int xe_guc_engine_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) +int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_device *xe = guc_to_xe(guc); - struct xe_engine *e; + struct xe_exec_queue *q; u32 guc_id = msg[0]; if (unlikely(len < 1)) { @@ -1591,34 +1591,34 @@ int xe_guc_engine_reset_handler(struct xe_guc *guc, u32 *msg, u32 len) return -EPROTO; } - e = g2h_engine_lookup(guc, guc_id); - if (unlikely(!e)) + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) return -EPROTO; drm_info(&xe->drm, "Engine reset: guc_id=%d", guc_id); /* FIXME: Do error capture, most likely async */ - trace_xe_engine_reset(e); + trace_xe_exec_queue_reset(q); /* * A banned engine is a NOP at this point (came from - * guc_engine_timedout_job). Otherwise, kick drm scheduler to cancel + * guc_exec_queue_timedout_job). Otherwise, kick drm scheduler to cancel * jobs by setting timeout of the job to the minimum value kicking - * guc_engine_timedout_job. + * guc_exec_queue_timedout_job. */ - set_engine_reset(e); - if (!engine_banned(e)) - xe_guc_engine_trigger_cleanup(e); + set_exec_queue_reset(q); + if (!exec_queue_banned(q)) + xe_guc_exec_queue_trigger_cleanup(q); return 0; } -int xe_guc_engine_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, - u32 len) +int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, + u32 len) { struct xe_device *xe = guc_to_xe(guc); - struct xe_engine *e; + struct xe_exec_queue *q; u32 guc_id = msg[0]; if (unlikely(len < 1)) { @@ -1626,22 +1626,22 @@ int xe_guc_engine_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, return -EPROTO; } - e = g2h_engine_lookup(guc, guc_id); - if (unlikely(!e)) + q = g2h_exec_queue_lookup(guc, guc_id); + if (unlikely(!q)) return -EPROTO; drm_warn(&xe->drm, "Engine memory cat error: guc_id=%d", guc_id); - trace_xe_engine_memory_cat_error(e); + trace_xe_exec_queue_memory_cat_error(q); /* Treat the same as engine reset */ - set_engine_reset(e); - if (!engine_banned(e)) - xe_guc_engine_trigger_cleanup(e); + set_exec_queue_reset(q); + if (!exec_queue_banned(q)) + xe_guc_exec_queue_trigger_cleanup(q); return 0; } -int xe_guc_engine_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) +int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) { struct xe_device *xe = guc_to_xe(guc); u8 guc_class, instance; @@ -1666,16 +1666,16 @@ int xe_guc_engine_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len) } static void -guc_engine_wq_snapshot_capture(struct xe_engine *e, - struct xe_guc_submit_engine_snapshot *snapshot) +guc_exec_queue_wq_snapshot_capture(struct xe_exec_queue *q, + struct xe_guc_submit_exec_queue_snapshot *snapshot) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(e->lrc); + struct iosys_map map = xe_lrc_parallel_map(q->lrc); int i; - snapshot->guc.wqi_head = e->guc->wqi_head; - snapshot->guc.wqi_tail = e->guc->wqi_tail; + snapshot->guc.wqi_head = q->guc->wqi_head; + snapshot->guc.wqi_tail = q->guc->wqi_tail; snapshot->parallel.wq_desc.head = parallel_read(xe, map, wq_desc.head); snapshot->parallel.wq_desc.tail = parallel_read(xe, map, wq_desc.tail); snapshot->parallel.wq_desc.status = parallel_read(xe, map, @@ -1692,8 +1692,8 @@ guc_engine_wq_snapshot_capture(struct xe_engine *e, } static void -guc_engine_wq_snapshot_print(struct xe_guc_submit_engine_snapshot *snapshot, - struct drm_printer *p) +guc_exec_queue_wq_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, + struct drm_printer *p) { int i; @@ -1714,23 +1714,23 @@ guc_engine_wq_snapshot_print(struct xe_guc_submit_engine_snapshot *snapshot, } /** - * xe_guc_engine_snapshot_capture - Take a quick snapshot of the GuC Engine. - * @e: Xe Engine. + * xe_guc_exec_queue_snapshot_capture - Take a quick snapshot of the GuC Engine. + * @q: Xe exec queue. * * This can be printed out in a later stage like during dev_coredump * analysis. * * Returns: a GuC Submit Engine snapshot object that must be freed by the - * caller, using `xe_guc_engine_snapshot_free`. + * caller, using `xe_guc_exec_queue_snapshot_free`. */ -struct xe_guc_submit_engine_snapshot * -xe_guc_engine_snapshot_capture(struct xe_engine *e) +struct xe_guc_submit_exec_queue_snapshot * +xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q) { - struct xe_guc *guc = engine_to_guc(e); + struct xe_guc *guc = exec_queue_to_guc(q); struct xe_device *xe = guc_to_xe(guc); - struct xe_gpu_scheduler *sched = &e->guc->sched; + struct xe_gpu_scheduler *sched = &q->guc->sched; struct xe_sched_job *job; - struct xe_guc_submit_engine_snapshot *snapshot; + struct xe_guc_submit_exec_queue_snapshot *snapshot; int i; snapshot = kzalloc(sizeof(*snapshot), GFP_ATOMIC); @@ -1740,25 +1740,25 @@ xe_guc_engine_snapshot_capture(struct xe_engine *e) return NULL; } - snapshot->guc.id = e->guc->id; - memcpy(&snapshot->name, &e->name, sizeof(snapshot->name)); - snapshot->class = e->class; - snapshot->logical_mask = e->logical_mask; - snapshot->width = e->width; - snapshot->refcount = kref_read(&e->refcount); + snapshot->guc.id = q->guc->id; + memcpy(&snapshot->name, &q->name, sizeof(snapshot->name)); + snapshot->class = q->class; + snapshot->logical_mask = q->logical_mask; + snapshot->width = q->width; + snapshot->refcount = kref_read(&q->refcount); snapshot->sched_timeout = sched->base.timeout; - snapshot->sched_props.timeslice_us = e->sched_props.timeslice_us; + snapshot->sched_props.timeslice_us = q->sched_props.timeslice_us; snapshot->sched_props.preempt_timeout_us = - e->sched_props.preempt_timeout_us; + q->sched_props.preempt_timeout_us; - snapshot->lrc = kmalloc_array(e->width, sizeof(struct lrc_snapshot), + snapshot->lrc = kmalloc_array(q->width, sizeof(struct lrc_snapshot), GFP_ATOMIC); if (!snapshot->lrc) { drm_err(&xe->drm, "Skipping GuC Engine LRC snapshot.\n"); } else { - for (i = 0; i < e->width; ++i) { - struct xe_lrc *lrc = e->lrc + i; + for (i = 0; i < q->width; ++i) { + struct xe_lrc *lrc = q->lrc + i; snapshot->lrc[i].context_desc = lower_32_bits(xe_lrc_ggtt_addr(lrc)); @@ -1771,12 +1771,12 @@ xe_guc_engine_snapshot_capture(struct xe_engine *e) } } - snapshot->schedule_state = atomic_read(&e->guc->state); - snapshot->engine_flags = e->flags; + snapshot->schedule_state = atomic_read(&q->guc->state); + snapshot->exec_queue_flags = q->flags; - snapshot->parallel_execution = xe_engine_is_parallel(e); + snapshot->parallel_execution = xe_exec_queue_is_parallel(q); if (snapshot->parallel_execution) - guc_engine_wq_snapshot_capture(e, snapshot); + guc_exec_queue_wq_snapshot_capture(q, snapshot); spin_lock(&sched->base.job_list_lock); snapshot->pending_list_size = list_count_nodes(&sched->base.pending_list); @@ -1806,15 +1806,15 @@ xe_guc_engine_snapshot_capture(struct xe_engine *e) } /** - * xe_guc_engine_snapshot_print - Print out a given GuC Engine snapshot. + * xe_guc_exec_queue_snapshot_print - Print out a given GuC Engine snapshot. * @snapshot: GuC Submit Engine snapshot object. * @p: drm_printer where it will be printed out. * * This function prints out a given GuC Submit Engine snapshot object. */ void -xe_guc_engine_snapshot_print(struct xe_guc_submit_engine_snapshot *snapshot, - struct drm_printer *p) +xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, + struct drm_printer *p) { int i; @@ -1846,10 +1846,10 @@ xe_guc_engine_snapshot_print(struct xe_guc_submit_engine_snapshot *snapshot, drm_printf(p, "\tSeqno: (memory) %d\n", snapshot->lrc[i].seqno); } drm_printf(p, "\tSchedule State: 0x%x\n", snapshot->schedule_state); - drm_printf(p, "\tFlags: 0x%lx\n", snapshot->engine_flags); + drm_printf(p, "\tFlags: 0x%lx\n", snapshot->exec_queue_flags); if (snapshot->parallel_execution) - guc_engine_wq_snapshot_print(snapshot, p); + guc_exec_queue_wq_snapshot_print(snapshot, p); for (i = 0; snapshot->pending_list && i < snapshot->pending_list_size; i++) @@ -1860,14 +1860,14 @@ xe_guc_engine_snapshot_print(struct xe_guc_submit_engine_snapshot *snapshot, } /** - * xe_guc_engine_snapshot_free - Free all allocated objects for a given + * xe_guc_exec_queue_snapshot_free - Free all allocated objects for a given * snapshot. * @snapshot: GuC Submit Engine snapshot object. * * This function free all the memory that needed to be allocated at capture * time. */ -void xe_guc_engine_snapshot_free(struct xe_guc_submit_engine_snapshot *snapshot) +void xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot) { if (!snapshot) return; @@ -1877,13 +1877,13 @@ void xe_guc_engine_snapshot_free(struct xe_guc_submit_engine_snapshot *snapshot) kfree(snapshot); } -static void guc_engine_print(struct xe_engine *e, struct drm_printer *p) +static void guc_exec_queue_print(struct xe_exec_queue *q, struct drm_printer *p) { - struct xe_guc_submit_engine_snapshot *snapshot; + struct xe_guc_submit_exec_queue_snapshot *snapshot; - snapshot = xe_guc_engine_snapshot_capture(e); - xe_guc_engine_snapshot_print(snapshot, p); - xe_guc_engine_snapshot_free(snapshot); + snapshot = xe_guc_exec_queue_snapshot_capture(q); + xe_guc_exec_queue_snapshot_print(snapshot, p); + xe_guc_exec_queue_snapshot_free(snapshot); } /** @@ -1895,14 +1895,14 @@ static void guc_engine_print(struct xe_engine *e, struct drm_printer *p) */ void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p) { - struct xe_engine *e; + struct xe_exec_queue *q; unsigned long index; if (!xe_device_guc_submission_enabled(guc_to_xe(guc))) return; mutex_lock(&guc->submission_state.lock); - xa_for_each(&guc->submission_state.engine_lookup, index, e) - guc_engine_print(e, p); + xa_for_each(&guc->submission_state.exec_queue_lookup, index, q) + guc_exec_queue_print(q, p); mutex_unlock(&guc->submission_state.lock); } diff --git a/drivers/gpu/drm/xe/xe_guc_submit.h b/drivers/gpu/drm/xe/xe_guc_submit.h index 4153c2d22013..fc97869c5b86 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.h +++ b/drivers/gpu/drm/xe/xe_guc_submit.h @@ -9,7 +9,7 @@ #include struct drm_printer; -struct xe_engine; +struct xe_exec_queue; struct xe_guc; int xe_guc_submit_init(struct xe_guc *guc); @@ -21,18 +21,18 @@ int xe_guc_submit_start(struct xe_guc *guc); int xe_guc_sched_done_handler(struct xe_guc *guc, u32 *msg, u32 len); int xe_guc_deregister_done_handler(struct xe_guc *guc, u32 *msg, u32 len); -int xe_guc_engine_reset_handler(struct xe_guc *guc, u32 *msg, u32 len); -int xe_guc_engine_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, - u32 len); -int xe_guc_engine_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_exec_queue_reset_handler(struct xe_guc *guc, u32 *msg, u32 len); +int xe_guc_exec_queue_memory_cat_error_handler(struct xe_guc *guc, u32 *msg, + u32 len); +int xe_guc_exec_queue_reset_failure_handler(struct xe_guc *guc, u32 *msg, u32 len); -struct xe_guc_submit_engine_snapshot * -xe_guc_engine_snapshot_capture(struct xe_engine *e); +struct xe_guc_submit_exec_queue_snapshot * +xe_guc_exec_queue_snapshot_capture(struct xe_exec_queue *q); void -xe_guc_engine_snapshot_print(struct xe_guc_submit_engine_snapshot *snapshot, - struct drm_printer *p); +xe_guc_exec_queue_snapshot_print(struct xe_guc_submit_exec_queue_snapshot *snapshot, + struct drm_printer *p); void -xe_guc_engine_snapshot_free(struct xe_guc_submit_engine_snapshot *snapshot); +xe_guc_exec_queue_snapshot_free(struct xe_guc_submit_exec_queue_snapshot *snapshot); void xe_guc_submit_print(struct xe_guc *guc, struct drm_printer *p); #endif diff --git a/drivers/gpu/drm/xe/xe_guc_submit_types.h b/drivers/gpu/drm/xe/xe_guc_submit_types.h index 6765b2c6eab1..649b0a852692 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit_types.h +++ b/drivers/gpu/drm/xe/xe_guc_submit_types.h @@ -79,20 +79,20 @@ struct pending_list_snapshot { }; /** - * struct xe_guc_submit_engine_snapshot - Snapshot for devcoredump + * struct xe_guc_submit_exec_queue_snapshot - Snapshot for devcoredump */ -struct xe_guc_submit_engine_snapshot { - /** @name: name of this engine */ +struct xe_guc_submit_exec_queue_snapshot { + /** @name: name of this exec queue */ char name[MAX_FENCE_NAME_LEN]; - /** @class: class of this engine */ + /** @class: class of this exec queue */ enum xe_engine_class class; /** - * @logical_mask: logical mask of where job submitted to engine can run + * @logical_mask: logical mask of where job submitted to exec queue can run */ u32 logical_mask; - /** @width: width (number BB submitted per exec) of this engine */ + /** @width: width (number BB submitted per exec) of this exec queue */ u16 width; - /** @refcount: ref count of this engine */ + /** @refcount: ref count of this exec queue */ u32 refcount; /** * @sched_timeout: the time after which a job is removed from the @@ -113,8 +113,8 @@ struct xe_guc_submit_engine_snapshot { /** @schedule_state: Schedule State at the moment of Crash */ u32 schedule_state; - /** @engine_flags: Flags of the faulty engine */ - unsigned long engine_flags; + /** @exec_queue_flags: Flags of the faulty exec_queue */ + unsigned long exec_queue_flags; /** @guc: GuC Engine Snapshot */ struct { @@ -122,7 +122,7 @@ struct xe_guc_submit_engine_snapshot { u32 wqi_head; /** @wqi_tail: work queue item tail */ u32 wqi_tail; - /** @id: GuC id for this xe_engine */ + /** @id: GuC id for this exec_queue */ u16 id; } guc; diff --git a/drivers/gpu/drm/xe/xe_guc_types.h b/drivers/gpu/drm/xe/xe_guc_types.h index a304dce4e9f4..a5e58917a499 100644 --- a/drivers/gpu/drm/xe/xe_guc_types.h +++ b/drivers/gpu/drm/xe/xe_guc_types.h @@ -33,8 +33,8 @@ struct xe_guc { struct xe_guc_pc pc; /** @submission_state: GuC submission state */ struct { - /** @engine_lookup: Lookup an xe_engine from guc_id */ - struct xarray engine_lookup; + /** @exec_queue_lookup: Lookup an xe_engine from guc_id */ + struct xarray exec_queue_lookup; /** @guc_ids: used to allocate new guc_ids, single-lrc */ struct ida guc_ids; /** @guc_ids_bitmap: used to allocate new guc_ids, multi-lrc */ diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index 05f3d8d68379..09db8da261a3 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -12,7 +12,7 @@ #include "regs/xe_regs.h" #include "xe_bo.h" #include "xe_device.h" -#include "xe_engine_types.h" +#include "xe_exec_queue_types.h" #include "xe_gt.h" #include "xe_hw_fence.h" #include "xe_map.h" @@ -604,7 +604,7 @@ static void xe_lrc_set_ppgtt(struct xe_lrc *lrc, struct xe_vm *vm) #define ACC_NOTIFY_S 16 int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - struct xe_engine *e, struct xe_vm *vm, u32 ring_size) + struct xe_exec_queue *q, struct xe_vm *vm, u32 ring_size) { struct xe_gt *gt = hwe->gt; struct xe_tile *tile = gt_to_tile(gt); @@ -669,12 +669,12 @@ int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, RING_CTL_SIZE(lrc->ring.size) | RING_VALID); if (xe->info.has_asid && vm) xe_lrc_write_ctx_reg(lrc, PVC_CTX_ASID, - (e->usm.acc_granularity << + (q->usm.acc_granularity << ACC_GRANULARITY_S) | vm->usm.asid); if (xe->info.supports_usm && vm) xe_lrc_write_ctx_reg(lrc, PVC_CTX_ACC_CTR_THOLD, - (e->usm.acc_notify << ACC_NOTIFY_S) | - e->usm.acc_trigger); + (q->usm.acc_notify << ACC_NOTIFY_S) | + q->usm.acc_trigger); lrc->desc = GEN8_CTX_VALID; lrc->desc |= INTEL_LEGACY_64B_CONTEXT << GEN8_CTX_ADDRESSING_MODE_SHIFT; diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index e37f89e75ef8..3a6e8fc5a837 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -8,7 +8,7 @@ #include "xe_lrc_types.h" struct xe_device; -struct xe_engine; +struct xe_exec_queue; enum xe_engine_class; struct xe_hw_engine; struct xe_vm; @@ -16,7 +16,7 @@ struct xe_vm; #define LRC_PPHWSP_SCRATCH_ADDR (0x34 * 4) int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - struct xe_engine *e, struct xe_vm *vm, u32 ring_size); + struct xe_exec_queue *q, struct xe_vm *vm, u32 ring_size); void xe_lrc_finish(struct xe_lrc *lrc); size_t xe_lrc_size(struct xe_device *xe, enum xe_engine_class class); diff --git a/drivers/gpu/drm/xe/xe_migrate.c b/drivers/gpu/drm/xe/xe_migrate.c index 60f7226c92ff..d0816d2090f0 100644 --- a/drivers/gpu/drm/xe/xe_migrate.c +++ b/drivers/gpu/drm/xe/xe_migrate.c @@ -34,8 +34,8 @@ * struct xe_migrate - migrate context. */ struct xe_migrate { - /** @eng: Default engine used for migration */ - struct xe_engine *eng; + /** @q: Default exec queue used for migration */ + struct xe_exec_queue *q; /** @tile: Backpointer to the tile this struct xe_migrate belongs to. */ struct xe_tile *tile; /** @job_mutex: Timeline mutex for @eng. */ @@ -78,9 +78,9 @@ struct xe_migrate { * * Return: The default migrate engine */ -struct xe_engine *xe_tile_migrate_engine(struct xe_tile *tile) +struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile) { - return tile->migrate->eng; + return tile->migrate->q; } static void xe_migrate_fini(struct drm_device *dev, void *arg) @@ -88,11 +88,11 @@ static void xe_migrate_fini(struct drm_device *dev, void *arg) struct xe_migrate *m = arg; struct ww_acquire_ctx ww; - xe_vm_lock(m->eng->vm, &ww, 0, false); + xe_vm_lock(m->q->vm, &ww, 0, false); xe_bo_unpin(m->pt_bo); if (m->cleared_bo) xe_bo_unpin(m->cleared_bo); - xe_vm_unlock(m->eng->vm, &ww); + xe_vm_unlock(m->q->vm, &ww); dma_fence_put(m->fence); if (m->cleared_bo) @@ -100,8 +100,8 @@ static void xe_migrate_fini(struct drm_device *dev, void *arg) xe_bo_put(m->pt_bo); drm_suballoc_manager_fini(&m->vm_update_sa); mutex_destroy(&m->job_mutex); - xe_vm_close_and_put(m->eng->vm); - xe_engine_put(m->eng); + xe_vm_close_and_put(m->q->vm); + xe_exec_queue_put(m->q); } static u64 xe_migrate_vm_addr(u64 slot, u32 level) @@ -341,20 +341,20 @@ struct xe_migrate *xe_migrate_init(struct xe_tile *tile) if (!hwe) return ERR_PTR(-EINVAL); - m->eng = xe_engine_create(xe, vm, - BIT(hwe->logical_instance), 1, - hwe, ENGINE_FLAG_KERNEL); + m->q = xe_exec_queue_create(xe, vm, + BIT(hwe->logical_instance), 1, + hwe, EXEC_QUEUE_FLAG_KERNEL); } else { - m->eng = xe_engine_create_class(xe, primary_gt, vm, - XE_ENGINE_CLASS_COPY, - ENGINE_FLAG_KERNEL); + m->q = xe_exec_queue_create_class(xe, primary_gt, vm, + XE_ENGINE_CLASS_COPY, + EXEC_QUEUE_FLAG_KERNEL); } - if (IS_ERR(m->eng)) { + if (IS_ERR(m->q)) { xe_vm_close_and_put(vm); - return ERR_CAST(m->eng); + return ERR_CAST(m->q); } if (xe->info.supports_usm) - m->eng->priority = XE_ENGINE_PRIORITY_KERNEL; + m->q->priority = XE_EXEC_QUEUE_PRIORITY_KERNEL; mutex_init(&m->job_mutex); @@ -456,7 +456,7 @@ static void emit_pte(struct xe_migrate *m, addr = xe_res_dma(cur) & PAGE_MASK; if (is_vram) { /* Is this a 64K PTE entry? */ - if ((m->eng->vm->flags & XE_VM_FLAG_64K) && + if ((m->q->vm->flags & XE_VM_FLAG_64K) && !(cur_ofs & (16 * 8 - 1))) { XE_WARN_ON(!IS_ALIGNED(addr, SZ_64K)); addr |= XE_PTE_PS64; @@ -714,7 +714,7 @@ struct dma_fence *xe_migrate_copy(struct xe_migrate *m, src_L0, ccs_ofs, copy_ccs); mutex_lock(&m->job_mutex); - job = xe_bb_create_migration_job(m->eng, bb, + job = xe_bb_create_migration_job(m->q, bb, xe_migrate_batch_base(m, usm), update_idx); if (IS_ERR(job)) { @@ -938,7 +938,7 @@ struct dma_fence *xe_migrate_clear(struct xe_migrate *m, } mutex_lock(&m->job_mutex); - job = xe_bb_create_migration_job(m->eng, bb, + job = xe_bb_create_migration_job(m->q, bb, xe_migrate_batch_base(m, usm), update_idx); if (IS_ERR(job)) { @@ -1024,7 +1024,7 @@ static void write_pgtable(struct xe_tile *tile, struct xe_bb *bb, u64 ppgtt_ofs, struct xe_vm *xe_migrate_get_vm(struct xe_migrate *m) { - return xe_vm_get(m->eng->vm); + return xe_vm_get(m->q->vm); } #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST) @@ -1106,7 +1106,7 @@ static bool no_in_syncs(struct xe_sync_entry *syncs, u32 num_syncs) * @m: The migrate context. * @vm: The vm we'll be updating. * @bo: The bo whose dma-resv we will await before updating, or NULL if userptr. - * @eng: The engine to be used for the update or NULL if the default + * @q: The exec queue to be used for the update or NULL if the default * migration engine is to be used. * @updates: An array of update descriptors. * @num_updates: Number of descriptors in @updates. @@ -1132,7 +1132,7 @@ struct dma_fence * xe_migrate_update_pgtables(struct xe_migrate *m, struct xe_vm *vm, struct xe_bo *bo, - struct xe_engine *eng, + struct xe_exec_queue *q, const struct xe_vm_pgtable_update *updates, u32 num_updates, struct xe_sync_entry *syncs, u32 num_syncs, @@ -1150,13 +1150,13 @@ xe_migrate_update_pgtables(struct xe_migrate *m, u32 i, batch_size, ppgtt_ofs, update_idx, page_ofs = 0; u64 addr; int err = 0; - bool usm = !eng && xe->info.supports_usm; + bool usm = !q && xe->info.supports_usm; bool first_munmap_rebind = vma && vma->gpuva.flags & XE_VMA_FIRST_REBIND; - struct xe_engine *eng_override = !eng ? m->eng : eng; + struct xe_exec_queue *q_override = !q ? m->q : q; /* Use the CPU if no in syncs and engine is idle */ - if (no_in_syncs(syncs, num_syncs) && xe_engine_is_idle(eng_override)) { + if (no_in_syncs(syncs, num_syncs) && xe_exec_queue_is_idle(q_override)) { fence = xe_migrate_update_pgtables_cpu(m, vm, bo, updates, num_updates, first_munmap_rebind, @@ -1186,14 +1186,14 @@ xe_migrate_update_pgtables(struct xe_migrate *m, */ XE_WARN_ON(batch_size >= SZ_128K); - bb = xe_bb_new(gt, batch_size, !eng && xe->info.supports_usm); + bb = xe_bb_new(gt, batch_size, !q && xe->info.supports_usm); if (IS_ERR(bb)) return ERR_CAST(bb); /* For sysmem PTE's, need to map them in our hole.. */ if (!IS_DGFX(xe)) { ppgtt_ofs = NUM_KERNEL_PDE - 1; - if (eng) { + if (q) { XE_WARN_ON(num_updates > NUM_VMUSA_WRITES_PER_UNIT); sa_bo = drm_suballoc_new(&m->vm_update_sa, 1, @@ -1249,10 +1249,10 @@ xe_migrate_update_pgtables(struct xe_migrate *m, write_pgtable(tile, bb, 0, &updates[i], pt_update); } - if (!eng) + if (!q) mutex_lock(&m->job_mutex); - job = xe_bb_create_migration_job(eng ?: m->eng, bb, + job = xe_bb_create_migration_job(q ?: m->q, bb, xe_migrate_batch_base(m, usm), update_idx); if (IS_ERR(job)) { @@ -1295,7 +1295,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m, fence = dma_fence_get(&job->drm.s_fence->finished); xe_sched_job_push(job); - if (!eng) + if (!q) mutex_unlock(&m->job_mutex); xe_bb_free(bb, fence); @@ -1306,7 +1306,7 @@ xe_migrate_update_pgtables(struct xe_migrate *m, err_job: xe_sched_job_put(job); err_bb: - if (!eng) + if (!q) mutex_unlock(&m->job_mutex); xe_bb_free(bb, NULL); err: diff --git a/drivers/gpu/drm/xe/xe_migrate.h b/drivers/gpu/drm/xe/xe_migrate.h index 0d62aff6421c..c729241776ad 100644 --- a/drivers/gpu/drm/xe/xe_migrate.h +++ b/drivers/gpu/drm/xe/xe_migrate.h @@ -14,7 +14,7 @@ struct ttm_resource; struct xe_bo; struct xe_gt; -struct xe_engine; +struct xe_exec_queue; struct xe_migrate; struct xe_migrate_pt_update; struct xe_sync_entry; @@ -97,7 +97,7 @@ struct dma_fence * xe_migrate_update_pgtables(struct xe_migrate *m, struct xe_vm *vm, struct xe_bo *bo, - struct xe_engine *eng, + struct xe_exec_queue *q, const struct xe_vm_pgtable_update *updates, u32 num_updates, struct xe_sync_entry *syncs, u32 num_syncs, @@ -105,5 +105,5 @@ xe_migrate_update_pgtables(struct xe_migrate *m, void xe_migrate_wait(struct xe_migrate *m); -struct xe_engine *xe_tile_migrate_engine(struct xe_tile *tile); +struct xe_exec_queue *xe_tile_migrate_engine(struct xe_tile *tile); #endif diff --git a/drivers/gpu/drm/xe/xe_mocs.h b/drivers/gpu/drm/xe/xe_mocs.h index 25f7b35a76da..d0f1ec4b0336 100644 --- a/drivers/gpu/drm/xe/xe_mocs.h +++ b/drivers/gpu/drm/xe/xe_mocs.h @@ -8,7 +8,7 @@ #include -struct xe_engine; +struct xe_exec_queue; struct xe_gt; void xe_mocs_init_early(struct xe_gt *gt); diff --git a/drivers/gpu/drm/xe/xe_preempt_fence.c b/drivers/gpu/drm/xe/xe_preempt_fence.c index e86604e0174d..7bce2a332603 100644 --- a/drivers/gpu/drm/xe/xe_preempt_fence.c +++ b/drivers/gpu/drm/xe/xe_preempt_fence.c @@ -15,19 +15,19 @@ static void preempt_fence_work_func(struct work_struct *w) bool cookie = dma_fence_begin_signalling(); struct xe_preempt_fence *pfence = container_of(w, typeof(*pfence), preempt_work); - struct xe_engine *e = pfence->engine; + struct xe_exec_queue *q = pfence->q; if (pfence->error) dma_fence_set_error(&pfence->base, pfence->error); else - e->ops->suspend_wait(e); + q->ops->suspend_wait(q); dma_fence_signal(&pfence->base); dma_fence_end_signalling(cookie); - xe_vm_queue_rebind_worker(e->vm); + xe_vm_queue_rebind_worker(q->vm); - xe_engine_put(e); + xe_exec_queue_put(q); } static const char * @@ -46,9 +46,9 @@ static bool preempt_fence_enable_signaling(struct dma_fence *fence) { struct xe_preempt_fence *pfence = container_of(fence, typeof(*pfence), base); - struct xe_engine *e = pfence->engine; + struct xe_exec_queue *q = pfence->q; - pfence->error = e->ops->suspend(e); + pfence->error = q->ops->suspend(q); queue_work(system_unbound_wq, &pfence->preempt_work); return true; } @@ -104,43 +104,43 @@ void xe_preempt_fence_free(struct xe_preempt_fence *pfence) * xe_preempt_fence_alloc(). * @pfence: The struct xe_preempt_fence pointer returned from * xe_preempt_fence_alloc(). - * @e: The struct xe_engine used for arming. + * @q: The struct xe_exec_queue used for arming. * @context: The dma-fence context used for arming. * @seqno: The dma-fence seqno used for arming. * * Inserts the preempt fence into @context's timeline, takes @link off any - * list, and registers the struct xe_engine as the xe_engine to be preempted. + * list, and registers the struct xe_exec_queue as the xe_engine to be preempted. * * Return: A pointer to a struct dma_fence embedded into the preempt fence. * This function doesn't error. */ struct dma_fence * -xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_engine *e, +xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_exec_queue *q, u64 context, u32 seqno) { list_del_init(&pfence->link); - pfence->engine = xe_engine_get(e); + pfence->q = xe_exec_queue_get(q); dma_fence_init(&pfence->base, &preempt_fence_ops, - &e->compute.lock, context, seqno); + &q->compute.lock, context, seqno); return &pfence->base; } /** * xe_preempt_fence_create() - Helper to create and arm a preempt fence. - * @e: The struct xe_engine used for arming. + * @q: The struct xe_exec_queue used for arming. * @context: The dma-fence context used for arming. * @seqno: The dma-fence seqno used for arming. * * Allocates and inserts the preempt fence into @context's timeline, - * and registers @e as the struct xe_engine to be preempted. + * and registers @e as the struct xe_exec_queue to be preempted. * * Return: A pointer to the resulting struct dma_fence on success. An error * pointer on error. In particular if allocation fails it returns * ERR_PTR(-ENOMEM); */ struct dma_fence * -xe_preempt_fence_create(struct xe_engine *e, +xe_preempt_fence_create(struct xe_exec_queue *q, u64 context, u32 seqno) { struct xe_preempt_fence *pfence; @@ -149,7 +149,7 @@ xe_preempt_fence_create(struct xe_engine *e, if (IS_ERR(pfence)) return ERR_CAST(pfence); - return xe_preempt_fence_arm(pfence, e, context, seqno); + return xe_preempt_fence_arm(pfence, q, context, seqno); } bool xe_fence_is_xe_preempt(const struct dma_fence *fence) diff --git a/drivers/gpu/drm/xe/xe_preempt_fence.h b/drivers/gpu/drm/xe/xe_preempt_fence.h index 4f3966103203..9406c6fea525 100644 --- a/drivers/gpu/drm/xe/xe_preempt_fence.h +++ b/drivers/gpu/drm/xe/xe_preempt_fence.h @@ -11,7 +11,7 @@ struct list_head; struct dma_fence * -xe_preempt_fence_create(struct xe_engine *e, +xe_preempt_fence_create(struct xe_exec_queue *q, u64 context, u32 seqno); struct xe_preempt_fence *xe_preempt_fence_alloc(void); @@ -19,7 +19,7 @@ struct xe_preempt_fence *xe_preempt_fence_alloc(void); void xe_preempt_fence_free(struct xe_preempt_fence *pfence); struct dma_fence * -xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_engine *e, +xe_preempt_fence_arm(struct xe_preempt_fence *pfence, struct xe_exec_queue *q, u64 context, u32 seqno); static inline struct xe_preempt_fence * diff --git a/drivers/gpu/drm/xe/xe_preempt_fence_types.h b/drivers/gpu/drm/xe/xe_preempt_fence_types.h index 9d9efd8ff0ed..b54b5c29b533 100644 --- a/drivers/gpu/drm/xe/xe_preempt_fence_types.h +++ b/drivers/gpu/drm/xe/xe_preempt_fence_types.h @@ -9,12 +9,11 @@ #include #include -struct xe_engine; +struct xe_exec_queue; /** * struct xe_preempt_fence - XE preempt fence * - * A preemption fence which suspends the execution of an xe_engine on the * hardware and triggers a callback once the xe_engine is complete. */ struct xe_preempt_fence { @@ -22,8 +21,8 @@ struct xe_preempt_fence { struct dma_fence base; /** @link: link into list of pending preempt fences */ struct list_head link; - /** @engine: xe engine for this preempt fence */ - struct xe_engine *engine; + /** @q: exec queue for this preempt fence */ + struct xe_exec_queue *q; /** @preempt_work: work struct which issues preemption */ struct work_struct preempt_work; /** @error: preempt fence is in error state */ diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index b82ce01cc4cb..c21d2681b419 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -1307,7 +1307,7 @@ static void xe_pt_calc_rfence_interval(struct xe_vma *vma, * address range. * @tile: The tile to bind for. * @vma: The vma to bind. - * @e: The engine with which to do pipelined page-table updates. + * @q: The exec_queue with which to do pipelined page-table updates. * @syncs: Entries to sync on before binding the built tree to the live vm tree. * @num_syncs: Number of @sync entries. * @rebind: Whether we're rebinding this vma to the same address range without @@ -1325,7 +1325,7 @@ static void xe_pt_calc_rfence_interval(struct xe_vma *vma, * on success, an error pointer on error. */ struct dma_fence * -__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, +__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs, bool rebind) { @@ -1351,7 +1351,7 @@ __xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, vm_dbg(&xe_vma_vm(vma)->xe->drm, "Preparing bind, with range [%llx...%llx) engine %p.\n", - xe_vma_start(vma), xe_vma_end(vma) - 1, e); + xe_vma_start(vma), xe_vma_end(vma) - 1, q); err = xe_pt_prepare_bind(tile, vma, entries, &num_entries, rebind); if (err) @@ -1388,7 +1388,7 @@ __xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, } fence = xe_migrate_update_pgtables(tile->migrate, - vm, xe_vma_bo(vma), e, + vm, xe_vma_bo(vma), q, entries, num_entries, syncs, num_syncs, &bind_pt_update.base); @@ -1663,7 +1663,7 @@ static const struct xe_migrate_pt_update_ops userptr_unbind_ops = { * address range. * @tile: The tile to unbind for. * @vma: The vma to unbind. - * @e: The engine with which to do pipelined page-table updates. + * @q: The exec_queue with which to do pipelined page-table updates. * @syncs: Entries to sync on before disconnecting the tree to be destroyed. * @num_syncs: Number of @sync entries. * @@ -1679,7 +1679,7 @@ static const struct xe_migrate_pt_update_ops userptr_unbind_ops = { * on success, an error pointer on error. */ struct dma_fence * -__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, +__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs) { struct xe_vm_pgtable_update entries[XE_VM_MAX_LEVEL * 2 + 1]; @@ -1704,7 +1704,7 @@ __xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e vm_dbg(&xe_vma_vm(vma)->xe->drm, "Preparing unbind, with range [%llx...%llx) engine %p.\n", - xe_vma_start(vma), xe_vma_end(vma) - 1, e); + xe_vma_start(vma), xe_vma_end(vma) - 1, q); num_entries = xe_pt_stage_unbind(tile, vma, entries); XE_WARN_ON(num_entries > ARRAY_SIZE(entries)); @@ -1729,8 +1729,8 @@ __xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e * lower level, because it needs to be more conservative. */ fence = xe_migrate_update_pgtables(tile->migrate, - vm, NULL, e ? e : - vm->eng[tile->id], + vm, NULL, q ? q : + vm->q[tile->id], entries, num_entries, syncs, num_syncs, &unbind_pt_update.base); diff --git a/drivers/gpu/drm/xe/xe_pt.h b/drivers/gpu/drm/xe/xe_pt.h index bbb00d6461ff..01be7ab08f87 100644 --- a/drivers/gpu/drm/xe/xe_pt.h +++ b/drivers/gpu/drm/xe/xe_pt.h @@ -12,7 +12,7 @@ struct dma_fence; struct xe_bo; struct xe_device; -struct xe_engine; +struct xe_exec_queue; struct xe_sync_entry; struct xe_tile; struct xe_vm; @@ -35,12 +35,12 @@ void xe_pt_populate_empty(struct xe_tile *tile, struct xe_vm *vm, void xe_pt_destroy(struct xe_pt *pt, u32 flags, struct llist_head *deferred); struct dma_fence * -__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, +__xe_pt_bind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs, bool rebind); struct dma_fence * -__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_engine *e, +__xe_pt_unbind_vma(struct xe_tile *tile, struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs); bool xe_pt_zap_ptes(struct xe_tile *tile, struct xe_vma *vma); diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 382851f436b7..7ea235c71385 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -203,7 +203,7 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) config->info[XE_QUERY_CONFIG_MEM_REGION_COUNT] = hweight_long(xe->info.mem_region_mask); config->info[XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY] = - xe_engine_device_get_max_priority(xe); + xe_exec_queue_device_get_max_priority(xe); if (copy_to_user(query_ptr, config, size)) { kfree(config); diff --git a/drivers/gpu/drm/xe/xe_ring_ops.c b/drivers/gpu/drm/xe/xe_ring_ops.c index 2d0d392cd691..6346ed24e279 100644 --- a/drivers/gpu/drm/xe/xe_ring_ops.c +++ b/drivers/gpu/drm/xe/xe_ring_ops.c @@ -10,7 +10,7 @@ #include "regs/xe_gt_regs.h" #include "regs/xe_lrc_layout.h" #include "regs/xe_regs.h" -#include "xe_engine_types.h" +#include "xe_exec_queue_types.h" #include "xe_gt.h" #include "xe_lrc.h" #include "xe_macros.h" @@ -156,7 +156,7 @@ static int emit_store_imm_ppgtt_posted(u64 addr, u64 value, static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) { - struct xe_gt *gt = job->engine->gt; + struct xe_gt *gt = job->q->gt; bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); u32 flags; @@ -172,7 +172,7 @@ static int emit_render_cache_flush(struct xe_sched_job *job, u32 *dw, int i) if (lacks_render) flags &= ~PIPE_CONTROL_3D_ARCH_FLAGS; - else if (job->engine->class == XE_ENGINE_CLASS_COMPUTE) + else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) flags &= ~PIPE_CONTROL_3D_ENGINE_FLAGS; dw[i++] = GFX_OP_PIPE_CONTROL(6) | PIPE_CONTROL0_HDC_PIPELINE_FLUSH; @@ -202,7 +202,7 @@ static int emit_pipe_imm_ggtt(u32 addr, u32 value, bool stall_only, u32 *dw, static u32 get_ppgtt_flag(struct xe_sched_job *job) { - return !(job->engine->flags & ENGINE_FLAG_WA) ? BIT(8) : 0; + return !(job->q->flags & EXEC_QUEUE_FLAG_WA) ? BIT(8) : 0; } static void __emit_job_gen12_copy(struct xe_sched_job *job, struct xe_lrc *lrc, @@ -210,7 +210,7 @@ static void __emit_job_gen12_copy(struct xe_sched_job *job, struct xe_lrc *lrc, { u32 dw[MAX_JOB_SIZE_DW], i = 0; u32 ppgtt_flag = get_ppgtt_flag(job); - struct xe_vm *vm = job->engine->vm; + struct xe_vm *vm = job->q->vm; if (vm->batch_invalidate_tlb) { dw[i++] = preparser_disable(true); @@ -255,10 +255,10 @@ static void __emit_job_gen12_video(struct xe_sched_job *job, struct xe_lrc *lrc, { u32 dw[MAX_JOB_SIZE_DW], i = 0; u32 ppgtt_flag = get_ppgtt_flag(job); - struct xe_gt *gt = job->engine->gt; + struct xe_gt *gt = job->q->gt; struct xe_device *xe = gt_to_xe(gt); - bool decode = job->engine->class == XE_ENGINE_CLASS_VIDEO_DECODE; - struct xe_vm *vm = job->engine->vm; + bool decode = job->q->class == XE_ENGINE_CLASS_VIDEO_DECODE; + struct xe_vm *vm = job->q->vm; dw[i++] = preparser_disable(true); @@ -302,16 +302,16 @@ static void __emit_job_gen12_render_compute(struct xe_sched_job *job, { u32 dw[MAX_JOB_SIZE_DW], i = 0; u32 ppgtt_flag = get_ppgtt_flag(job); - struct xe_gt *gt = job->engine->gt; + struct xe_gt *gt = job->q->gt; struct xe_device *xe = gt_to_xe(gt); bool lacks_render = !(gt->info.engine_mask & XE_HW_ENGINE_RCS_MASK); - struct xe_vm *vm = job->engine->vm; + struct xe_vm *vm = job->q->vm; u32 mask_flags = 0; dw[i++] = preparser_disable(true); if (lacks_render) mask_flags = PIPE_CONTROL_3D_ARCH_FLAGS; - else if (job->engine->class == XE_ENGINE_CLASS_COMPUTE) + else if (job->q->class == XE_ENGINE_CLASS_COMPUTE) mask_flags = PIPE_CONTROL_3D_ENGINE_FLAGS; /* See __xe_pt_bind_vma() for a discussion on TLB invalidations. */ @@ -378,14 +378,14 @@ static void emit_job_gen12_copy(struct xe_sched_job *job) { int i; - if (xe_sched_job_is_migration(job->engine)) { - emit_migration_job_gen12(job, job->engine->lrc, + if (xe_sched_job_is_migration(job->q)) { + emit_migration_job_gen12(job, job->q->lrc, xe_sched_job_seqno(job)); return; } - for (i = 0; i < job->engine->width; ++i) - __emit_job_gen12_copy(job, job->engine->lrc + i, + for (i = 0; i < job->q->width; ++i) + __emit_job_gen12_copy(job, job->q->lrc + i, job->batch_addr[i], xe_sched_job_seqno(job)); } @@ -395,8 +395,8 @@ static void emit_job_gen12_video(struct xe_sched_job *job) int i; /* FIXME: Not doing parallel handshake for now */ - for (i = 0; i < job->engine->width; ++i) - __emit_job_gen12_video(job, job->engine->lrc + i, + for (i = 0; i < job->q->width; ++i) + __emit_job_gen12_video(job, job->q->lrc + i, job->batch_addr[i], xe_sched_job_seqno(job)); } @@ -405,8 +405,8 @@ static void emit_job_gen12_render_compute(struct xe_sched_job *job) { int i; - for (i = 0; i < job->engine->width; ++i) - __emit_job_gen12_render_compute(job, job->engine->lrc + i, + for (i = 0; i < job->q->width; ++i) + __emit_job_gen12_render_compute(job, job->q->lrc + i, job->batch_addr[i], xe_sched_job_seqno(job)); } diff --git a/drivers/gpu/drm/xe/xe_sched_job.c b/drivers/gpu/drm/xe/xe_sched_job.c index 9944858de4d2..de2851d24c96 100644 --- a/drivers/gpu/drm/xe/xe_sched_job.c +++ b/drivers/gpu/drm/xe/xe_sched_job.c @@ -57,58 +57,58 @@ static struct xe_sched_job *job_alloc(bool parallel) xe_sched_job_slab, GFP_KERNEL); } -bool xe_sched_job_is_migration(struct xe_engine *e) +bool xe_sched_job_is_migration(struct xe_exec_queue *q) { - return e->vm && (e->vm->flags & XE_VM_FLAG_MIGRATION) && - !(e->flags & ENGINE_FLAG_WA); + return q->vm && (q->vm->flags & XE_VM_FLAG_MIGRATION) && + !(q->flags & EXEC_QUEUE_FLAG_WA); } static void job_free(struct xe_sched_job *job) { - struct xe_engine *e = job->engine; - bool is_migration = xe_sched_job_is_migration(e); + struct xe_exec_queue *q = job->q; + bool is_migration = xe_sched_job_is_migration(q); - kmem_cache_free(xe_engine_is_parallel(job->engine) || is_migration ? + kmem_cache_free(xe_exec_queue_is_parallel(job->q) || is_migration ? xe_sched_job_parallel_slab : xe_sched_job_slab, job); } static struct xe_device *job_to_xe(struct xe_sched_job *job) { - return gt_to_xe(job->engine->gt); + return gt_to_xe(job->q->gt); } -struct xe_sched_job *xe_sched_job_create(struct xe_engine *e, +struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q, u64 *batch_addr) { struct xe_sched_job *job; struct dma_fence **fences; - bool is_migration = xe_sched_job_is_migration(e); + bool is_migration = xe_sched_job_is_migration(q); int err; int i, j; u32 width; /* Migration and kernel engines have their own locking */ - if (!(e->flags & (ENGINE_FLAG_KERNEL | ENGINE_FLAG_VM | - ENGINE_FLAG_WA))) { - lockdep_assert_held(&e->vm->lock); - if (!xe_vm_no_dma_fences(e->vm)) - xe_vm_assert_held(e->vm); + if (!(q->flags & (EXEC_QUEUE_FLAG_KERNEL | EXEC_QUEUE_FLAG_VM | + EXEC_QUEUE_FLAG_WA))) { + lockdep_assert_held(&q->vm->lock); + if (!xe_vm_no_dma_fences(q->vm)) + xe_vm_assert_held(q->vm); } - job = job_alloc(xe_engine_is_parallel(e) || is_migration); + job = job_alloc(xe_exec_queue_is_parallel(q) || is_migration); if (!job) return ERR_PTR(-ENOMEM); - job->engine = e; + job->q = q; kref_init(&job->refcount); - xe_engine_get(job->engine); + xe_exec_queue_get(job->q); - err = drm_sched_job_init(&job->drm, e->entity, 1, NULL); + err = drm_sched_job_init(&job->drm, q->entity, 1, NULL); if (err) goto err_free; - if (!xe_engine_is_parallel(e)) { - job->fence = xe_lrc_create_seqno_fence(e->lrc); + if (!xe_exec_queue_is_parallel(q)) { + job->fence = xe_lrc_create_seqno_fence(q->lrc); if (IS_ERR(job->fence)) { err = PTR_ERR(job->fence); goto err_sched_job; @@ -116,38 +116,38 @@ struct xe_sched_job *xe_sched_job_create(struct xe_engine *e, } else { struct dma_fence_array *cf; - fences = kmalloc_array(e->width, sizeof(*fences), GFP_KERNEL); + fences = kmalloc_array(q->width, sizeof(*fences), GFP_KERNEL); if (!fences) { err = -ENOMEM; goto err_sched_job; } - for (j = 0; j < e->width; ++j) { - fences[j] = xe_lrc_create_seqno_fence(e->lrc + j); + for (j = 0; j < q->width; ++j) { + fences[j] = xe_lrc_create_seqno_fence(q->lrc + j); if (IS_ERR(fences[j])) { err = PTR_ERR(fences[j]); goto err_fences; } } - cf = dma_fence_array_create(e->width, fences, - e->parallel.composite_fence_ctx, - e->parallel.composite_fence_seqno++, + cf = dma_fence_array_create(q->width, fences, + q->parallel.composite_fence_ctx, + q->parallel.composite_fence_seqno++, false); if (!cf) { - --e->parallel.composite_fence_seqno; + --q->parallel.composite_fence_seqno; err = -ENOMEM; goto err_fences; } /* Sanity check */ - for (j = 0; j < e->width; ++j) + for (j = 0; j < q->width; ++j) XE_WARN_ON(cf->base.seqno != fences[j]->seqno); job->fence = &cf->base; } - width = e->width; + width = q->width; if (is_migration) width = 2; @@ -155,7 +155,7 @@ struct xe_sched_job *xe_sched_job_create(struct xe_engine *e, job->batch_addr[i] = batch_addr[i]; /* All other jobs require a VM to be open which has a ref */ - if (unlikely(e->flags & ENGINE_FLAG_KERNEL)) + if (unlikely(q->flags & EXEC_QUEUE_FLAG_KERNEL)) xe_device_mem_access_get(job_to_xe(job)); xe_device_assert_mem_access(job_to_xe(job)); @@ -164,14 +164,14 @@ struct xe_sched_job *xe_sched_job_create(struct xe_engine *e, err_fences: for (j = j - 1; j >= 0; --j) { - --e->lrc[j].fence_ctx.next_seqno; + --q->lrc[j].fence_ctx.next_seqno; dma_fence_put(fences[j]); } kfree(fences); err_sched_job: drm_sched_job_cleanup(&job->drm); err_free: - xe_engine_put(e); + xe_exec_queue_put(q); job_free(job); return ERR_PTR(err); } @@ -188,9 +188,9 @@ void xe_sched_job_destroy(struct kref *ref) struct xe_sched_job *job = container_of(ref, struct xe_sched_job, refcount); - if (unlikely(job->engine->flags & ENGINE_FLAG_KERNEL)) + if (unlikely(job->q->flags & EXEC_QUEUE_FLAG_KERNEL)) xe_device_mem_access_put(job_to_xe(job)); - xe_engine_put(job->engine); + xe_exec_queue_put(job->q); dma_fence_put(job->fence); drm_sched_job_cleanup(&job->drm); job_free(job); @@ -222,12 +222,12 @@ void xe_sched_job_set_error(struct xe_sched_job *job, int error) trace_xe_sched_job_set_error(job); dma_fence_enable_sw_signaling(job->fence); - xe_hw_fence_irq_run(job->engine->fence_irq); + xe_hw_fence_irq_run(job->q->fence_irq); } bool xe_sched_job_started(struct xe_sched_job *job) { - struct xe_lrc *lrc = job->engine->lrc; + struct xe_lrc *lrc = job->q->lrc; return !__dma_fence_is_later(xe_sched_job_seqno(job), xe_lrc_start_seqno(lrc), @@ -236,7 +236,7 @@ bool xe_sched_job_started(struct xe_sched_job *job) bool xe_sched_job_completed(struct xe_sched_job *job) { - struct xe_lrc *lrc = job->engine->lrc; + struct xe_lrc *lrc = job->q->lrc; /* * Can safely check just LRC[0] seqno as that is last seqno written when diff --git a/drivers/gpu/drm/xe/xe_sched_job.h b/drivers/gpu/drm/xe/xe_sched_job.h index 5315ad8656c2..6ca1d426c036 100644 --- a/drivers/gpu/drm/xe/xe_sched_job.h +++ b/drivers/gpu/drm/xe/xe_sched_job.h @@ -14,7 +14,7 @@ int xe_sched_job_module_init(void); void xe_sched_job_module_exit(void); -struct xe_sched_job *xe_sched_job_create(struct xe_engine *e, +struct xe_sched_job *xe_sched_job_create(struct xe_exec_queue *q, u64 *batch_addr); void xe_sched_job_destroy(struct kref *ref); @@ -71,6 +71,6 @@ xe_sched_job_add_migrate_flush(struct xe_sched_job *job, u32 flags) job->migrate_flush_flags = flags; } -bool xe_sched_job_is_migration(struct xe_engine *e); +bool xe_sched_job_is_migration(struct xe_exec_queue *q); #endif diff --git a/drivers/gpu/drm/xe/xe_sched_job_types.h b/drivers/gpu/drm/xe/xe_sched_job_types.h index 5534bfacaa16..71213ba9735b 100644 --- a/drivers/gpu/drm/xe/xe_sched_job_types.h +++ b/drivers/gpu/drm/xe/xe_sched_job_types.h @@ -10,7 +10,7 @@ #include -struct xe_engine; +struct xe_exec_queue; /** * struct xe_sched_job - XE schedule job (batch buffer tracking) @@ -18,8 +18,8 @@ struct xe_engine; struct xe_sched_job { /** @drm: base DRM scheduler job */ struct drm_sched_job drm; - /** @engine: XE submission engine */ - struct xe_engine *engine; + /** @q: Exec queue */ + struct xe_exec_queue *q; /** @refcount: ref count of this job */ struct kref refcount; /** diff --git a/drivers/gpu/drm/xe/xe_trace.h b/drivers/gpu/drm/xe/xe_trace.h index 82ca25d8d017..5ea458dadf69 100644 --- a/drivers/gpu/drm/xe/xe_trace.h +++ b/drivers/gpu/drm/xe/xe_trace.h @@ -13,11 +13,11 @@ #include #include "xe_bo_types.h" -#include "xe_engine_types.h" +#include "xe_exec_queue_types.h" #include "xe_gpu_scheduler_types.h" #include "xe_gt_tlb_invalidation_types.h" #include "xe_gt_types.h" -#include "xe_guc_engine_types.h" +#include "xe_guc_exec_queue_types.h" #include "xe_sched_job.h" #include "xe_vm.h" @@ -105,9 +105,9 @@ DEFINE_EVENT(xe_bo, xe_bo_move, TP_ARGS(bo) ); -DECLARE_EVENT_CLASS(xe_engine, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e), +DECLARE_EVENT_CLASS(xe_exec_queue, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q), TP_STRUCT__entry( __field(enum xe_engine_class, class) @@ -120,13 +120,13 @@ DECLARE_EVENT_CLASS(xe_engine, ), TP_fast_assign( - __entry->class = e->class; - __entry->logical_mask = e->logical_mask; - __entry->gt_id = e->gt->info.id; - __entry->width = e->width; - __entry->guc_id = e->guc->id; - __entry->guc_state = atomic_read(&e->guc->state); - __entry->flags = e->flags; + __entry->class = q->class; + __entry->logical_mask = q->logical_mask; + __entry->gt_id = q->gt->info.id; + __entry->width = q->width; + __entry->guc_id = q->guc->id; + __entry->guc_state = atomic_read(&q->guc->state); + __entry->flags = q->flags; ), TP_printk("%d:0x%x, gt=%d, width=%d, guc_id=%d, guc_state=0x%x, flags=0x%x", @@ -135,94 +135,94 @@ DECLARE_EVENT_CLASS(xe_engine, __entry->guc_state, __entry->flags) ); -DEFINE_EVENT(xe_engine, xe_engine_create, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_create, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_supress_resume, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_supress_resume, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_submit, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_submit, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_scheduling_enable, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_scheduling_enable, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_scheduling_disable, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_scheduling_disable, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_scheduling_done, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_scheduling_done, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_register, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_register, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_deregister, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_deregister, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_deregister_done, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_deregister_done, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_close, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_close, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_kill, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_kill, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_cleanup_entity, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_cleanup_entity, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_destroy, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_destroy, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_reset, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_reset, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_memory_cat_error, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_memory_cat_error, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_stop, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_stop, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_resubmit, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_resubmit, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); -DEFINE_EVENT(xe_engine, xe_engine_lr_cleanup, - TP_PROTO(struct xe_engine *e), - TP_ARGS(e) +DEFINE_EVENT(xe_exec_queue, xe_exec_queue_lr_cleanup, + TP_PROTO(struct xe_exec_queue *q), + TP_ARGS(q) ); DECLARE_EVENT_CLASS(xe_sched_job, @@ -241,10 +241,10 @@ DECLARE_EVENT_CLASS(xe_sched_job, TP_fast_assign( __entry->seqno = xe_sched_job_seqno(job); - __entry->guc_id = job->engine->guc->id; + __entry->guc_id = job->q->guc->id; __entry->guc_state = - atomic_read(&job->engine->guc->state); - __entry->flags = job->engine->flags; + atomic_read(&job->q->guc->state); + __entry->flags = job->q->flags; __entry->error = job->fence->error; __entry->fence = (unsigned long)job->fence; __entry->batch_addr = (u64)job->batch_addr[0]; @@ -303,7 +303,7 @@ DECLARE_EVENT_CLASS(xe_sched_msg, TP_fast_assign( __entry->opcode = msg->opcode; __entry->guc_id = - ((struct xe_engine *)msg->private_data)->guc->id; + ((struct xe_exec_queue *)msg->private_data)->guc->id; ), TP_printk("guc_id=%d, opcode=%u", __entry->guc_id, diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index d3e82c4aed42..374f111eea9c 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -165,15 +165,15 @@ out: static bool preempt_fences_waiting(struct xe_vm *vm) { - struct xe_engine *e; + struct xe_exec_queue *q; lockdep_assert_held(&vm->lock); xe_vm_assert_held(vm); - list_for_each_entry(e, &vm->preempt.engines, compute.link) { - if (!e->compute.pfence || (e->compute.pfence && - test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, - &e->compute.pfence->flags))) { + list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { + if (!q->compute.pfence || + (q->compute.pfence && test_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT, + &q->compute.pfence->flags))) { return true; } } @@ -195,10 +195,10 @@ static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list, lockdep_assert_held(&vm->lock); xe_vm_assert_held(vm); - if (*count >= vm->preempt.num_engines) + if (*count >= vm->preempt.num_exec_queues) return 0; - for (; *count < vm->preempt.num_engines; ++(*count)) { + for (; *count < vm->preempt.num_exec_queues; ++(*count)) { struct xe_preempt_fence *pfence = xe_preempt_fence_alloc(); if (IS_ERR(pfence)) @@ -212,18 +212,18 @@ static int alloc_preempt_fences(struct xe_vm *vm, struct list_head *list, static int wait_for_existing_preempt_fences(struct xe_vm *vm) { - struct xe_engine *e; + struct xe_exec_queue *q; xe_vm_assert_held(vm); - list_for_each_entry(e, &vm->preempt.engines, compute.link) { - if (e->compute.pfence) { - long timeout = dma_fence_wait(e->compute.pfence, false); + list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { + if (q->compute.pfence) { + long timeout = dma_fence_wait(q->compute.pfence, false); if (timeout < 0) return -ETIME; - dma_fence_put(e->compute.pfence); - e->compute.pfence = NULL; + dma_fence_put(q->compute.pfence); + q->compute.pfence = NULL; } } @@ -232,11 +232,11 @@ static int wait_for_existing_preempt_fences(struct xe_vm *vm) static bool xe_vm_is_idle(struct xe_vm *vm) { - struct xe_engine *e; + struct xe_exec_queue *q; xe_vm_assert_held(vm); - list_for_each_entry(e, &vm->preempt.engines, compute.link) { - if (!xe_engine_is_idle(e)) + list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { + if (!xe_exec_queue_is_idle(q)) return false; } @@ -246,36 +246,36 @@ static bool xe_vm_is_idle(struct xe_vm *vm) static void arm_preempt_fences(struct xe_vm *vm, struct list_head *list) { struct list_head *link; - struct xe_engine *e; + struct xe_exec_queue *q; - list_for_each_entry(e, &vm->preempt.engines, compute.link) { + list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { struct dma_fence *fence; link = list->next; XE_WARN_ON(link == list); fence = xe_preempt_fence_arm(to_preempt_fence_from_link(link), - e, e->compute.context, - ++e->compute.seqno); - dma_fence_put(e->compute.pfence); - e->compute.pfence = fence; + q, q->compute.context, + ++q->compute.seqno); + dma_fence_put(q->compute.pfence); + q->compute.pfence = fence; } } static int add_preempt_fences(struct xe_vm *vm, struct xe_bo *bo) { - struct xe_engine *e; + struct xe_exec_queue *q; struct ww_acquire_ctx ww; int err; - err = xe_bo_lock(bo, &ww, vm->preempt.num_engines, true); + err = xe_bo_lock(bo, &ww, vm->preempt.num_exec_queues, true); if (err) return err; - list_for_each_entry(e, &vm->preempt.engines, compute.link) - if (e->compute.pfence) { + list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) + if (q->compute.pfence) { dma_resv_add_fence(bo->ttm.base.resv, - e->compute.pfence, + q->compute.pfence, DMA_RESV_USAGE_BOOKKEEP); } @@ -304,22 +304,22 @@ void xe_vm_fence_all_extobjs(struct xe_vm *vm, struct dma_fence *fence, static void resume_and_reinstall_preempt_fences(struct xe_vm *vm) { - struct xe_engine *e; + struct xe_exec_queue *q; lockdep_assert_held(&vm->lock); xe_vm_assert_held(vm); - list_for_each_entry(e, &vm->preempt.engines, compute.link) { - e->ops->resume(e); + list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) { + q->ops->resume(q); - dma_resv_add_fence(xe_vm_resv(vm), e->compute.pfence, + dma_resv_add_fence(xe_vm_resv(vm), q->compute.pfence, DMA_RESV_USAGE_BOOKKEEP); - xe_vm_fence_all_extobjs(vm, e->compute.pfence, + xe_vm_fence_all_extobjs(vm, q->compute.pfence, DMA_RESV_USAGE_BOOKKEEP); } } -int xe_vm_add_compute_engine(struct xe_vm *vm, struct xe_engine *e) +int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q) { struct ttm_validate_buffer tv_onstack[XE_ONSTACK_TV]; struct ttm_validate_buffer *tv; @@ -337,16 +337,16 @@ int xe_vm_add_compute_engine(struct xe_vm *vm, struct xe_engine *e) if (err) goto out_unlock_outer; - pfence = xe_preempt_fence_create(e, e->compute.context, - ++e->compute.seqno); + pfence = xe_preempt_fence_create(q, q->compute.context, + ++q->compute.seqno); if (!pfence) { err = -ENOMEM; goto out_unlock; } - list_add(&e->compute.link, &vm->preempt.engines); - ++vm->preempt.num_engines; - e->compute.pfence = pfence; + list_add(&q->compute.link, &vm->preempt.exec_queues); + ++vm->preempt.num_exec_queues; + q->compute.pfence = pfence; down_read(&vm->userptr.notifier_lock); @@ -518,7 +518,7 @@ void xe_vm_unlock_dma_resv(struct xe_vm *vm, static void xe_vm_kill(struct xe_vm *vm) { struct ww_acquire_ctx ww; - struct xe_engine *e; + struct xe_exec_queue *q; lockdep_assert_held(&vm->lock); @@ -526,8 +526,8 @@ static void xe_vm_kill(struct xe_vm *vm) vm->flags |= XE_VM_FLAG_BANNED; trace_xe_vm_kill(vm); - list_for_each_entry(e, &vm->preempt.engines, compute.link) - e->ops->kill(e); + list_for_each_entry(q, &vm->preempt.exec_queues, compute.link) + q->ops->kill(q); xe_vm_unlock(vm, &ww); /* TODO: Inform user the VM is banned */ @@ -584,7 +584,7 @@ retry: } err = xe_vm_lock_dma_resv(vm, &ww, tv_onstack, &tv, &objs, - false, vm->preempt.num_engines); + false, vm->preempt.num_exec_queues); if (err) goto out_unlock_outer; @@ -833,7 +833,7 @@ int xe_vm_userptr_check_repin(struct xe_vm *vm) } static struct dma_fence * -xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e, +xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs, bool first_op, bool last_op); @@ -1241,7 +1241,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) INIT_WORK(&vm->destroy_work, vm_destroy_work_func); - INIT_LIST_HEAD(&vm->preempt.engines); + INIT_LIST_HEAD(&vm->preempt.exec_queues); vm->preempt.min_run_period_ms = 10; /* FIXME: Wire up to uAPI */ for_each_tile(tile, xe, id) @@ -1320,21 +1320,21 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) for_each_tile(tile, xe, id) { struct xe_gt *gt = tile->primary_gt; struct xe_vm *migrate_vm; - struct xe_engine *eng; + struct xe_exec_queue *q; if (!vm->pt_root[id]) continue; migrate_vm = xe_migrate_get_vm(tile->migrate); - eng = xe_engine_create_class(xe, gt, migrate_vm, - XE_ENGINE_CLASS_COPY, - ENGINE_FLAG_VM); + q = xe_exec_queue_create_class(xe, gt, migrate_vm, + XE_ENGINE_CLASS_COPY, + EXEC_QUEUE_FLAG_VM); xe_vm_put(migrate_vm); - if (IS_ERR(eng)) { - err = PTR_ERR(eng); + if (IS_ERR(q)) { + err = PTR_ERR(q); goto err_close; } - vm->eng[id] = eng; + vm->q[id] = q; number_tiles++; } } @@ -1422,7 +1422,7 @@ void xe_vm_close_and_put(struct xe_vm *vm) struct drm_gpuva *gpuva, *next; u8 id; - XE_WARN_ON(vm->preempt.num_engines); + XE_WARN_ON(vm->preempt.num_exec_queues); xe_vm_close(vm); flush_async_ops(vm); @@ -1430,10 +1430,10 @@ void xe_vm_close_and_put(struct xe_vm *vm) flush_work(&vm->preempt.rebind_work); for_each_tile(tile, xe, id) { - if (vm->eng[id]) { - xe_engine_kill(vm->eng[id]); - xe_engine_put(vm->eng[id]); - vm->eng[id] = NULL; + if (vm->q[id]) { + xe_exec_queue_kill(vm->q[id]); + xe_exec_queue_put(vm->q[id]); + vm->q[id] = NULL; } } @@ -1573,7 +1573,7 @@ u64 xe_vm_pdp4_descriptor(struct xe_vm *vm, struct xe_tile *tile) } static struct dma_fence * -xe_vm_unbind_vma(struct xe_vma *vma, struct xe_engine *e, +xe_vm_unbind_vma(struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs, bool first_op, bool last_op) { @@ -1600,7 +1600,7 @@ xe_vm_unbind_vma(struct xe_vma *vma, struct xe_engine *e, if (!(vma->tile_present & BIT(id))) goto next; - fence = __xe_pt_unbind_vma(tile, vma, e, first_op ? syncs : NULL, + fence = __xe_pt_unbind_vma(tile, vma, q, first_op ? syncs : NULL, first_op ? num_syncs : 0); if (IS_ERR(fence)) { err = PTR_ERR(fence); @@ -1611,8 +1611,8 @@ xe_vm_unbind_vma(struct xe_vma *vma, struct xe_engine *e, fences[cur_fence++] = fence; next: - if (e && vm->pt_root[id] && !list_empty(&e->multi_gt_list)) - e = list_next_entry(e, multi_gt_list); + if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list)) + q = list_next_entry(q, multi_gt_list); } if (fences) { @@ -1648,7 +1648,7 @@ err_fences: } static struct dma_fence * -xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e, +xe_vm_bind_vma(struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs, bool first_op, bool last_op) { @@ -1675,7 +1675,7 @@ xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e, if (!(vma->tile_mask & BIT(id))) goto next; - fence = __xe_pt_bind_vma(tile, vma, e ? e : vm->eng[id], + fence = __xe_pt_bind_vma(tile, vma, q ? q : vm->q[id], first_op ? syncs : NULL, first_op ? num_syncs : 0, vma->tile_present & BIT(id)); @@ -1688,8 +1688,8 @@ xe_vm_bind_vma(struct xe_vma *vma, struct xe_engine *e, fences[cur_fence++] = fence; next: - if (e && vm->pt_root[id] && !list_empty(&e->multi_gt_list)) - e = list_next_entry(e, multi_gt_list); + if (q && vm->pt_root[id] && !list_empty(&q->multi_gt_list)) + q = list_next_entry(q, multi_gt_list); } if (fences) { @@ -1805,7 +1805,7 @@ int xe_vm_async_fence_wait_start(struct dma_fence *fence) } static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, - struct xe_engine *e, struct xe_sync_entry *syncs, + struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs, struct async_op_fence *afence, bool immediate, bool first_op, bool last_op) { @@ -1814,7 +1814,7 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, xe_vm_assert_held(vm); if (immediate) { - fence = xe_vm_bind_vma(vma, e, syncs, num_syncs, first_op, + fence = xe_vm_bind_vma(vma, q, syncs, num_syncs, first_op, last_op); if (IS_ERR(fence)) return PTR_ERR(fence); @@ -1836,7 +1836,7 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, return 0; } -static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_engine *e, +static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q, struct xe_bo *bo, struct xe_sync_entry *syncs, u32 num_syncs, struct async_op_fence *afence, bool immediate, bool first_op, bool last_op) @@ -1852,12 +1852,12 @@ static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_engine *e, return err; } - return __xe_vm_bind(vm, vma, e, syncs, num_syncs, afence, immediate, + return __xe_vm_bind(vm, vma, q, syncs, num_syncs, afence, immediate, first_op, last_op); } static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, - struct xe_engine *e, struct xe_sync_entry *syncs, + struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs, struct async_op_fence *afence, bool first_op, bool last_op) { @@ -1866,7 +1866,7 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, xe_vm_assert_held(vm); xe_bo_assert_held(xe_vma_bo(vma)); - fence = xe_vm_unbind_vma(vma, e, syncs, num_syncs, first_op, last_op); + fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op); if (IS_ERR(fence)) return PTR_ERR(fence); if (afence) @@ -2074,7 +2074,7 @@ int xe_vm_destroy_ioctl(struct drm_device *dev, void *data, vm = xa_load(&xef->vm.xa, args->vm_id); if (XE_IOCTL_DBG(xe, !vm)) err = -ENOENT; - else if (XE_IOCTL_DBG(xe, vm->preempt.num_engines)) + else if (XE_IOCTL_DBG(xe, vm->preempt.num_exec_queues)) err = -EBUSY; else xa_erase(&xef->vm.xa, args->vm_id); @@ -2093,7 +2093,7 @@ static const u32 region_to_mem_type[] = { }; static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, - struct xe_engine *e, u32 region, + struct xe_exec_queue *q, u32 region, struct xe_sync_entry *syncs, u32 num_syncs, struct async_op_fence *afence, bool first_op, bool last_op) @@ -2109,7 +2109,7 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, } if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) { - return xe_vm_bind(vm, vma, e, xe_vma_bo(vma), syncs, num_syncs, + return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs, afence, true, first_op, last_op); } else { int i; @@ -2414,7 +2414,7 @@ static u64 xe_vma_max_pte_size(struct xe_vma *vma) * Parse operations list and create any resources needed for the operations * prior to fully committing to the operations. This setup can fail. */ -static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_engine *e, +static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, struct drm_gpuva_ops **ops, int num_ops_list, struct xe_sync_entry *syncs, u32 num_syncs, struct list_head *ops_list, bool async) @@ -2434,9 +2434,9 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_engine *e, if (!fence) return -ENOMEM; - seqno = e ? ++e->bind.fence_seqno : ++vm->async_ops.fence.seqno; + seqno = q ? ++q->bind.fence_seqno : ++vm->async_ops.fence.seqno; dma_fence_init(&fence->fence, &async_op_fence_ops, - &vm->async_ops.lock, e ? e->bind.fence_ctx : + &vm->async_ops.lock, q ? q->bind.fence_ctx : vm->async_ops.fence.context, seqno); if (!xe_vm_no_dma_fences(vm)) { @@ -2467,7 +2467,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_engine *e, op->syncs = syncs; } - op->engine = e; + op->q = q; switch (op->base.op) { case DRM_GPUVA_OP_MAP: @@ -2677,7 +2677,7 @@ again: switch (op->base.op) { case DRM_GPUVA_OP_MAP: - err = xe_vm_bind(vm, vma, op->engine, xe_vma_bo(vma), + err = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma), op->syncs, op->num_syncs, op->fence, op->map.immediate || !xe_vm_in_fault_mode(vm), op->flags & XE_VMA_OP_FIRST, @@ -2693,7 +2693,7 @@ again: vm->async_ops.munmap_rebind_inflight = true; vma->gpuva.flags |= XE_VMA_FIRST_REBIND; } - err = xe_vm_unbind(vm, vma, op->engine, op->syncs, + err = xe_vm_unbind(vm, vma, op->q, op->syncs, op->num_syncs, !prev && !next ? op->fence : NULL, op->flags & XE_VMA_OP_FIRST, @@ -2706,7 +2706,7 @@ again: if (prev) { op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND; - err = xe_vm_bind(vm, op->remap.prev, op->engine, + err = xe_vm_bind(vm, op->remap.prev, op->q, xe_vma_bo(op->remap.prev), op->syncs, op->num_syncs, !next ? op->fence : NULL, true, false, @@ -2719,7 +2719,7 @@ again: if (next) { op->remap.next->gpuva.flags |= XE_VMA_LAST_REBIND; - err = xe_vm_bind(vm, op->remap.next, op->engine, + err = xe_vm_bind(vm, op->remap.next, op->q, xe_vma_bo(op->remap.next), op->syncs, op->num_syncs, op->fence, true, false, @@ -2734,13 +2734,13 @@ again: break; } case DRM_GPUVA_OP_UNMAP: - err = xe_vm_unbind(vm, vma, op->engine, op->syncs, + err = xe_vm_unbind(vm, vma, op->q, op->syncs, op->num_syncs, op->fence, op->flags & XE_VMA_OP_FIRST, op->flags & XE_VMA_OP_LAST); break; case DRM_GPUVA_OP_PREFETCH: - err = xe_vm_prefetch(vm, vma, op->engine, op->prefetch.region, + err = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region, op->syncs, op->num_syncs, op->fence, op->flags & XE_VMA_OP_FIRST, op->flags & XE_VMA_OP_LAST); @@ -2819,8 +2819,8 @@ static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op) while (op->num_syncs--) xe_sync_entry_cleanup(&op->syncs[op->num_syncs]); kfree(op->syncs); - if (op->engine) - xe_engine_put(op->engine); + if (op->q) + xe_exec_queue_put(op->q); if (op->fence) dma_fence_put(&op->fence->fence); } @@ -3174,7 +3174,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) struct xe_bo **bos = NULL; struct drm_gpuva_ops **ops = NULL; struct xe_vm *vm; - struct xe_engine *e = NULL; + struct xe_exec_queue *q = NULL; u32 num_syncs; struct xe_sync_entry *syncs = NULL; struct drm_xe_vm_bind_op *bind_ops; @@ -3187,23 +3187,23 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (err) return err; - if (args->engine_id) { - e = xe_engine_lookup(xef, args->engine_id); - if (XE_IOCTL_DBG(xe, !e)) { + if (args->exec_queue_id) { + q = xe_exec_queue_lookup(xef, args->exec_queue_id); + if (XE_IOCTL_DBG(xe, !q)) { err = -ENOENT; goto free_objs; } - if (XE_IOCTL_DBG(xe, !(e->flags & ENGINE_FLAG_VM))) { + if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { err = -EINVAL; - goto put_engine; + goto put_exec_queue; } } vm = xe_vm_lookup(xef, args->vm_id); if (XE_IOCTL_DBG(xe, !vm)) { err = -EINVAL; - goto put_engine; + goto put_exec_queue; } err = down_write_killable(&vm->lock); @@ -3357,7 +3357,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) } } - err = vm_bind_ioctl_ops_parse(vm, e, ops, args->num_binds, + err = vm_bind_ioctl_ops_parse(vm, q, ops, args->num_binds, syncs, num_syncs, &ops_list, async); if (err) goto unwind_ops; @@ -3391,9 +3391,9 @@ release_vm_lock: up_write(&vm->lock); put_vm: xe_vm_put(vm); -put_engine: - if (e) - xe_engine_put(e); +put_exec_queue: + if (q) + xe_exec_queue_put(q); free_objs: kfree(bos); kfree(ops); diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index a1d30de37d20..805236578140 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -18,7 +18,7 @@ struct drm_file; struct ttm_buffer_object; struct ttm_validate_buffer; -struct xe_engine; +struct xe_exec_queue; struct xe_file; struct xe_sync_entry; @@ -164,7 +164,7 @@ static inline bool xe_vm_no_dma_fences(struct xe_vm *vm) return xe_vm_in_compute_mode(vm) || xe_vm_in_fault_mode(vm); } -int xe_vm_add_compute_engine(struct xe_vm *vm, struct xe_engine *e); +int xe_vm_add_compute_exec_queue(struct xe_vm *vm, struct xe_exec_queue *q); int xe_vm_userptr_pin(struct xe_vm *vm); diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index f7522f9ca40e..f8675c3da3b1 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -138,8 +138,8 @@ struct xe_vm { struct xe_device *xe; - /* engine used for (un)binding vma's */ - struct xe_engine *eng[XE_MAX_TILES_PER_DEVICE]; + /* exec queue used for (un)binding vma's */ + struct xe_exec_queue *q[XE_MAX_TILES_PER_DEVICE]; /** @lru_bulk_move: Bulk LRU move list for this VM's BOs */ struct ttm_lru_bulk_move lru_bulk_move; @@ -278,10 +278,10 @@ struct xe_vm { * an engine again */ s64 min_run_period_ms; - /** @engines: list of engines attached to this VM */ - struct list_head engines; - /** @num_engines: number user engines attached to this VM */ - int num_engines; + /** @exec_queues: list of exec queues attached to this VM */ + struct list_head exec_queues; + /** @num_exec_queues: number exec queues attached to this VM */ + int num_exec_queues; /** * @rebind_deactivated: Whether rebind has been temporarily deactivated * due to no work available. Protected by the vm resv. @@ -386,8 +386,8 @@ struct xe_vma_op { * operations is processed */ struct drm_gpuva_ops *ops; - /** @engine: engine for this operation */ - struct xe_engine *engine; + /** @q: exec queue for this operation */ + struct xe_exec_queue *q; /** * @syncs: syncs for this operation, only used on first and last * operation diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 3d09e9e9267b..86f16d50e9cc 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -103,14 +103,14 @@ struct xe_user_extension { #define DRM_XE_VM_CREATE 0x03 #define DRM_XE_VM_DESTROY 0x04 #define DRM_XE_VM_BIND 0x05 -#define DRM_XE_ENGINE_CREATE 0x06 -#define DRM_XE_ENGINE_DESTROY 0x07 +#define DRM_XE_EXEC_QUEUE_CREATE 0x06 +#define DRM_XE_EXEC_QUEUE_DESTROY 0x07 #define DRM_XE_EXEC 0x08 #define DRM_XE_MMIO 0x09 -#define DRM_XE_ENGINE_SET_PROPERTY 0x0a +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x0a #define DRM_XE_WAIT_USER_FENCE 0x0b #define DRM_XE_VM_MADVISE 0x0c -#define DRM_XE_ENGINE_GET_PROPERTY 0x0d +#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x0d /* Must be kept compact -- no holes */ #define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query) @@ -119,12 +119,12 @@ struct xe_user_extension { #define DRM_IOCTL_XE_VM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_CREATE, struct drm_xe_vm_create) #define DRM_IOCTL_XE_VM_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) #define DRM_IOCTL_XE_VM_BIND DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) -#define DRM_IOCTL_XE_ENGINE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_ENGINE_CREATE, struct drm_xe_engine_create) -#define DRM_IOCTL_XE_ENGINE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_ENGINE_GET_PROPERTY, struct drm_xe_engine_get_property) -#define DRM_IOCTL_XE_ENGINE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_ENGINE_DESTROY, struct drm_xe_engine_destroy) +#define DRM_IOCTL_XE_EXEC_QUEUE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_CREATE, struct drm_xe_exec_queue_create) +#define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property) +#define DRM_IOCTL_XE_EXEC_QUEUE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_DESTROY, struct drm_xe_exec_queue_destroy) #define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) #define DRM_IOCTL_XE_MMIO DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_MMIO, struct drm_xe_mmio) -#define DRM_IOCTL_XE_ENGINE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_ENGINE_SET_PROPERTY, struct drm_xe_engine_set_property) +#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property) #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) #define DRM_IOCTL_XE_VM_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_MADVISE, struct drm_xe_vm_madvise) @@ -649,11 +649,11 @@ struct drm_xe_vm_bind { __u32 vm_id; /** - * @engine_id: engine_id, must be of class DRM_XE_ENGINE_CLASS_VM_BIND - * and engine must have same vm_id. If zero, the default VM bind engine + * @exec_queue_id: exec_queue_id, must be of class DRM_XE_ENGINE_CLASS_VM_BIND + * and exec queue must have same vm_id. If zero, the default VM bind engine * is used. */ - __u32 engine_id; + __u32 exec_queue_id; /** @num_binds: number of binds in this IOCTL */ __u32 num_binds; @@ -685,8 +685,8 @@ struct drm_xe_vm_bind { __u64 reserved[2]; }; -/** struct drm_xe_ext_engine_set_property - engine set property extension */ -struct drm_xe_ext_engine_set_property { +/** struct drm_xe_ext_exec_queue_set_property - exec queue set property extension */ +struct drm_xe_ext_exec_queue_set_property { /** @base: base user extension */ struct xe_user_extension base; @@ -701,32 +701,32 @@ struct drm_xe_ext_engine_set_property { }; /** - * struct drm_xe_engine_set_property - engine set property + * struct drm_xe_exec_queue_set_property - exec queue set property * - * Same namespace for extensions as drm_xe_engine_create + * Same namespace for extensions as drm_xe_exec_queue_create */ -struct drm_xe_engine_set_property { +struct drm_xe_exec_queue_set_property { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; - /** @engine_id: Engine ID */ - __u32 engine_id; + /** @exec_queue_id: Exec queue ID */ + __u32 exec_queue_id; -#define XE_ENGINE_SET_PROPERTY_PRIORITY 0 -#define XE_ENGINE_SET_PROPERTY_TIMESLICE 1 -#define XE_ENGINE_SET_PROPERTY_PREEMPTION_TIMEOUT 2 +#define XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 +#define XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1 +#define XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT 2 /* * Long running or ULLS engine mode. DMA fences not allowed in this * mode. Must match the value of DRM_XE_VM_CREATE_COMPUTE_MODE, serves * as a sanity check the UMD knows what it is doing. Can only be set at * engine create time. */ -#define XE_ENGINE_SET_PROPERTY_COMPUTE_MODE 3 -#define XE_ENGINE_SET_PROPERTY_PERSISTENCE 4 -#define XE_ENGINE_SET_PROPERTY_JOB_TIMEOUT 5 -#define XE_ENGINE_SET_PROPERTY_ACC_TRIGGER 6 -#define XE_ENGINE_SET_PROPERTY_ACC_NOTIFY 7 -#define XE_ENGINE_SET_PROPERTY_ACC_GRANULARITY 8 +#define XE_EXEC_QUEUE_SET_PROPERTY_COMPUTE_MODE 3 +#define XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE 4 +#define XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT 5 +#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER 6 +#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY 7 +#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY 8 /** @property: property to set */ __u32 property; @@ -755,25 +755,25 @@ struct drm_xe_engine_class_instance { __u16 gt_id; }; -struct drm_xe_engine_create { -#define XE_ENGINE_EXTENSION_SET_PROPERTY 0 +struct drm_xe_exec_queue_create { +#define XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; - /** @width: submission width (number BB per exec) for this engine */ + /** @width: submission width (number BB per exec) for this exec queue */ __u16 width; - /** @num_placements: number of valid placements for this engine */ + /** @num_placements: number of valid placements for this exec queue */ __u16 num_placements; - /** @vm_id: VM to use for this engine */ + /** @vm_id: VM to use for this exec queue */ __u32 vm_id; /** @flags: MBZ */ __u32 flags; - /** @engine_id: Returned engine ID */ - __u32 engine_id; + /** @exec_queue_id: Returned exec queue ID */ + __u32 exec_queue_id; /** * @instances: user pointer to a 2-d array of struct @@ -788,14 +788,14 @@ struct drm_xe_engine_create { __u64 reserved[2]; }; -struct drm_xe_engine_get_property { +struct drm_xe_exec_queue_get_property { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; - /** @engine_id: Engine ID */ - __u32 engine_id; + /** @exec_queue_id: Exec queue ID */ + __u32 exec_queue_id; -#define XE_ENGINE_GET_PROPERTY_BAN 0 +#define XE_EXEC_QUEUE_GET_PROPERTY_BAN 0 /** @property: property to get */ __u32 property; @@ -806,9 +806,9 @@ struct drm_xe_engine_get_property { __u64 reserved[2]; }; -struct drm_xe_engine_destroy { - /** @engine_id: Engine ID */ - __u32 engine_id; +struct drm_xe_exec_queue_destroy { + /** @exec_queue_id: Exec queue ID */ + __u32 exec_queue_id; /** @pad: MBZ */ __u32 pad; @@ -855,8 +855,8 @@ struct drm_xe_exec { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; - /** @engine_id: Engine ID for the batch buffer */ - __u32 engine_id; + /** @exec_queue_id: Exec queue ID for the batch buffer */ + __u32 exec_queue_id; /** @num_syncs: Amount of struct drm_xe_sync in array. */ __u32 num_syncs; -- cgit v1.2.3 From 2793fac1dbe068da5965acd9a78a181b33ad469b Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 30 Aug 2023 17:47:14 -0400 Subject: drm/xe/uapi: Typo lingo and other small backwards compatible fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix typos, lingo and other small things identified during uapi review. v2: Also fix ALIGNMENT typo at xe_query.c v3: Do not touch property to get/set. (Francois) Link: https://lore.kernel.org/all/863bebd0c624d6fc2b38c0a06b63e468b4185128.camel@linux.intel.com/ Suggested-by: Thomas Hellström Cc: Thomas Hellström Signed-off-by: Rodrigo Vivi Reviewed-by: Thomas Hellström Reviewed-by: Francois Dugast --- drivers/gpu/drm/xe/xe_query.c | 2 +- include/uapi/drm/xe_drm.h | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 1db77a7c9039..c3d396904c7b 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -195,7 +195,7 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) if (xe_device_get_root_tile(xe)->mem.vram.usable_size) config->info[XE_QUERY_CONFIG_FLAGS] = XE_QUERY_CONFIG_FLAGS_HAS_VRAM; - config->info[XE_QUERY_CONFIG_MIN_ALIGNEMENT] = + config->info[XE_QUERY_CONFIG_MIN_ALIGNMENT] = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; config->info[XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; config->info[XE_QUERY_CONFIG_GT_COUNT] = xe->info.gt_count; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 86f16d50e9cc..902b5c4f3f5c 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -256,7 +256,7 @@ struct drm_xe_query_config { #define XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 #define XE_QUERY_CONFIG_FLAGS 1 #define XE_QUERY_CONFIG_FLAGS_HAS_VRAM (0x1 << 0) -#define XE_QUERY_CONFIG_MIN_ALIGNEMENT 2 +#define XE_QUERY_CONFIG_MIN_ALIGNMENT 2 #define XE_QUERY_CONFIG_VA_BITS 3 #define XE_QUERY_CONFIG_GT_COUNT 4 #define XE_QUERY_CONFIG_MEM_REGION_COUNT 5 @@ -449,7 +449,6 @@ struct drm_xe_gem_create { * If a VM is specified, this BO must: * * 1. Only ever be bound to that VM. - * * 2. Cannot be exported as a PRIME fd. */ __u32 vm_id; @@ -489,7 +488,7 @@ struct drm_xe_gem_mmap_offset { * struct drm_xe_vm_bind_op_error_capture - format of VM bind op error capture */ struct drm_xe_vm_bind_op_error_capture { - /** @error: errno that occured */ + /** @error: errno that occurred */ __s32 error; /** @op: operation that encounter an error */ @@ -609,7 +608,7 @@ struct drm_xe_vm_bind_op { * caused the error will be captured in drm_xe_vm_bind_op_error_capture. * Once the user sees the error (via a ufence + * XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS), it should free memory - * via non-async unbinds, and then restart all queue'd async binds op via + * via non-async unbinds, and then restart all queued async binds op via * XE_VM_BIND_OP_RESTART. Or alternatively the user should destroy the * VM. * @@ -620,7 +619,7 @@ struct drm_xe_vm_bind_op { #define XE_VM_BIND_FLAG_ASYNC (0x1 << 17) /* * Valid on a faulting VM only, do the MAP operation immediately rather - * than differing the MAP to the page fault handler. + * than deferring the MAP to the page fault handler. */ #define XE_VM_BIND_FLAG_IMMEDIATE (0x1 << 18) /* @@ -907,7 +906,7 @@ struct drm_xe_mmio { /** * struct drm_xe_wait_user_fence - wait user fence * - * Wait on user fence, XE will wakeup on every HW engine interrupt in the + * Wait on user fence, XE will wake-up on every HW engine interrupt in the * instances list and check if user fence is complete:: * * (*addr & MASK) OP (VALUE & MASK) @@ -1039,9 +1038,11 @@ struct drm_xe_vm_madvise { */ #define DRM_XE_VM_MADVISE_PRIORITY 5 #define DRM_XE_VMA_PRIORITY_LOW 0 -#define DRM_XE_VMA_PRIORITY_NORMAL 1 /* Default */ -#define DRM_XE_VMA_PRIORITY_HIGH 2 /* Must be elevated user */ - /* Pin the VMA in memory, must be elevated user */ + /* Default */ +#define DRM_XE_VMA_PRIORITY_NORMAL 1 + /* Must be user with elevated privileges */ +#define DRM_XE_VMA_PRIORITY_HIGH 2 + /* Pin the VMA in memory, must be user with elevated privileges */ #define DRM_XE_VM_MADVISE_PIN 6 /** @property: property to set */ __u32 property; -- cgit v1.2.3 From 9e6fe003d8c7e35bcd93f0a962b8fdc8889db35b Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 30 Aug 2023 17:47:15 -0400 Subject: drm/xe/uapi: Remove useless max_page_size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The min_page_size is useful information to ensure alignment and it is an API actually in use. However max_page_size doesn't bring any useful information to the userspace hence being not used at all. So, let's remove and only bring it back if that ever gets used. Suggested-by: Thomas Hellström Cc: Thomas Hellström Signed-off-by: Rodrigo Vivi Reviewed-by: Francois Dugast --- drivers/gpu/drm/xe/xe_query.c | 3 --- include/uapi/drm/xe_drm.h | 4 ---- 2 files changed, 7 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index c3d396904c7b..a951205100fe 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -127,7 +127,6 @@ static int query_memory_usage(struct xe_device *xe, usage->regions[0].mem_class = XE_MEM_REGION_CLASS_SYSMEM; usage->regions[0].instance = 0; usage->regions[0].min_page_size = PAGE_SIZE; - usage->regions[0].max_page_size = PAGE_SIZE; usage->regions[0].total_size = man->size << PAGE_SHIFT; if (perfmon_capable()) usage->regions[0].used = ttm_resource_manager_usage(man); @@ -143,8 +142,6 @@ static int query_memory_usage(struct xe_device *xe, usage->regions[usage->num_regions].min_page_size = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : PAGE_SIZE; - usage->regions[usage->num_regions].max_page_size = - SZ_1G; usage->regions[usage->num_regions].total_size = man->size; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 902b5c4f3f5c..00d5cb4ef85e 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -174,10 +174,6 @@ struct drm_xe_query_mem_region { * kernel. */ __u32 min_page_size; - /** - * @max_page_size: Max page-size in bytes for this region. - */ - __u32 max_page_size; /** * @total_size: The usable size in bytes for this region. */ -- cgit v1.2.3 From 3856b0f71f52b8397887c1765e14d0245d722233 Mon Sep 17 00:00:00 2001 From: Aravind Iddamsetty Date: Wed, 30 Aug 2023 08:48:53 +0530 Subject: drm/xe/pmu: Enable PMU interface There are a set of engine group busyness counters provided by HW which are perfect fit to be exposed via PMU perf events. BSPEC: 46559, 46560, 46722, 46729, 52071, 71028 events can be listed using: perf list xe_0000_03_00.0/any-engine-group-busy-gt0/ [Kernel PMU event] xe_0000_03_00.0/copy-group-busy-gt0/ [Kernel PMU event] xe_0000_03_00.0/interrupts/ [Kernel PMU event] xe_0000_03_00.0/media-group-busy-gt0/ [Kernel PMU event] xe_0000_03_00.0/render-group-busy-gt0/ [Kernel PMU event] and can be read using: perf stat -e "xe_0000_8c_00.0/render-group-busy-gt0/" -I 1000 time counts unit events 1.001139062 0 ns xe_0000_8c_00.0/render-group-busy-gt0/ 2.003294678 0 ns xe_0000_8c_00.0/render-group-busy-gt0/ 3.005199582 0 ns xe_0000_8c_00.0/render-group-busy-gt0/ 4.007076497 0 ns xe_0000_8c_00.0/render-group-busy-gt0/ 5.008553068 0 ns xe_0000_8c_00.0/render-group-busy-gt0/ 6.010531563 43520 ns xe_0000_8c_00.0/render-group-busy-gt0/ 7.012468029 44800 ns xe_0000_8c_00.0/render-group-busy-gt0/ 8.013463515 0 ns xe_0000_8c_00.0/render-group-busy-gt0/ 9.015300183 0 ns xe_0000_8c_00.0/render-group-busy-gt0/ 10.017233010 0 ns xe_0000_8c_00.0/render-group-busy-gt0/ 10.971934120 0 ns xe_0000_8c_00.0/render-group-busy-gt0/ The pmu base implementation is taken from i915. v2: Store last known value when device is awake return that while the GT is suspended and then update the driver copy when read during awake. v3: 1. drop init_samples, as storing counters before going to suspend should be sufficient. 2. ported the "drm/i915/pmu: Make PMU sample array two-dimensional" and dropped helpers to store and read samples. 3. use xe_device_mem_access_get_if_ongoing to check if device is active before reading the OA registers. 4. dropped format attr as no longer needed 5. introduce xe_pmu_suspend to call engine_group_busyness_store 6. few other nits. v4: minor nits. v5: take forcewake when accessing the OAG registers v6: 1. drop engine_busyness_sample_type 2. update UAPI documentation v7: 1. update UAPI documentation 2. drop MEDIA_GT specific change for media busyness counter. Co-developed-by: Tvrtko Ursulin Co-developed-by: Bommu Krishnaiah Signed-off-by: Aravind Iddamsetty Reviewed-by: Ashutosh Dixit Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/Makefile | 2 + drivers/gpu/drm/xe/regs/xe_gt_regs.h | 5 + drivers/gpu/drm/xe/xe_device.c | 2 + drivers/gpu/drm/xe/xe_device_types.h | 4 + drivers/gpu/drm/xe/xe_gt.c | 2 + drivers/gpu/drm/xe/xe_irq.c | 18 + drivers/gpu/drm/xe/xe_module.c | 5 + drivers/gpu/drm/xe/xe_pmu.c | 654 +++++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_pmu.h | 25 ++ drivers/gpu/drm/xe/xe_pmu_types.h | 76 ++++ include/uapi/drm/xe_drm.h | 40 +++ 11 files changed, 833 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_pmu.c create mode 100644 drivers/gpu/drm/xe/xe_pmu.h create mode 100644 drivers/gpu/drm/xe/xe_pmu_types.h (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index be93745e8a30..d3b97bc11af7 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -124,6 +124,8 @@ xe-y += xe_bb.o \ obj-$(CONFIG_DRM_XE) += xe.o obj-$(CONFIG_DRM_XE_KUNIT_TEST) += tests/ +xe-$(CONFIG_PERF_EVENTS) += xe_pmu.o + # header test hdrtest_find_args := -not -path xe_rtp_helpers.h diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index 271ed0cdbe21..e13fbbdf6929 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -294,6 +294,11 @@ #define INVALIDATION_BROADCAST_MODE_DIS REG_BIT(12) #define GLOBAL_INVALIDATION_MODE REG_BIT(2) +#define XE_OAG_RC0_ANY_ENGINE_BUSY_FREE XE_REG(0xdb80) +#define XE_OAG_ANY_MEDIA_FF_BUSY_FREE XE_REG(0xdba0) +#define XE_OAG_BLT_BUSY_FREE XE_REG(0xdbbc) +#define XE_OAG_RENDER_BUSY_FREE XE_REG(0xdbdc) + #define SAMPLER_MODE XE_REG_MCR(0xe18c, XE_REG_OPTION_MASKED) #define ENABLE_SMALLPL REG_BIT(15) #define SC_DISABLE_POWER_OPTIMIZATION_EBB REG_BIT(9) diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 986a02a66166..89bf926bc0f3 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -304,6 +304,8 @@ int xe_device_probe(struct xe_device *xe) xe_debugfs_register(xe); + xe_pmu_register(&xe->pmu); + err = drmm_add_action_or_reset(&xe->drm, xe_device_sanitize, xe); if (err) return err; diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index 552e8a343d8f..496d7f3fb897 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -15,6 +15,7 @@ #include "xe_devcoredump_types.h" #include "xe_gt_types.h" #include "xe_platform_types.h" +#include "xe_pmu.h" #include "xe_step_types.h" struct xe_ggtt; @@ -342,6 +343,9 @@ struct xe_device { */ struct task_struct *pm_callback_task; + /** @pmu: performance monitoring unit */ + struct xe_pmu pmu; + /* For pcode */ struct mutex sb_lock; diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 5d86bb2bb94d..06147f26384f 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -652,6 +652,8 @@ int xe_gt_suspend(struct xe_gt *gt) if (err) goto err_msg; + xe_pmu_suspend(gt); + err = xe_uc_suspend(>->uc); if (err) goto err_force_wake; diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c index ef434142bcd9..772b8006d98f 100644 --- a/drivers/gpu/drm/xe/xe_irq.c +++ b/drivers/gpu/drm/xe/xe_irq.c @@ -26,6 +26,20 @@ #define IIR(offset) XE_REG(offset + 0x8) #define IER(offset) XE_REG(offset + 0xc) +/* + * Interrupt statistic for PMU. Increments the counter only if the + * interrupt originated from the GPU so interrupts from a device which + * shares the interrupt line are not accounted. + */ +static __always_inline void xe_pmu_irq_stats(struct xe_device *xe) +{ + /* + * A clever compiler translates that into INC. A not so clever one + * should at least prevent store tearing. + */ + WRITE_ONCE(xe->pmu.irq_count, xe->pmu.irq_count + 1); +} + static void assert_iir_is_zero(struct xe_gt *mmio, struct xe_reg reg) { u32 val = xe_mmio_read32(mmio, reg); @@ -332,6 +346,8 @@ static irqreturn_t xelp_irq_handler(int irq, void *arg) xelp_intr_enable(xe, false); + xe_pmu_irq_stats(xe); + return IRQ_HANDLED; } @@ -425,6 +441,8 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg) dg1_intr_enable(xe, false); + xe_pmu_irq_stats(xe); + return IRQ_HANDLED; } diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c index ed3772a69762..d76fabe056d0 100644 --- a/drivers/gpu/drm/xe/xe_module.c +++ b/drivers/gpu/drm/xe/xe_module.c @@ -12,6 +12,7 @@ #include "xe_hw_fence.h" #include "xe_module.h" #include "xe_pci.h" +#include "xe_pmu.h" #include "xe_sched_job.h" bool force_execlist = false; @@ -45,6 +46,10 @@ static const struct init_funcs init_funcs[] = { .init = xe_sched_job_module_init, .exit = xe_sched_job_module_exit, }, + { + .init = xe_pmu_init, + .exit = xe_pmu_exit, + }, { .init = xe_register_pci_driver, .exit = xe_unregister_pci_driver, diff --git a/drivers/gpu/drm/xe/xe_pmu.c b/drivers/gpu/drm/xe/xe_pmu.c new file mode 100644 index 000000000000..abfc0b3aeac4 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pmu.c @@ -0,0 +1,654 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2023 Intel Corporation + */ + +#include +#include +#include + +#include "regs/xe_gt_regs.h" +#include "xe_device.h" +#include "xe_gt_clock.h" +#include "xe_mmio.h" + +static cpumask_t xe_pmu_cpumask; +static unsigned int xe_pmu_target_cpu = -1; + +static unsigned int config_gt_id(const u64 config) +{ + return config >> __XE_PMU_GT_SHIFT; +} + +static u64 config_counter(const u64 config) +{ + return config & ~(~0ULL << __XE_PMU_GT_SHIFT); +} + +static void xe_pmu_event_destroy(struct perf_event *event) +{ + struct xe_device *xe = + container_of(event->pmu, typeof(*xe), pmu.base); + + drm_WARN_ON(&xe->drm, event->parent); + + drm_dev_put(&xe->drm); +} + +static u64 __engine_group_busyness_read(struct xe_gt *gt, int sample_type) +{ + u64 val; + + switch (sample_type) { + case __XE_SAMPLE_RENDER_GROUP_BUSY: + val = xe_mmio_read32(gt, XE_OAG_RENDER_BUSY_FREE); + break; + case __XE_SAMPLE_COPY_GROUP_BUSY: + val = xe_mmio_read32(gt, XE_OAG_BLT_BUSY_FREE); + break; + case __XE_SAMPLE_MEDIA_GROUP_BUSY: + val = xe_mmio_read32(gt, XE_OAG_ANY_MEDIA_FF_BUSY_FREE); + break; + case __XE_SAMPLE_ANY_ENGINE_GROUP_BUSY: + val = xe_mmio_read32(gt, XE_OAG_RC0_ANY_ENGINE_BUSY_FREE); + break; + default: + drm_warn(>->tile->xe->drm, "unknown pmu event\n"); + } + + return xe_gt_clock_cycles_to_ns(gt, val * 16); +} + +static u64 engine_group_busyness_read(struct xe_gt *gt, u64 config) +{ + int sample_type = config_counter(config) - 1; + const unsigned int gt_id = gt->info.id; + struct xe_device *xe = gt->tile->xe; + struct xe_pmu *pmu = &xe->pmu; + unsigned long flags; + bool device_awake; + u64 val; + + device_awake = xe_device_mem_access_get_if_ongoing(xe); + if (device_awake) { + XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FW_GT)); + val = __engine_group_busyness_read(gt, sample_type); + XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FW_GT)); + xe_device_mem_access_put(xe); + } + + spin_lock_irqsave(&pmu->lock, flags); + + if (device_awake) + pmu->sample[gt_id][sample_type] = val; + else + val = pmu->sample[gt_id][sample_type]; + + spin_unlock_irqrestore(&pmu->lock, flags); + + return val; +} + +static void engine_group_busyness_store(struct xe_gt *gt) +{ + struct xe_pmu *pmu = >->tile->xe->pmu; + unsigned int gt_id = gt->info.id; + unsigned long flags; + int i; + + spin_lock_irqsave(&pmu->lock, flags); + + for (i = __XE_SAMPLE_RENDER_GROUP_BUSY; i <= __XE_SAMPLE_ANY_ENGINE_GROUP_BUSY; i++) + pmu->sample[gt_id][i] = __engine_group_busyness_read(gt, i); + + spin_unlock_irqrestore(&pmu->lock, flags); +} + +static int +config_status(struct xe_device *xe, u64 config) +{ + unsigned int gt_id = config_gt_id(config); + struct xe_gt *gt = xe_device_get_gt(xe, gt_id); + + if (gt_id >= XE_PMU_MAX_GT) + return -ENOENT; + + switch (config_counter(config)) { + case XE_PMU_INTERRUPTS(0): + if (gt_id) + return -ENOENT; + break; + case XE_PMU_RENDER_GROUP_BUSY(0): + case XE_PMU_COPY_GROUP_BUSY(0): + case XE_PMU_ANY_ENGINE_GROUP_BUSY(0): + if (gt->info.type == XE_GT_TYPE_MEDIA) + return -ENOENT; + break; + case XE_PMU_MEDIA_GROUP_BUSY(0): + if (!(gt->info.engine_mask & (BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VECS0)))) + return -ENOENT; + break; + default: + return -ENOENT; + } + + return 0; +} + +static int xe_pmu_event_init(struct perf_event *event) +{ + struct xe_device *xe = + container_of(event->pmu, typeof(*xe), pmu.base); + struct xe_pmu *pmu = &xe->pmu; + int ret; + + if (pmu->closed) + return -ENODEV; + + if (event->attr.type != event->pmu->type) + return -ENOENT; + + /* unsupported modes and filters */ + if (event->attr.sample_period) /* no sampling */ + return -EINVAL; + + if (has_branch_stack(event)) + return -EOPNOTSUPP; + + if (event->cpu < 0) + return -EINVAL; + + /* only allow running on one cpu at a time */ + if (!cpumask_test_cpu(event->cpu, &xe_pmu_cpumask)) + return -EINVAL; + + ret = config_status(xe, event->attr.config); + if (ret) + return ret; + + if (!event->parent) { + drm_dev_get(&xe->drm); + event->destroy = xe_pmu_event_destroy; + } + + return 0; +} + +static u64 __xe_pmu_event_read(struct perf_event *event) +{ + struct xe_device *xe = + container_of(event->pmu, typeof(*xe), pmu.base); + const unsigned int gt_id = config_gt_id(event->attr.config); + const u64 config = event->attr.config; + struct xe_gt *gt = xe_device_get_gt(xe, gt_id); + struct xe_pmu *pmu = &xe->pmu; + u64 val; + + switch (config_counter(config)) { + case XE_PMU_INTERRUPTS(0): + val = READ_ONCE(pmu->irq_count); + break; + case XE_PMU_RENDER_GROUP_BUSY(0): + case XE_PMU_COPY_GROUP_BUSY(0): + case XE_PMU_ANY_ENGINE_GROUP_BUSY(0): + case XE_PMU_MEDIA_GROUP_BUSY(0): + val = engine_group_busyness_read(gt, config); + break; + default: + drm_warn(>->tile->xe->drm, "unknown pmu event\n"); + } + + return val; +} + +static void xe_pmu_event_read(struct perf_event *event) +{ + struct xe_device *xe = + container_of(event->pmu, typeof(*xe), pmu.base); + struct hw_perf_event *hwc = &event->hw; + struct xe_pmu *pmu = &xe->pmu; + u64 prev, new; + + if (pmu->closed) { + event->hw.state = PERF_HES_STOPPED; + return; + } +again: + prev = local64_read(&hwc->prev_count); + new = __xe_pmu_event_read(event); + + if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev) + goto again; + + local64_add(new - prev, &event->count); +} + +static void xe_pmu_enable(struct perf_event *event) +{ + /* + * Store the current counter value so we can report the correct delta + * for all listeners. Even when the event was already enabled and has + * an existing non-zero value. + */ + local64_set(&event->hw.prev_count, __xe_pmu_event_read(event)); +} + +static void xe_pmu_event_start(struct perf_event *event, int flags) +{ + struct xe_device *xe = + container_of(event->pmu, typeof(*xe), pmu.base); + struct xe_pmu *pmu = &xe->pmu; + + if (pmu->closed) + return; + + xe_pmu_enable(event); + event->hw.state = 0; +} + +static void xe_pmu_event_stop(struct perf_event *event, int flags) +{ + if (flags & PERF_EF_UPDATE) + xe_pmu_event_read(event); + + event->hw.state = PERF_HES_STOPPED; +} + +static int xe_pmu_event_add(struct perf_event *event, int flags) +{ + struct xe_device *xe = + container_of(event->pmu, typeof(*xe), pmu.base); + struct xe_pmu *pmu = &xe->pmu; + + if (pmu->closed) + return -ENODEV; + + if (flags & PERF_EF_START) + xe_pmu_event_start(event, flags); + + return 0; +} + +static void xe_pmu_event_del(struct perf_event *event, int flags) +{ + xe_pmu_event_stop(event, PERF_EF_UPDATE); +} + +static int xe_pmu_event_event_idx(struct perf_event *event) +{ + return 0; +} + +struct xe_ext_attribute { + struct device_attribute attr; + unsigned long val; +}; + +static ssize_t xe_pmu_event_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct xe_ext_attribute *eattr; + + eattr = container_of(attr, struct xe_ext_attribute, attr); + return sprintf(buf, "config=0x%lx\n", eattr->val); +} + +static ssize_t cpumask_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + return cpumap_print_to_pagebuf(true, buf, &xe_pmu_cpumask); +} + +static DEVICE_ATTR_RO(cpumask); + +static struct attribute *xe_cpumask_attrs[] = { + &dev_attr_cpumask.attr, + NULL, +}; + +static const struct attribute_group xe_pmu_cpumask_attr_group = { + .attrs = xe_cpumask_attrs, +}; + +#define __event(__counter, __name, __unit) \ +{ \ + .counter = (__counter), \ + .name = (__name), \ + .unit = (__unit), \ + .global = false, \ +} + +#define __global_event(__counter, __name, __unit) \ +{ \ + .counter = (__counter), \ + .name = (__name), \ + .unit = (__unit), \ + .global = true, \ +} + +static struct xe_ext_attribute * +add_xe_attr(struct xe_ext_attribute *attr, const char *name, u64 config) +{ + sysfs_attr_init(&attr->attr.attr); + attr->attr.attr.name = name; + attr->attr.attr.mode = 0444; + attr->attr.show = xe_pmu_event_show; + attr->val = config; + + return ++attr; +} + +static struct perf_pmu_events_attr * +add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name, + const char *str) +{ + sysfs_attr_init(&attr->attr.attr); + attr->attr.attr.name = name; + attr->attr.attr.mode = 0444; + attr->attr.show = perf_event_sysfs_show; + attr->event_str = str; + + return ++attr; +} + +static struct attribute ** +create_event_attributes(struct xe_pmu *pmu) +{ + struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); + static const struct { + unsigned int counter; + const char *name; + const char *unit; + bool global; + } events[] = { + __global_event(0, "interrupts", NULL), + __event(1, "render-group-busy", "ns"), + __event(2, "copy-group-busy", "ns"), + __event(3, "media-group-busy", "ns"), + __event(4, "any-engine-group-busy", "ns"), + }; + + struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter; + struct xe_ext_attribute *xe_attr = NULL, *xe_iter; + struct attribute **attr = NULL, **attr_iter; + unsigned int count = 0; + unsigned int i, j; + struct xe_gt *gt; + + /* Count how many counters we will be exposing. */ + for_each_gt(gt, xe, j) { + for (i = 0; i < ARRAY_SIZE(events); i++) { + u64 config = ___XE_PMU_OTHER(j, events[i].counter); + + if (!config_status(xe, config)) + count++; + } + } + + /* Allocate attribute objects and table. */ + xe_attr = kcalloc(count, sizeof(*xe_attr), GFP_KERNEL); + if (!xe_attr) + goto err_alloc; + + pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL); + if (!pmu_attr) + goto err_alloc; + + /* Max one pointer of each attribute type plus a termination entry. */ + attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL); + if (!attr) + goto err_alloc; + + xe_iter = xe_attr; + pmu_iter = pmu_attr; + attr_iter = attr; + + for_each_gt(gt, xe, j) { + for (i = 0; i < ARRAY_SIZE(events); i++) { + u64 config = ___XE_PMU_OTHER(j, events[i].counter); + char *str; + + if (config_status(xe, config)) + continue; + + if (events[i].global) + str = kstrdup(events[i].name, GFP_KERNEL); + else + str = kasprintf(GFP_KERNEL, "%s-gt%u", + events[i].name, j); + if (!str) + goto err; + + *attr_iter++ = &xe_iter->attr.attr; + xe_iter = add_xe_attr(xe_iter, str, config); + + if (events[i].unit) { + if (events[i].global) + str = kasprintf(GFP_KERNEL, "%s.unit", + events[i].name); + else + str = kasprintf(GFP_KERNEL, "%s-gt%u.unit", + events[i].name, j); + if (!str) + goto err; + + *attr_iter++ = &pmu_iter->attr.attr; + pmu_iter = add_pmu_attr(pmu_iter, str, + events[i].unit); + } + } + } + + pmu->xe_attr = xe_attr; + pmu->pmu_attr = pmu_attr; + + return attr; + +err: + for (attr_iter = attr; *attr_iter; attr_iter++) + kfree((*attr_iter)->name); + +err_alloc: + kfree(attr); + kfree(xe_attr); + kfree(pmu_attr); + + return NULL; +} + +static void free_event_attributes(struct xe_pmu *pmu) +{ + struct attribute **attr_iter = pmu->events_attr_group.attrs; + + for (; *attr_iter; attr_iter++) + kfree((*attr_iter)->name); + + kfree(pmu->events_attr_group.attrs); + kfree(pmu->xe_attr); + kfree(pmu->pmu_attr); + + pmu->events_attr_group.attrs = NULL; + pmu->xe_attr = NULL; + pmu->pmu_attr = NULL; +} + +static int xe_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) +{ + struct xe_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); + + /* Select the first online CPU as a designated reader. */ + if (cpumask_empty(&xe_pmu_cpumask)) + cpumask_set_cpu(cpu, &xe_pmu_cpumask); + + return 0; +} + +static int xe_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) +{ + struct xe_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); + unsigned int target = xe_pmu_target_cpu; + + /* + * Unregistering an instance generates a CPU offline event which we must + * ignore to avoid incorrectly modifying the shared xe_pmu_cpumask. + */ + if (pmu->closed) + return 0; + + if (cpumask_test_and_clear_cpu(cpu, &xe_pmu_cpumask)) { + target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); + + /* Migrate events if there is a valid target */ + if (target < nr_cpu_ids) { + cpumask_set_cpu(target, &xe_pmu_cpumask); + xe_pmu_target_cpu = target; + } + } + + if (target < nr_cpu_ids && target != pmu->cpuhp.cpu) { + perf_pmu_migrate_context(&pmu->base, cpu, target); + pmu->cpuhp.cpu = target; + } + + return 0; +} + +static enum cpuhp_state cpuhp_slot = CPUHP_INVALID; + +int xe_pmu_init(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, + "perf/x86/intel/xe:online", + xe_pmu_cpu_online, + xe_pmu_cpu_offline); + if (ret < 0) + pr_notice("Failed to setup cpuhp state for xe PMU! (%d)\n", + ret); + else + cpuhp_slot = ret; + + return 0; +} + +void xe_pmu_exit(void) +{ + if (cpuhp_slot != CPUHP_INVALID) + cpuhp_remove_multi_state(cpuhp_slot); +} + +static int xe_pmu_register_cpuhp_state(struct xe_pmu *pmu) +{ + if (cpuhp_slot == CPUHP_INVALID) + return -EINVAL; + + return cpuhp_state_add_instance(cpuhp_slot, &pmu->cpuhp.node); +} + +static void xe_pmu_unregister_cpuhp_state(struct xe_pmu *pmu) +{ + cpuhp_state_remove_instance(cpuhp_slot, &pmu->cpuhp.node); +} + +void xe_pmu_suspend(struct xe_gt *gt) +{ + engine_group_busyness_store(gt); +} + +static void xe_pmu_unregister(struct drm_device *device, void *arg) +{ + struct xe_pmu *pmu = arg; + + if (!pmu->base.event_init) + return; + + /* + * "Disconnect" the PMU callbacks - since all are atomic synchronize_rcu + * ensures all currently executing ones will have exited before we + * proceed with unregistration. + */ + pmu->closed = true; + synchronize_rcu(); + + xe_pmu_unregister_cpuhp_state(pmu); + + perf_pmu_unregister(&pmu->base); + pmu->base.event_init = NULL; + kfree(pmu->base.attr_groups); + kfree(pmu->name); + free_event_attributes(pmu); +} + +void xe_pmu_register(struct xe_pmu *pmu) +{ + struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); + const struct attribute_group *attr_groups[] = { + &pmu->events_attr_group, + &xe_pmu_cpumask_attr_group, + NULL + }; + + int ret = -ENOMEM; + + spin_lock_init(&pmu->lock); + pmu->cpuhp.cpu = -1; + + pmu->name = kasprintf(GFP_KERNEL, + "xe_%s", + dev_name(xe->drm.dev)); + if (pmu->name) + /* tools/perf reserves colons as special. */ + strreplace((char *)pmu->name, ':', '_'); + + if (!pmu->name) + goto err; + + pmu->events_attr_group.name = "events"; + pmu->events_attr_group.attrs = create_event_attributes(pmu); + if (!pmu->events_attr_group.attrs) + goto err_name; + + pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups), + GFP_KERNEL); + if (!pmu->base.attr_groups) + goto err_attr; + + pmu->base.module = THIS_MODULE; + pmu->base.task_ctx_nr = perf_invalid_context; + pmu->base.event_init = xe_pmu_event_init; + pmu->base.add = xe_pmu_event_add; + pmu->base.del = xe_pmu_event_del; + pmu->base.start = xe_pmu_event_start; + pmu->base.stop = xe_pmu_event_stop; + pmu->base.read = xe_pmu_event_read; + pmu->base.event_idx = xe_pmu_event_event_idx; + + ret = perf_pmu_register(&pmu->base, pmu->name, -1); + if (ret) + goto err_groups; + + ret = xe_pmu_register_cpuhp_state(pmu); + if (ret) + goto err_unreg; + + ret = drmm_add_action_or_reset(&xe->drm, xe_pmu_unregister, pmu); + if (ret) + goto err_cpuhp; + + return; + +err_cpuhp: + xe_pmu_unregister_cpuhp_state(pmu); +err_unreg: + perf_pmu_unregister(&pmu->base); +err_groups: + kfree(pmu->base.attr_groups); +err_attr: + pmu->base.event_init = NULL; + free_event_attributes(pmu); +err_name: + kfree(pmu->name); +err: + drm_notice(&xe->drm, "Failed to register PMU!\n"); +} diff --git a/drivers/gpu/drm/xe/xe_pmu.h b/drivers/gpu/drm/xe/xe_pmu.h new file mode 100644 index 000000000000..a99d4ddd023e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pmu.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_PMU_H_ +#define _XE_PMU_H_ + +#include "xe_gt_types.h" +#include "xe_pmu_types.h" + +#if IS_ENABLED(CONFIG_PERF_EVENTS) +int xe_pmu_init(void); +void xe_pmu_exit(void); +void xe_pmu_register(struct xe_pmu *pmu); +void xe_pmu_suspend(struct xe_gt *gt); +#else +static inline int xe_pmu_init(void) { return 0; } +static inline void xe_pmu_exit(void) {} +static inline void xe_pmu_register(struct xe_pmu *pmu) {} +static inline void xe_pmu_suspend(struct xe_gt *gt) {} +#endif + +#endif + diff --git a/drivers/gpu/drm/xe/xe_pmu_types.h b/drivers/gpu/drm/xe/xe_pmu_types.h new file mode 100644 index 000000000000..4ccc7e9042f6 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_pmu_types.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2023 Intel Corporation + */ + +#ifndef _XE_PMU_TYPES_H_ +#define _XE_PMU_TYPES_H_ + +#include +#include +#include + +enum { + __XE_SAMPLE_RENDER_GROUP_BUSY, + __XE_SAMPLE_COPY_GROUP_BUSY, + __XE_SAMPLE_MEDIA_GROUP_BUSY, + __XE_SAMPLE_ANY_ENGINE_GROUP_BUSY, + __XE_NUM_PMU_SAMPLERS +}; + +#define XE_PMU_MAX_GT 2 + +struct xe_pmu { + /** + * @cpuhp: Struct used for CPU hotplug handling. + */ + struct { + struct hlist_node node; + unsigned int cpu; + } cpuhp; + /** + * @base: PMU base. + */ + struct pmu base; + /** + * @closed: xe is unregistering. + */ + bool closed; + /** + * @name: Name as registered with perf core. + */ + const char *name; + /** + * @lock: Lock protecting enable mask and ref count handling. + */ + spinlock_t lock; + /** + * @sample: Current and previous (raw) counters. + * + * These counters are updated when the device is awake. + * + */ + u64 sample[XE_PMU_MAX_GT][__XE_NUM_PMU_SAMPLERS]; + /** + * @irq_count: Number of interrupts + * + * Intentionally unsigned long to avoid atomics or heuristics on 32bit. + * 4e9 interrupts are a lot and postprocessing can really deal with an + * occasional wraparound easily. It's 32bit after all. + */ + unsigned long irq_count; + /** + * @events_attr_group: Device events attribute group. + */ + struct attribute_group events_attr_group; + /** + * @xe_attr: Memory block holding device attributes. + */ + void *xe_attr; + /** + * @pmu_attr: Memory block holding device attributes. + */ + void *pmu_attr; +}; + +#endif diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 00d5cb4ef85e..d48d8e3c898c 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1053,6 +1053,46 @@ struct drm_xe_vm_madvise { __u64 reserved[2]; }; +/** + * DOC: XE PMU event config IDs + * + * Check 'man perf_event_open' to use the ID's XE_PMU_XXXX listed in xe_drm.h + * in 'struct perf_event_attr' as part of perf_event_open syscall to read a + * particular event. + * + * For example to open the XE_PMU_INTERRUPTS(0): + * + * .. code-block:: C + * + * struct perf_event_attr attr; + * long long count; + * int cpu = 0; + * int fd; + * + * memset(&attr, 0, sizeof(struct perf_event_attr)); + * attr.type = type; // eg: /sys/bus/event_source/devices/xe_0000_56_00.0/type + * attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED; + * attr.use_clockid = 1; + * attr.clockid = CLOCK_MONOTONIC; + * attr.config = XE_PMU_INTERRUPTS(0); + * + * fd = syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0); + */ + +/* + * Top bits of every counter are GT id. + */ +#define __XE_PMU_GT_SHIFT (56) + +#define ___XE_PMU_OTHER(gt, x) \ + (((__u64)(x)) | ((__u64)(gt) << __XE_PMU_GT_SHIFT)) + +#define XE_PMU_INTERRUPTS(gt) ___XE_PMU_OTHER(gt, 0) +#define XE_PMU_RENDER_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 1) +#define XE_PMU_COPY_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 2) +#define XE_PMU_MEDIA_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 3) +#define XE_PMU_ANY_ENGINE_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 4) + #if defined(__cplusplus) } #endif -- cgit v1.2.3 From 7793d00d1bf5923e77bbe7ace8089bfdfa19dc38 Mon Sep 17 00:00:00 2001 From: Umesh Nerlige Ramappa Date: Mon, 14 Aug 2023 15:37:34 -0700 Subject: drm/xe: Correlate engine and cpu timestamps with better accuracy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Perf measurements rely on CPU and engine timestamps to correlate events of interest across these time domains. Current mechanisms get these timestamps separately and the calculated delta between these timestamps lack enough accuracy. To improve the accuracy of these time measurements to within a few us, add a query that returns the engine and cpu timestamps captured as close to each other as possible. Mesa MR: https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/24591 v2: - Fix kernel-doc warnings (CI) - Document input params and group them together (Jose) - s/cs/engine/ (Jose) - Remove padding in the query (Ashutosh) Signed-off-by: Umesh Nerlige Ramappa Reviewed-by: José Roberto de Souza Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi [Rodrigo finished the s/cs/engine renaming] --- drivers/gpu/drm/xe/xe_query.c | 138 ++++++++++++++++++++++++++++++++++++++++++ include/uapi/drm/xe_drm.h | 104 +++++++++++++++++++++++-------- 2 files changed, 218 insertions(+), 24 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index cbccd5c3dbc8..cd3e0f3208a6 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -6,10 +6,12 @@ #include "xe_query.h" #include +#include #include #include +#include "regs/xe_engine_regs.h" #include "xe_bo.h" #include "xe_device.h" #include "xe_exec_queue.h" @@ -17,6 +19,7 @@ #include "xe_gt.h" #include "xe_guc_hwconfig.h" #include "xe_macros.h" +#include "xe_mmio.h" #include "xe_ttm_vram_mgr.h" static const u16 xe_to_user_engine_class[] = { @@ -27,6 +30,14 @@ static const u16 xe_to_user_engine_class[] = { [XE_ENGINE_CLASS_COMPUTE] = DRM_XE_ENGINE_CLASS_COMPUTE, }; +static const enum xe_engine_class user_to_xe_engine_class[] = { + [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, + [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, + [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, + [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, + [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, +}; + static size_t calc_hw_engine_info_size(struct xe_device *xe) { struct xe_hw_engine *hwe; @@ -45,6 +56,132 @@ static size_t calc_hw_engine_info_size(struct xe_device *xe) return i * sizeof(struct drm_xe_engine_class_instance); } +typedef u64 (*__ktime_func_t)(void); +static __ktime_func_t __clock_id_to_func(clockid_t clk_id) +{ + /* + * Use logic same as the perf subsystem to allow user to select the + * reference clock id to be used for timestamps. + */ + switch (clk_id) { + case CLOCK_MONOTONIC: + return &ktime_get_ns; + case CLOCK_MONOTONIC_RAW: + return &ktime_get_raw_ns; + case CLOCK_REALTIME: + return &ktime_get_real_ns; + case CLOCK_BOOTTIME: + return &ktime_get_boottime_ns; + case CLOCK_TAI: + return &ktime_get_clocktai_ns; + default: + return NULL; + } +} + +static void +__read_timestamps(struct xe_gt *gt, + struct xe_reg lower_reg, + struct xe_reg upper_reg, + u64 *engine_ts, + u64 *cpu_ts, + u64 *cpu_delta, + __ktime_func_t cpu_clock) +{ + u32 upper, lower, old_upper, loop = 0; + + upper = xe_mmio_read32(gt, upper_reg); + do { + *cpu_delta = local_clock(); + *cpu_ts = cpu_clock(); + lower = xe_mmio_read32(gt, lower_reg); + *cpu_delta = local_clock() - *cpu_delta; + old_upper = upper; + upper = xe_mmio_read32(gt, upper_reg); + } while (upper != old_upper && loop++ < 2); + + *engine_ts = (u64)upper << 32 | lower; +} + +static int +query_engine_cycles(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + struct drm_xe_query_engine_cycles __user *query_ptr; + struct drm_xe_engine_class_instance *eci; + struct drm_xe_query_engine_cycles resp; + size_t size = sizeof(resp); + __ktime_func_t cpu_clock; + struct xe_hw_engine *hwe; + struct xe_gt *gt; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + query_ptr = u64_to_user_ptr(query->data); + if (copy_from_user(&resp, query_ptr, size)) + return -EFAULT; + + cpu_clock = __clock_id_to_func(resp.clockid); + if (!cpu_clock) + return -EINVAL; + + eci = &resp.eci; + if (eci->gt_id > XE_MAX_GT_PER_TILE) + return -EINVAL; + + gt = xe_device_get_gt(xe, eci->gt_id); + if (!gt) + return -EINVAL; + + if (eci->engine_class >= ARRAY_SIZE(user_to_xe_engine_class)) + return -EINVAL; + + hwe = xe_gt_hw_engine(gt, user_to_xe_engine_class[eci->engine_class], + eci->engine_instance, true); + if (!hwe) + return -EINVAL; + + resp.engine_frequency = gt->info.clock_freq; + + xe_device_mem_access_get(xe); + xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); + + __read_timestamps(gt, + RING_TIMESTAMP(hwe->mmio_base), + RING_TIMESTAMP_UDW(hwe->mmio_base), + &resp.engine_cycles, + &resp.cpu_timestamp, + &resp.cpu_delta, + cpu_clock); + + xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); + xe_device_mem_access_put(xe); + resp.width = 36; + + /* Only write to the output fields of user query */ + if (put_user(resp.engine_frequency, &query_ptr->engine_frequency)) + return -EFAULT; + + if (put_user(resp.cpu_timestamp, &query_ptr->cpu_timestamp)) + return -EFAULT; + + if (put_user(resp.cpu_delta, &query_ptr->cpu_delta)) + return -EFAULT; + + if (put_user(resp.engine_cycles, &query_ptr->engine_cycles)) + return -EFAULT; + + if (put_user(resp.width, &query_ptr->width)) + return -EFAULT; + + return 0; +} + static int query_engines(struct xe_device *xe, struct drm_xe_device_query *query) { @@ -369,6 +506,7 @@ static int (* const xe_query_funcs[])(struct xe_device *xe, query_gts, query_hwconfig, query_gt_topology, + query_engine_cycles, }; int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index d48d8e3c898c..079213a3df55 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -128,6 +128,25 @@ struct xe_user_extension { #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) #define DRM_IOCTL_XE_VM_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_MADVISE, struct drm_xe_vm_madvise) +/** struct drm_xe_engine_class_instance - instance of an engine class */ +struct drm_xe_engine_class_instance { +#define DRM_XE_ENGINE_CLASS_RENDER 0 +#define DRM_XE_ENGINE_CLASS_COPY 1 +#define DRM_XE_ENGINE_CLASS_VIDEO_DECODE 2 +#define DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE 3 +#define DRM_XE_ENGINE_CLASS_COMPUTE 4 + /* + * Kernel only class (not actual hardware engine class). Used for + * creating ordered queues of VM bind operations. + */ +#define DRM_XE_ENGINE_CLASS_VM_BIND 5 + __u16 engine_class; + + __u16 engine_instance; + __u16 gt_id; + __u16 rsvd; +}; + /** * enum drm_xe_memory_class - Supported memory classes. */ @@ -219,6 +238,60 @@ struct drm_xe_query_mem_region { __u64 reserved[6]; }; +/** + * struct drm_xe_query_engine_cycles - correlate CPU and GPU timestamps + * + * If a query is made with a struct drm_xe_device_query where .query is equal to + * DRM_XE_DEVICE_QUERY_ENGINE_CYCLES, then the reply uses struct drm_xe_query_engine_cycles + * in .data. struct drm_xe_query_engine_cycles is allocated by the user and + * .data points to this allocated structure. + * + * The query returns the engine cycles and the frequency that can + * be used to calculate the engine timestamp. In addition the + * query returns a set of cpu timestamps that indicate when the command + * streamer cycle count was captured. + */ +struct drm_xe_query_engine_cycles { + /** + * @eci: This is input by the user and is the engine for which command + * streamer cycles is queried. + */ + struct drm_xe_engine_class_instance eci; + + /** + * @clockid: This is input by the user and is the reference clock id for + * CPU timestamp. For definition, see clock_gettime(2) and + * perf_event_open(2). Supported clock ids are CLOCK_MONOTONIC, + * CLOCK_MONOTONIC_RAW, CLOCK_REALTIME, CLOCK_BOOTTIME, CLOCK_TAI. + */ + __s32 clockid; + + /** @width: Width of the engine cycle counter in bits. */ + __u32 width; + + /** + * @engine_cycles: Engine cycles as read from its register + * at 0x358 offset. + */ + __u64 engine_cycles; + + /** @engine_frequency: Frequency of the engine cycles in Hz. */ + __u64 engine_frequency; + + /** + * @cpu_timestamp: CPU timestamp in ns. The timestamp is captured before + * reading the engine_cycles register using the reference clockid set by the + * user. + */ + __u64 cpu_timestamp; + + /** + * @cpu_delta: Time delta in ns captured around reading the lower dword + * of the engine_cycles register. + */ + __u64 cpu_delta; +}; + /** * struct drm_xe_query_mem_usage - describe memory regions and usage * @@ -385,12 +458,13 @@ struct drm_xe_device_query { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; -#define DRM_XE_DEVICE_QUERY_ENGINES 0 -#define DRM_XE_DEVICE_QUERY_MEM_USAGE 1 -#define DRM_XE_DEVICE_QUERY_CONFIG 2 -#define DRM_XE_DEVICE_QUERY_GTS 3 -#define DRM_XE_DEVICE_QUERY_HWCONFIG 4 -#define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5 +#define DRM_XE_DEVICE_QUERY_ENGINES 0 +#define DRM_XE_DEVICE_QUERY_MEM_USAGE 1 +#define DRM_XE_DEVICE_QUERY_CONFIG 2 +#define DRM_XE_DEVICE_QUERY_GTS 3 +#define DRM_XE_DEVICE_QUERY_HWCONFIG 4 +#define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5 +#define DRM_XE_DEVICE_QUERY_ENGINE_CYCLES 6 /** @query: The type of data to query */ __u32 query; @@ -732,24 +806,6 @@ struct drm_xe_exec_queue_set_property { __u64 reserved[2]; }; -/** struct drm_xe_engine_class_instance - instance of an engine class */ -struct drm_xe_engine_class_instance { -#define DRM_XE_ENGINE_CLASS_RENDER 0 -#define DRM_XE_ENGINE_CLASS_COPY 1 -#define DRM_XE_ENGINE_CLASS_VIDEO_DECODE 2 -#define DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE 3 -#define DRM_XE_ENGINE_CLASS_COMPUTE 4 - /* - * Kernel only class (not actual hardware engine class). Used for - * creating ordered queues of VM bind operations. - */ -#define DRM_XE_ENGINE_CLASS_VM_BIND 5 - __u16 engine_class; - - __u16 engine_instance; - __u16 gt_id; -}; - struct drm_xe_exec_queue_create { #define XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 /** @extensions: Pointer to the first extension struct, if any */ -- cgit v1.2.3 From ea0640fc6971f555c8f921e2060376d768685805 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 20 Sep 2023 15:29:24 -0400 Subject: drm/xe/uapi: Separate VM_BIND's operation and flag Use different members in the drm_xe_vm_bind_op for op and for flags as it is done in other structures. Type is left to u32 to leave enough room for future operations and flags. v2: Remove the XE_VM_BIND_* flags shift (Rodrigo Vivi) Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/303 Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi Reviewed-by: Matthew Brost --- drivers/gpu/drm/xe/xe_vm.c | 29 ++++++++++++++++------------- include/uapi/drm/xe_drm.h | 14 ++++++++------ 2 files changed, 24 insertions(+), 19 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 035f3232e3b9..3ae911ade7e4 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2282,11 +2282,11 @@ static void vm_set_async_error(struct xe_vm *vm, int err) } static int vm_bind_ioctl_lookup_vma(struct xe_vm *vm, struct xe_bo *bo, - u64 addr, u64 range, u32 op) + u64 addr, u64 range, u32 op, u32 flags) { struct xe_device *xe = vm->xe; struct xe_vma *vma; - bool async = !!(op & XE_VM_BIND_FLAG_ASYNC); + bool async = !!(flags & XE_VM_BIND_FLAG_ASYNC); lockdep_assert_held(&vm->lock); @@ -2387,7 +2387,7 @@ static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) static struct drm_gpuva_ops * vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, u64 bo_offset_or_userptr, u64 addr, u64 range, - u32 operation, u8 tile_mask, u32 region) + u32 operation, u32 flags, u8 tile_mask, u32 region) { struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; struct drm_gpuva_ops *ops; @@ -2416,10 +2416,10 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, op->tile_mask = tile_mask; op->map.immediate = - operation & XE_VM_BIND_FLAG_IMMEDIATE; + flags & XE_VM_BIND_FLAG_IMMEDIATE; op->map.read_only = - operation & XE_VM_BIND_FLAG_READONLY; - op->map.is_null = operation & XE_VM_BIND_FLAG_NULL; + flags & XE_VM_BIND_FLAG_READONLY; + op->map.is_null = flags & XE_VM_BIND_FLAG_NULL; } break; case XE_VM_BIND_OP_UNMAP: @@ -3236,15 +3236,16 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, u64 range = (*bind_ops)[i].range; u64 addr = (*bind_ops)[i].addr; u32 op = (*bind_ops)[i].op; + u32 flags = (*bind_ops)[i].flags; u32 obj = (*bind_ops)[i].obj; u64 obj_offset = (*bind_ops)[i].obj_offset; u32 region = (*bind_ops)[i].region; - bool is_null = op & XE_VM_BIND_FLAG_NULL; + bool is_null = flags & XE_VM_BIND_FLAG_NULL; if (i == 0) { - *async = !!(op & XE_VM_BIND_FLAG_ASYNC); + *async = !!(flags & XE_VM_BIND_FLAG_ASYNC); } else if (XE_IOCTL_DBG(xe, !*async) || - XE_IOCTL_DBG(xe, !(op & XE_VM_BIND_FLAG_ASYNC)) || + XE_IOCTL_DBG(xe, !(flags & XE_VM_BIND_FLAG_ASYNC)) || XE_IOCTL_DBG(xe, VM_BIND_OP(op) == XE_VM_BIND_OP_RESTART)) { err = -EINVAL; @@ -3265,7 +3266,7 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, if (XE_IOCTL_DBG(xe, VM_BIND_OP(op) > XE_VM_BIND_OP_PREFETCH) || - XE_IOCTL_DBG(xe, op & ~SUPPORTED_FLAGS) || + XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) || XE_IOCTL_DBG(xe, obj && is_null) || XE_IOCTL_DBG(xe, obj_offset && is_null) || XE_IOCTL_DBG(xe, VM_BIND_OP(op) != XE_VM_BIND_OP_MAP && @@ -3480,8 +3481,9 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) u64 range = bind_ops[i].range; u64 addr = bind_ops[i].addr; u32 op = bind_ops[i].op; + u32 flags = bind_ops[i].flags; - err = vm_bind_ioctl_lookup_vma(vm, bos[i], addr, range, op); + err = vm_bind_ioctl_lookup_vma(vm, bos[i], addr, range, op, flags); if (err) goto free_syncs; } @@ -3490,13 +3492,14 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) u64 range = bind_ops[i].range; u64 addr = bind_ops[i].addr; u32 op = bind_ops[i].op; + u32 flags = bind_ops[i].flags; u64 obj_offset = bind_ops[i].obj_offset; u8 tile_mask = bind_ops[i].tile_mask; u32 region = bind_ops[i].region; ops[i] = vm_bind_ioctl_ops_create(vm, bos[i], obj_offset, - addr, range, op, tile_mask, - region); + addr, range, op, flags, + tile_mask, region); if (IS_ERR(ops[i])) { err = PTR_ERR(ops[i]); ops[i] = NULL; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 079213a3df55..46db9334159b 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -660,8 +660,10 @@ struct drm_xe_vm_bind_op { #define XE_VM_BIND_OP_RESTART 0x3 #define XE_VM_BIND_OP_UNMAP_ALL 0x4 #define XE_VM_BIND_OP_PREFETCH 0x5 + /** @op: Bind operation to perform */ + __u32 op; -#define XE_VM_BIND_FLAG_READONLY (0x1 << 16) +#define XE_VM_BIND_FLAG_READONLY (0x1 << 0) /* * A bind ops completions are always async, hence the support for out * sync. This flag indicates the allocation of the memory for new page @@ -686,12 +688,12 @@ struct drm_xe_vm_bind_op { * configured in the VM and must be set if the VM is configured with * DRM_XE_VM_CREATE_ASYNC_BIND_OPS and not in an error state. */ -#define XE_VM_BIND_FLAG_ASYNC (0x1 << 17) +#define XE_VM_BIND_FLAG_ASYNC (0x1 << 1) /* * Valid on a faulting VM only, do the MAP operation immediately rather * than deferring the MAP to the page fault handler. */ -#define XE_VM_BIND_FLAG_IMMEDIATE (0x1 << 18) +#define XE_VM_BIND_FLAG_IMMEDIATE (0x1 << 2) /* * When the NULL flag is set, the page tables are setup with a special * bit which indicates writes are dropped and all reads return zero. In @@ -699,9 +701,9 @@ struct drm_xe_vm_bind_op { * operations, the BO handle MBZ, and the BO offset MBZ. This flag is * intended to implement VK sparse bindings. */ -#define XE_VM_BIND_FLAG_NULL (0x1 << 19) - /** @op: Operation to perform (lower 16 bits) and flags (upper 16 bits) */ - __u32 op; +#define XE_VM_BIND_FLAG_NULL (0x1 << 3) + /** @flags: Bind flags */ + __u32 flags; /** @mem_region: Memory region to prefetch VMA to, instance not a mask */ __u32 region; -- cgit v1.2.3 From 924e6a9789a05ef01ffdf849aa3a3c75f5a29a8b Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 20 Sep 2023 15:29:26 -0400 Subject: drm/xe/uapi: Remove MMIO ioctl MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This was previously used in UMD for timestamp correlation, which can now be done with DRM_XE_QUERY_CS_CYCLES. Link: https://lore.kernel.org/all/20230706042044.GR6953@mdroper-desk1.amr.corp.intel.com/ Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/636 Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza Reviewed-by: Lucas De Marchi Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_device.c | 1 - drivers/gpu/drm/xe/xe_mmio.c | 102 ----------------------------------------- drivers/gpu/drm/xe/xe_mmio.h | 3 -- include/uapi/drm/xe_drm.h | 31 ++----------- 4 files changed, 4 insertions(+), 133 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 2bbd3aa2809b..ae0b7349c3e3 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -121,7 +121,6 @@ static const struct drm_ioctl_desc xe_ioctls[] = { DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(XE_MMIO, xe_mmio_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, diff --git a/drivers/gpu/drm/xe/xe_mmio.c b/drivers/gpu/drm/xe/xe_mmio.c index e4cf9bfec422..0da4f75c07bf 100644 --- a/drivers/gpu/drm/xe/xe_mmio.c +++ b/drivers/gpu/drm/xe/xe_mmio.c @@ -429,108 +429,6 @@ int xe_mmio_init(struct xe_device *xe) return 0; } -#define VALID_MMIO_FLAGS (\ - DRM_XE_MMIO_BITS_MASK |\ - DRM_XE_MMIO_READ |\ - DRM_XE_MMIO_WRITE) - -static const struct xe_reg mmio_read_whitelist[] = { - RING_TIMESTAMP(RENDER_RING_BASE), -}; - -int xe_mmio_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct xe_device *xe = to_xe_device(dev); - struct xe_gt *gt = xe_root_mmio_gt(xe); - struct drm_xe_mmio *args = data; - unsigned int bits_flag, bytes; - struct xe_reg reg; - bool allowed; - int ret = 0; - - if (XE_IOCTL_DBG(xe, args->extensions) || - XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, args->flags & ~VALID_MMIO_FLAGS)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_MMIO_WRITE) && args->value)) - return -EINVAL; - - allowed = capable(CAP_SYS_ADMIN); - if (!allowed && ((args->flags & ~DRM_XE_MMIO_BITS_MASK) == DRM_XE_MMIO_READ)) { - unsigned int i; - - for (i = 0; i < ARRAY_SIZE(mmio_read_whitelist); i++) { - if (mmio_read_whitelist[i].addr == args->addr) { - allowed = true; - break; - } - } - } - - if (XE_IOCTL_DBG(xe, !allowed)) - return -EPERM; - - bits_flag = args->flags & DRM_XE_MMIO_BITS_MASK; - bytes = 1 << bits_flag; - if (XE_IOCTL_DBG(xe, args->addr + bytes > xe->mmio.size)) - return -EINVAL; - - /* - * TODO: migrate to xe_gt_mcr to lookup the mmio range and handle - * multicast registers. Steering would need uapi extension. - */ - reg = XE_REG(args->addr); - - xe_device_mem_access_get(xe); - xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); - - if (args->flags & DRM_XE_MMIO_WRITE) { - switch (bits_flag) { - case DRM_XE_MMIO_32BIT: - if (XE_IOCTL_DBG(xe, args->value > U32_MAX)) { - ret = -EINVAL; - goto exit; - } - xe_mmio_write32(gt, reg, args->value); - break; - default: - drm_dbg(&xe->drm, "Invalid MMIO bit size"); - fallthrough; - case DRM_XE_MMIO_8BIT: /* TODO */ - case DRM_XE_MMIO_16BIT: /* TODO */ - ret = -EOPNOTSUPP; - goto exit; - } - } - - if (args->flags & DRM_XE_MMIO_READ) { - switch (bits_flag) { - case DRM_XE_MMIO_32BIT: - args->value = xe_mmio_read32(gt, reg); - break; - case DRM_XE_MMIO_64BIT: - args->value = xe_mmio_read64_2x32(gt, reg); - break; - default: - drm_dbg(&xe->drm, "Invalid MMIO bit size"); - fallthrough; - case DRM_XE_MMIO_8BIT: /* TODO */ - case DRM_XE_MMIO_16BIT: /* TODO */ - ret = -EOPNOTSUPP; - } - } - -exit: - xe_force_wake_put(gt_to_fw(gt), XE_FORCEWAKE_ALL); - xe_device_mem_access_put(xe); - - return ret; -} - /** * xe_mmio_read64_2x32() - Read a 64-bit register as two 32-bit reads * @gt: MMIO target GT diff --git a/drivers/gpu/drm/xe/xe_mmio.h b/drivers/gpu/drm/xe/xe_mmio.h index ae09f777d711..24a23dad7dce 100644 --- a/drivers/gpu/drm/xe/xe_mmio.h +++ b/drivers/gpu/drm/xe/xe_mmio.h @@ -124,9 +124,6 @@ static inline int xe_mmio_wait32(struct xe_gt *gt, struct xe_reg reg, u32 mask, return ret; } -int xe_mmio_ioctl(struct drm_device *dev, void *data, - struct drm_file *file); - static inline bool xe_mmio_in_range(const struct xe_gt *gt, const struct xe_mmio_range *range, struct xe_reg reg) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 46db9334159b..ad21ba1d6e0b 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -106,11 +106,10 @@ struct xe_user_extension { #define DRM_XE_EXEC_QUEUE_CREATE 0x06 #define DRM_XE_EXEC_QUEUE_DESTROY 0x07 #define DRM_XE_EXEC 0x08 -#define DRM_XE_MMIO 0x09 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x0a -#define DRM_XE_WAIT_USER_FENCE 0x0b -#define DRM_XE_VM_MADVISE 0x0c -#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x0d +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x09 +#define DRM_XE_WAIT_USER_FENCE 0x0a +#define DRM_XE_VM_MADVISE 0x0b +#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x0c /* Must be kept compact -- no holes */ #define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query) @@ -123,7 +122,6 @@ struct xe_user_extension { #define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property) #define DRM_IOCTL_XE_EXEC_QUEUE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_DESTROY, struct drm_xe_exec_queue_destroy) #define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) -#define DRM_IOCTL_XE_MMIO DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_MMIO, struct drm_xe_mmio) #define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property) #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) #define DRM_IOCTL_XE_VM_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_MADVISE, struct drm_xe_vm_madvise) @@ -936,27 +934,6 @@ struct drm_xe_exec { __u64 reserved[2]; }; -struct drm_xe_mmio { - /** @extensions: Pointer to the first extension struct, if any */ - __u64 extensions; - - __u32 addr; - -#define DRM_XE_MMIO_8BIT 0x0 -#define DRM_XE_MMIO_16BIT 0x1 -#define DRM_XE_MMIO_32BIT 0x2 -#define DRM_XE_MMIO_64BIT 0x3 -#define DRM_XE_MMIO_BITS_MASK 0x3 -#define DRM_XE_MMIO_READ 0x4 -#define DRM_XE_MMIO_WRITE 0x8 - __u32 flags; - - __u64 value; - - /** @reserved: Reserved */ - __u64 reserved[2]; -}; - /** * struct drm_xe_wait_user_fence - wait user fence * -- cgit v1.2.3 From bffb2573726beabc8ad70532d5655a976f9053d8 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 20 Sep 2023 15:29:30 -0400 Subject: drm/xe: Remove XE_EXEC_QUEUE_SET_PROPERTY_COMPUTE_MODE from uAPI Functionality of XE_EXEC_QUEUE_SET_PROPERTY_COMPUTE_MODE deprecated in a previous patch, drop from uAPI. The property is just simply inherented from the VM. v2: - Update commit message (Niranjana) Reviewed-by: Niranjana Vishwanathapura Signed-off-by: Matthew Brost Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_exec_queue.c | 7 ------- include/uapi/drm/xe_drm.h | 19 ++++++------------- 2 files changed, 6 insertions(+), 20 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index d400e2bb3785..5714a7195349 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -320,12 +320,6 @@ static int exec_queue_set_preemption_timeout(struct xe_device *xe, return q->ops->set_preempt_timeout(q, value); } -static int exec_queue_set_compute_mode(struct xe_device *xe, struct xe_exec_queue *q, - u64 value, bool create) -{ - return 0; -} - static int exec_queue_set_persistence(struct xe_device *xe, struct xe_exec_queue *q, u64 value, bool create) { @@ -411,7 +405,6 @@ static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { [XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, [XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, [XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT] = exec_queue_set_preemption_timeout, - [XE_EXEC_QUEUE_SET_PROPERTY_COMPUTE_MODE] = exec_queue_set_compute_mode, [XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE] = exec_queue_set_persistence, [XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT] = exec_queue_set_job_timeout, [XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER] = exec_queue_set_acc_trigger, diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index ad21ba1d6e0b..2a9e04024723 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -781,21 +781,14 @@ struct drm_xe_exec_queue_set_property { /** @exec_queue_id: Exec queue ID */ __u32 exec_queue_id; -#define XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 +#define XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 #define XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1 #define XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT 2 - /* - * Long running or ULLS engine mode. DMA fences not allowed in this - * mode. Must match the value of DRM_XE_VM_CREATE_COMPUTE_MODE, serves - * as a sanity check the UMD knows what it is doing. Can only be set at - * engine create time. - */ -#define XE_EXEC_QUEUE_SET_PROPERTY_COMPUTE_MODE 3 -#define XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE 4 -#define XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT 5 -#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER 6 -#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY 7 -#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY 8 +#define XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE 3 +#define XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT 4 +#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER 5 +#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY 6 +#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY 7 /** @property: property to set */ __u32 property; -- cgit v1.2.3 From 5dc079d1a8e5e880ae18b4f4585d7dc28e51e68e Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Wed, 20 Sep 2023 15:29:31 -0400 Subject: drm/xe/uapi: Use common drm_xe_ext_set_property extension There really is no difference between 'struct drm_xe_ext_vm_set_property' and 'struct drm_xe_ext_exec_queue_set_property', they are extensions which specify a pair. Replace the two extensions with a single common 'struct drm_xe_ext_set_property' extension. The rationale is that rather than have each XE module (including future modules) invent their own property/value extensions, all XE modules use a common set_property extension when possible. Signed-off-by: Ashutosh Dixit Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi Reviewed-by: Matthew Brost --- drivers/gpu/drm/xe/xe_exec_queue.c | 2 +- drivers/gpu/drm/xe/xe_vm.c | 2 +- include/uapi/drm/xe_drm.h | 21 +++------------------ 3 files changed, 5 insertions(+), 20 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 5714a7195349..38ce777d0ba8 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -418,7 +418,7 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe, bool create) { u64 __user *address = u64_to_user_ptr(extension); - struct drm_xe_ext_exec_queue_set_property ext; + struct drm_xe_ext_set_property ext; int err; u32 idx; diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index d02c0db5e2ae..3d350b27732f 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2067,7 +2067,7 @@ static int vm_user_ext_set_property(struct xe_device *xe, struct xe_vm *vm, u64 extension) { u64 __user *address = u64_to_user_ptr(extension); - struct drm_xe_ext_vm_set_property ext; + struct drm_xe_ext_set_property ext; int err; err = __copy_from_user(&ext, address, sizeof(ext)); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 2a9e04024723..4987a634afc7 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -569,12 +569,11 @@ struct drm_xe_vm_bind_op_error_capture { __u64 size; }; -/** struct drm_xe_ext_vm_set_property - VM set property extension */ -struct drm_xe_ext_vm_set_property { +/** struct drm_xe_ext_set_property - XE set property extension */ +struct drm_xe_ext_set_property { /** @base: base user extension */ struct xe_user_extension base; -#define XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS 0 /** @property: property to set */ __u32 property; @@ -590,6 +589,7 @@ struct drm_xe_ext_vm_set_property { struct drm_xe_vm_create { #define XE_VM_EXTENSION_SET_PROPERTY 0 +#define XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS 0 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -754,21 +754,6 @@ struct drm_xe_vm_bind { __u64 reserved[2]; }; -/** struct drm_xe_ext_exec_queue_set_property - exec queue set property extension */ -struct drm_xe_ext_exec_queue_set_property { - /** @base: base user extension */ - struct xe_user_extension base; - - /** @property: property to set */ - __u32 property; - - /** @pad: MBZ */ - __u32 pad; - - /** @value: property value */ - __u64 value; -}; - /** * struct drm_xe_exec_queue_set_property - exec queue set property * -- cgit v1.2.3 From 7224788f675632956cb9177c039645d72d887cf8 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 20 Sep 2023 15:29:32 -0400 Subject: drm/xe: Kill XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This extension is currently not used and it is not aligned with the error handling on async VM_BIND. Let's remove it and along with that, since it was the only extension for the vm_create, remove VM extension entirely. v2: rebase on top of the removal of drm_xe_ext_exec_queue_set_property Cc: Thomas Hellström Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi Reviewed-by: Matthew Brost --- drivers/gpu/drm/xe/xe_vm.c | 129 ++------------------------------------------- include/uapi/drm/xe_drm.h | 23 +------- 2 files changed, 4 insertions(+), 148 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 3d350b27732f..c7e3b1fbd931 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1531,37 +1531,6 @@ static void flush_async_ops(struct xe_vm *vm) flush_work(&vm->async_ops.work); } -static void vm_error_capture(struct xe_vm *vm, int err, - u32 op, u64 addr, u64 size) -{ - struct drm_xe_vm_bind_op_error_capture capture; - u64 __user *address = - u64_to_user_ptr(vm->async_ops.error_capture.addr); - bool in_kthread = !current->mm; - - capture.error = err; - capture.op = op; - capture.addr = addr; - capture.size = size; - - if (in_kthread) { - if (!mmget_not_zero(vm->async_ops.error_capture.mm)) - goto mm_closed; - kthread_use_mm(vm->async_ops.error_capture.mm); - } - - if (copy_to_user(address, &capture, sizeof(capture))) - drm_warn(&vm->xe->drm, "Copy to user failed"); - - if (in_kthread) { - kthread_unuse_mm(vm->async_ops.error_capture.mm); - mmput(vm->async_ops.error_capture.mm); - } - -mm_closed: - wake_up_all(&vm->async_ops.error_capture.wq); -} - static void xe_vm_close(struct xe_vm *vm) { down_write(&vm->lock); @@ -2036,91 +2005,6 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, return 0; } -static int vm_set_error_capture_address(struct xe_device *xe, struct xe_vm *vm, - u64 value) -{ - if (XE_IOCTL_DBG(xe, !value)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, !(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) - return -EOPNOTSUPP; - - if (XE_IOCTL_DBG(xe, vm->async_ops.error_capture.addr)) - return -EOPNOTSUPP; - - vm->async_ops.error_capture.mm = current->mm; - vm->async_ops.error_capture.addr = value; - init_waitqueue_head(&vm->async_ops.error_capture.wq); - - return 0; -} - -typedef int (*xe_vm_set_property_fn)(struct xe_device *xe, struct xe_vm *vm, - u64 value); - -static const xe_vm_set_property_fn vm_set_property_funcs[] = { - [XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS] = - vm_set_error_capture_address, -}; - -static int vm_user_ext_set_property(struct xe_device *xe, struct xe_vm *vm, - u64 extension) -{ - u64 __user *address = u64_to_user_ptr(extension); - struct drm_xe_ext_set_property ext; - int err; - - err = __copy_from_user(&ext, address, sizeof(ext)); - if (XE_IOCTL_DBG(xe, err)) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, ext.property >= - ARRAY_SIZE(vm_set_property_funcs)) || - XE_IOCTL_DBG(xe, ext.pad) || - XE_IOCTL_DBG(xe, ext.reserved[0] || ext.reserved[1])) - return -EINVAL; - - return vm_set_property_funcs[ext.property](xe, vm, ext.value); -} - -typedef int (*xe_vm_user_extension_fn)(struct xe_device *xe, struct xe_vm *vm, - u64 extension); - -static const xe_vm_set_property_fn vm_user_extension_funcs[] = { - [XE_VM_EXTENSION_SET_PROPERTY] = vm_user_ext_set_property, -}; - -#define MAX_USER_EXTENSIONS 16 -static int vm_user_extensions(struct xe_device *xe, struct xe_vm *vm, - u64 extensions, int ext_number) -{ - u64 __user *address = u64_to_user_ptr(extensions); - struct xe_user_extension ext; - int err; - - if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) - return -E2BIG; - - err = __copy_from_user(&ext, address, sizeof(ext)); - if (XE_IOCTL_DBG(xe, err)) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, ext.pad) || - XE_IOCTL_DBG(xe, ext.name >= - ARRAY_SIZE(vm_user_extension_funcs))) - return -EINVAL; - - err = vm_user_extension_funcs[ext.name](xe, vm, extensions); - if (XE_IOCTL_DBG(xe, err)) - return err; - - if (ext.next_extension) - return vm_user_extensions(xe, vm, ext.next_extension, - ++ext_number); - - return 0; -} - #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_SCRATCH_PAGE | \ DRM_XE_VM_CREATE_COMPUTE_MODE | \ DRM_XE_VM_CREATE_ASYNC_BIND_OPS | \ @@ -2138,6 +2022,9 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, int err; u32 flags = 0; + if (XE_IOCTL_DBG(xe, args->extensions)) + return -EINVAL; + if (XE_WA(xe_root_mmio_gt(xe), 14016763929)) args->flags |= DRM_XE_VM_CREATE_SCRATCH_PAGE; @@ -2180,14 +2067,6 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, if (IS_ERR(vm)) return PTR_ERR(vm); - if (args->extensions) { - err = vm_user_extensions(xe, vm, args->extensions, 0); - if (XE_IOCTL_DBG(xe, err)) { - xe_vm_close_and_put(vm); - return err; - } - } - mutex_lock(&xef->vm.lock); err = xa_alloc(&xef->vm.xa, &id, vm, xa_limit_32b, GFP_KERNEL); mutex_unlock(&xef->vm.lock); @@ -3087,8 +2966,6 @@ static void xe_vma_op_work_func(struct work_struct *w) vm_set_async_error(vm, err); up_write(&vm->lock); - if (vm->async_ops.error_capture.addr) - vm_error_capture(vm, err, 0, 0, 0); break; } up_write(&vm->lock); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 4987a634afc7..e7cf42c7234b 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -552,23 +552,6 @@ struct drm_xe_gem_mmap_offset { __u64 reserved[2]; }; -/** - * struct drm_xe_vm_bind_op_error_capture - format of VM bind op error capture - */ -struct drm_xe_vm_bind_op_error_capture { - /** @error: errno that occurred */ - __s32 error; - - /** @op: operation that encounter an error */ - __u32 op; - - /** @addr: address of bind op */ - __u64 addr; - - /** @size: size of bind */ - __u64 size; -}; - /** struct drm_xe_ext_set_property - XE set property extension */ struct drm_xe_ext_set_property { /** @base: base user extension */ @@ -589,7 +572,6 @@ struct drm_xe_ext_set_property { struct drm_xe_vm_create { #define XE_VM_EXTENSION_SET_PROPERTY 0 -#define XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS 0 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -674,10 +656,7 @@ struct drm_xe_vm_bind_op { * practice the bind op is good and will complete. * * If this flag is set and doesn't return an error, the bind op can - * still fail and recovery is needed. If configured, the bind op that - * caused the error will be captured in drm_xe_vm_bind_op_error_capture. - * Once the user sees the error (via a ufence + - * XE_VM_PROPERTY_BIND_OP_ERROR_CAPTURE_ADDRESS), it should free memory + * still fail and recovery is needed. It should free memory * via non-async unbinds, and then restart all queued async binds op via * XE_VM_BIND_OP_RESTART. Or alternatively the user should destroy the * VM. -- cgit v1.2.3 From b21ae51dcf41ce12bb8e2a7c989863ee9d04ae4b Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 14 Sep 2023 13:40:49 -0700 Subject: drm/xe/uapi: Kill DRM_XE_UFENCE_WAIT_VM_ERROR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is not used nor does it align VM async document, kill this. Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_vm.c | 3 --- drivers/gpu/drm/xe/xe_vm_types.h | 11 --------- drivers/gpu/drm/xe/xe_wait_user_fence.c | 43 ++++----------------------------- include/uapi/drm/xe_drm.h | 17 +++---------- 4 files changed, 9 insertions(+), 65 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index c7e3b1fbd931..3132114d187f 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1621,9 +1621,6 @@ void xe_vm_close_and_put(struct xe_vm *vm) xe_vma_destroy_unlocked(vma); } - if (vm->async_ops.error_capture.addr) - wake_up_all(&vm->async_ops.error_capture.wq); - xe_assert(xe, list_empty(&vm->extobj.list)); up_write(&vm->lock); diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 9a1075a75606..828ed0fa7e60 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -215,17 +215,6 @@ struct xe_vm { struct work_struct work; /** @lock: protects list of pending async VM ops and fences */ spinlock_t lock; - /** @error_capture: error capture state */ - struct { - /** @mm: user MM */ - struct mm_struct *mm; - /** - * @addr: user pointer to copy error capture state too - */ - u64 addr; - /** @wq: user fence wait queue for VM errors */ - wait_queue_head_t wq; - } error_capture; /** @fence: fence state */ struct { /** @context: context of async fence */ diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.c b/drivers/gpu/drm/xe/xe_wait_user_fence.c index 3ac4cd24d5b4..78686908f7fb 100644 --- a/drivers/gpu/drm/xe/xe_wait_user_fence.c +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.c @@ -13,7 +13,6 @@ #include "xe_device.h" #include "xe_gt.h" #include "xe_macros.h" -#include "xe_vm.h" static int do_compare(u64 addr, u64 value, u64 mask, u16 op) { @@ -81,8 +80,7 @@ static int check_hw_engines(struct xe_device *xe, } #define VALID_FLAGS (DRM_XE_UFENCE_WAIT_SOFT_OP | \ - DRM_XE_UFENCE_WAIT_ABSTIME | \ - DRM_XE_UFENCE_WAIT_VM_ERROR) + DRM_XE_UFENCE_WAIT_ABSTIME) #define MAX_OP DRM_XE_UFENCE_WAIT_LTE static long to_jiffies_timeout(struct xe_device *xe, @@ -137,11 +135,9 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; struct drm_xe_engine_class_instance __user *user_eci = u64_to_user_ptr(args->instances); - struct xe_vm *vm = NULL; u64 addr = args->addr; int err; - bool no_engines = args->flags & DRM_XE_UFENCE_WAIT_SOFT_OP || - args->flags & DRM_XE_UFENCE_WAIT_VM_ERROR; + bool no_engines = args->flags & DRM_XE_UFENCE_WAIT_SOFT_OP; long timeout; ktime_t start; @@ -162,8 +158,7 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, !no_engines && !args->num_engines)) return -EINVAL; - if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_UFENCE_WAIT_VM_ERROR) && - addr & 0x7)) + if (XE_IOCTL_DBG(xe, addr & 0x7)) return -EINVAL; if (XE_IOCTL_DBG(xe, args->num_engines > XE_HW_ENGINE_MAX_INSTANCE)) @@ -181,22 +176,6 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, return -EINVAL; } - if (args->flags & DRM_XE_UFENCE_WAIT_VM_ERROR) { - if (XE_IOCTL_DBG(xe, args->vm_id >> 32)) - return -EINVAL; - - vm = xe_vm_lookup(to_xe_file(file), args->vm_id); - if (XE_IOCTL_DBG(xe, !vm)) - return -ENOENT; - - if (XE_IOCTL_DBG(xe, !vm->async_ops.error_capture.addr)) { - xe_vm_put(vm); - return -EOPNOTSUPP; - } - - addr = vm->async_ops.error_capture.addr; - } - timeout = to_jiffies_timeout(xe, args); start = ktime_get(); @@ -207,15 +186,8 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, * hardware engine. Open coding as 'do_compare' can sleep which doesn't * work with the wait_event_* macros. */ - if (vm) - add_wait_queue(&vm->async_ops.error_capture.wq, &w_wait); - else - add_wait_queue(&xe->ufence_wq, &w_wait); + add_wait_queue(&xe->ufence_wq, &w_wait); for (;;) { - if (vm && xe_vm_is_closed(vm)) { - err = -ENODEV; - break; - } err = do_compare(addr, args->value, args->mask, args->op); if (err <= 0) break; @@ -232,12 +204,7 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, timeout = wait_woken(&w_wait, TASK_INTERRUPTIBLE, timeout); } - if (vm) { - remove_wait_queue(&vm->async_ops.error_capture.wq, &w_wait); - xe_vm_put(vm); - } else { - remove_wait_queue(&xe->ufence_wq, &w_wait); - } + remove_wait_queue(&xe->ufence_wq, &w_wait); if (!(args->flags & DRM_XE_UFENCE_WAIT_ABSTIME)) { args->timeout -= ktime_to_ns(ktime_sub(ktime_get(), start)); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index e7cf42c7234b..f13974f17be9 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -905,18 +905,10 @@ struct drm_xe_wait_user_fence { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; - union { - /** - * @addr: user pointer address to wait on, must qword aligned - */ - __u64 addr; - - /** - * @vm_id: The ID of the VM which encounter an error used with - * DRM_XE_UFENCE_WAIT_VM_ERROR. Upper 32 bits must be clear. - */ - __u64 vm_id; - }; + /** + * @addr: user pointer address to wait on, must qword aligned + */ + __u64 addr; #define DRM_XE_UFENCE_WAIT_EQ 0 #define DRM_XE_UFENCE_WAIT_NEQ 1 @@ -929,7 +921,6 @@ struct drm_xe_wait_user_fence { #define DRM_XE_UFENCE_WAIT_SOFT_OP (1 << 0) /* e.g. Wait on VM bind */ #define DRM_XE_UFENCE_WAIT_ABSTIME (1 << 1) -#define DRM_XE_UFENCE_WAIT_VM_ERROR (1 << 2) /** @flags: wait flags */ __u16 flags; -- cgit v1.2.3 From f3e9b1f43458746e7e0211dbe4289412e5c0d16a Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Thu, 14 Sep 2023 13:40:50 -0700 Subject: drm/xe: Remove async worker and rework sync binds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Async worker is gone. All jobs and memory allocations done in IOCTL to align with dma fencing rules. Async vs. sync now means when do bind operations complete relative to the IOCTL. Async completes when out-syncs signal while sync completes when the IOCTL returns. In-syncs and out-syncs are only allowed in async mode. If memory allocations fail in the job creation step the VM is killed. This is temporary, eventually a proper unwind will be done and VM will be usable. Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_exec.c | 43 --- drivers/gpu/drm/xe/xe_exec_queue.c | 7 +- drivers/gpu/drm/xe/xe_exec_queue_types.h | 2 + drivers/gpu/drm/xe/xe_sync.c | 14 +- drivers/gpu/drm/xe/xe_sync.h | 2 +- drivers/gpu/drm/xe/xe_vm.c | 535 +++++++------------------------ drivers/gpu/drm/xe/xe_vm.h | 2 - drivers/gpu/drm/xe/xe_vm_types.h | 7 +- include/uapi/drm/xe_drm.h | 33 +- 9 files changed, 127 insertions(+), 518 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index 7cf4215b2b2e..85a8a793f527 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -196,27 +196,6 @@ int xe_exec_ioctl(struct drm_device *dev, void *data, struct drm_file *file) } } - /* - * We can't install a job into the VM dma-resv shared slot before an - * async VM bind passed in as a fence without the risk of deadlocking as - * the bind can trigger an eviction which in turn depends on anything in - * the VM dma-resv shared slots. Not an ideal solution, but we wait for - * all dependent async VM binds to start (install correct fences into - * dma-resv slots) before moving forward. - */ - if (!xe_vm_no_dma_fences(vm) && - vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS) { - for (i = 0; i < args->num_syncs; i++) { - struct dma_fence *fence = syncs[i].fence; - - if (fence) { - err = xe_vm_async_fence_wait_start(fence); - if (err) - goto err_syncs; - } - } - } - retry: if (!xe_vm_no_dma_fences(vm) && xe_vm_userptr_check_repin(vm)) { err = down_write_killable(&vm->lock); @@ -229,28 +208,6 @@ retry: if (err) goto err_syncs; - /* We don't allow execs while the VM is in error state */ - if (vm->async_ops.error) { - err = vm->async_ops.error; - goto err_unlock_list; - } - - /* - * Extreme corner where we exit a VM error state with a munmap style VM - * unbind inflight which requires a rebind. In this case the rebind - * needs to install some fences into the dma-resv slots. The worker to - * do this queued, let that worker make progress by dropping vm->lock, - * flushing the worker and retrying the exec. - */ - if (vm->async_ops.munmap_rebind_inflight) { - if (write_locked) - up_write(&vm->lock); - else - up_read(&vm->lock); - flush_work(&vm->async_ops.work); - goto retry; - } - if (write_locked) { err = xe_vm_userptr_pin(vm); downgrade_write(&vm->lock); diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 38ce777d0ba8..9b373b9ea472 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -621,7 +621,10 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) return -EINVAL; - if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { + if (eci[0].engine_class >= DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC) { + bool sync = eci[0].engine_class == + DRM_XE_ENGINE_CLASS_VM_BIND_SYNC; + for_each_gt(gt, xe, id) { struct xe_exec_queue *new; @@ -647,6 +650,8 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, args->width, hwe, EXEC_QUEUE_FLAG_PERSISTENT | EXEC_QUEUE_FLAG_VM | + (sync ? 0 : + EXEC_QUEUE_FLAG_VM_ASYNC) | (id ? EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 0)); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index c4813944b017..4e382304010e 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -77,6 +77,8 @@ struct xe_exec_queue { #define EXEC_QUEUE_FLAG_VM BIT(4) /* child of VM queue for multi-tile VM jobs */ #define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD BIT(5) +/* VM jobs for this queue are asynchronous */ +#define EXEC_QUEUE_FLAG_VM_ASYNC BIT(6) /** * @flags: flags for this exec queue, should statically setup aside from ban diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index 9fcd7802ba30..73ef259aa387 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -18,7 +18,6 @@ #include "xe_sched_job_types.h" #define SYNC_FLAGS_TYPE_MASK 0x3 -#define SYNC_FLAGS_FENCE_INSTALLED 0x10000 struct user_fence { struct xe_device *xe; @@ -223,12 +222,11 @@ int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) return 0; } -bool xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, +void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, struct dma_fence *fence) { - if (!(sync->flags & DRM_XE_SYNC_SIGNAL) || - sync->flags & SYNC_FLAGS_FENCE_INSTALLED) - return false; + if (!(sync->flags & DRM_XE_SYNC_SIGNAL)) + return; if (sync->chain_fence) { drm_syncobj_add_point(sync->syncobj, sync->chain_fence, @@ -260,12 +258,6 @@ bool xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, job->user_fence.addr = sync->addr; job->user_fence.value = sync->timeline_value; } - - /* TODO: external BO? */ - - sync->flags |= SYNC_FLAGS_FENCE_INSTALLED; - - return true; } void xe_sync_entry_cleanup(struct xe_sync_entry *sync) diff --git a/drivers/gpu/drm/xe/xe_sync.h b/drivers/gpu/drm/xe/xe_sync.h index 4cbcf7a19911..30958ddc4cdc 100644 --- a/drivers/gpu/drm/xe/xe_sync.h +++ b/drivers/gpu/drm/xe/xe_sync.h @@ -19,7 +19,7 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, int xe_sync_entry_wait(struct xe_sync_entry *sync); int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job); -bool xe_sync_entry_signal(struct xe_sync_entry *sync, +void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, struct dma_fence *fence); void xe_sync_entry_cleanup(struct xe_sync_entry *sync); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 3132114d187f..89df50f49e11 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -592,7 +592,7 @@ static void preempt_rebind_work_func(struct work_struct *w) unsigned int fence_count = 0; LIST_HEAD(preempt_fences); ktime_t end = 0; - int err; + int err = 0; long wait; int __maybe_unused tries = 0; @@ -608,22 +608,6 @@ static void preempt_rebind_work_func(struct work_struct *w) } retry: - if (vm->async_ops.error) - goto out_unlock_outer; - - /* - * Extreme corner where we exit a VM error state with a munmap style VM - * unbind inflight which requires a rebind. In this case the rebind - * needs to install some fences into the dma-resv slots. The worker to - * do this queued, let that worker make progress by dropping vm->lock - * and trying this again. - */ - if (vm->async_ops.munmap_rebind_inflight) { - up_write(&vm->lock); - flush_work(&vm->async_ops.work); - goto retry; - } - if (xe_vm_userptr_check_repin(vm)) { err = xe_vm_userptr_pin(vm); if (err) @@ -1357,7 +1341,6 @@ static const struct xe_pt_ops xelp_pt_ops = { .pde_encode_bo = xelp_pde_encode_bo, }; -static void xe_vma_op_work_func(struct work_struct *w); static void vm_destroy_work_func(struct work_struct *w); struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) @@ -1390,10 +1373,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) INIT_LIST_HEAD(&vm->notifier.rebind_list); spin_lock_init(&vm->notifier.list_lock); - INIT_LIST_HEAD(&vm->async_ops.pending); - INIT_WORK(&vm->async_ops.work, xe_vma_op_work_func); - spin_lock_init(&vm->async_ops.lock); - INIT_WORK(&vm->destroy_work, vm_destroy_work_func); INIT_LIST_HEAD(&vm->preempt.exec_queues); @@ -1458,11 +1437,6 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) vm->batch_invalidate_tlb = false; } - if (flags & XE_VM_FLAG_ASYNC_BIND_OPS) { - vm->async_ops.fence.context = dma_fence_context_alloc(1); - vm->flags |= XE_VM_FLAG_ASYNC_BIND_OPS; - } - /* Fill pt_root after allocating scratch tables */ for_each_tile(tile, xe, id) { if (!vm->pt_root[id]) @@ -1478,6 +1452,9 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) struct xe_gt *gt = tile->primary_gt; struct xe_vm *migrate_vm; struct xe_exec_queue *q; + u32 create_flags = EXEC_QUEUE_FLAG_VM | + ((flags & XE_VM_FLAG_ASYNC_DEFAULT) ? + EXEC_QUEUE_FLAG_VM_ASYNC : 0); if (!vm->pt_root[id]) continue; @@ -1485,7 +1462,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) migrate_vm = xe_migrate_get_vm(tile->migrate); q = xe_exec_queue_create_class(xe, gt, migrate_vm, XE_ENGINE_CLASS_COPY, - EXEC_QUEUE_FLAG_VM); + create_flags); xe_vm_put(migrate_vm); if (IS_ERR(q)) { err = PTR_ERR(q); @@ -1525,12 +1502,6 @@ err_no_resv: return ERR_PTR(err); } -static void flush_async_ops(struct xe_vm *vm) -{ - queue_work(system_unbound_wq, &vm->async_ops.work); - flush_work(&vm->async_ops.work); -} - static void xe_vm_close(struct xe_vm *vm) { down_write(&vm->lock); @@ -1550,7 +1521,6 @@ void xe_vm_close_and_put(struct xe_vm *vm) xe_assert(xe, !vm->preempt.num_exec_queues); xe_vm_close(vm); - flush_async_ops(vm); if (xe_vm_in_compute_mode(vm)) flush_work(&vm->preempt.rebind_work); @@ -1761,10 +1731,8 @@ next: err_fences: if (fences) { - while (cur_fence) { - /* FIXME: Rewind the previous binds? */ + while (cur_fence) dma_fence_put(fences[--cur_fence]); - } kfree(fences); } @@ -1838,100 +1806,24 @@ next: err_fences: if (fences) { - while (cur_fence) { - /* FIXME: Rewind the previous binds? */ + while (cur_fence) dma_fence_put(fences[--cur_fence]); - } kfree(fences); } return ERR_PTR(err); } -struct async_op_fence { - struct dma_fence fence; - struct dma_fence *wait_fence; - struct dma_fence_cb cb; - struct xe_vm *vm; - wait_queue_head_t wq; - bool started; -}; - -static const char *async_op_fence_get_driver_name(struct dma_fence *dma_fence) -{ - return "xe"; -} - -static const char * -async_op_fence_get_timeline_name(struct dma_fence *dma_fence) -{ - return "async_op_fence"; -} - -static const struct dma_fence_ops async_op_fence_ops = { - .get_driver_name = async_op_fence_get_driver_name, - .get_timeline_name = async_op_fence_get_timeline_name, -}; - -static void async_op_fence_cb(struct dma_fence *fence, struct dma_fence_cb *cb) -{ - struct async_op_fence *afence = - container_of(cb, struct async_op_fence, cb); - - afence->fence.error = afence->wait_fence->error; - dma_fence_signal(&afence->fence); - xe_vm_put(afence->vm); - dma_fence_put(afence->wait_fence); - dma_fence_put(&afence->fence); -} - -static void add_async_op_fence_cb(struct xe_vm *vm, - struct dma_fence *fence, - struct async_op_fence *afence) +static bool xe_vm_sync_mode(struct xe_vm *vm, struct xe_exec_queue *q) { - int ret; - - if (!xe_vm_no_dma_fences(vm)) { - afence->started = true; - smp_wmb(); - wake_up_all(&afence->wq); - } - - afence->wait_fence = dma_fence_get(fence); - afence->vm = xe_vm_get(vm); - dma_fence_get(&afence->fence); - ret = dma_fence_add_callback(fence, &afence->cb, async_op_fence_cb); - if (ret == -ENOENT) { - afence->fence.error = afence->wait_fence->error; - dma_fence_signal(&afence->fence); - } - if (ret) { - xe_vm_put(vm); - dma_fence_put(afence->wait_fence); - dma_fence_put(&afence->fence); - } - XE_WARN_ON(ret && ret != -ENOENT); -} - -int xe_vm_async_fence_wait_start(struct dma_fence *fence) -{ - if (fence->ops == &async_op_fence_ops) { - struct async_op_fence *afence = - container_of(fence, struct async_op_fence, fence); - - xe_assert(afence->vm->xe, !xe_vm_no_dma_fences(afence->vm)); - - smp_rmb(); - return wait_event_interruptible(afence->wq, afence->started); - } - - return 0; + return q ? !(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC) : + !(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT); } static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, - u32 num_syncs, struct async_op_fence *afence, - bool immediate, bool first_op, bool last_op) + u32 num_syncs, bool immediate, bool first_op, + bool last_op) { struct dma_fence *fence; @@ -1953,17 +1845,18 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, xe_sync_entry_signal(&syncs[i], NULL, fence); } } - if (afence) - add_async_op_fence_cb(vm, fence, afence); + if (last_op && xe_vm_sync_mode(vm, q)) + dma_fence_wait(fence, true); dma_fence_put(fence); + return 0; } static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q, struct xe_bo *bo, struct xe_sync_entry *syncs, - u32 num_syncs, struct async_op_fence *afence, - bool immediate, bool first_op, bool last_op) + u32 num_syncs, bool immediate, bool first_op, + bool last_op) { int err; @@ -1976,14 +1869,13 @@ static int xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue return err; } - return __xe_vm_bind(vm, vma, q, syncs, num_syncs, afence, immediate, - first_op, last_op); + return __xe_vm_bind(vm, vma, q, syncs, num_syncs, immediate, first_op, + last_op); } static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, - u32 num_syncs, struct async_op_fence *afence, - bool first_op, bool last_op) + u32 num_syncs, bool first_op, bool last_op) { struct dma_fence *fence; @@ -1993,10 +1885,10 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, fence = xe_vm_unbind_vma(vma, q, syncs, num_syncs, first_op, last_op); if (IS_ERR(fence)) return PTR_ERR(fence); - if (afence) - add_async_op_fence_cb(vm, fence, afence); xe_vma_destroy(vma, fence); + if (last_op && xe_vm_sync_mode(vm, q)) + dma_fence_wait(fence, true); dma_fence_put(fence); return 0; @@ -2004,7 +1896,7 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_SCRATCH_PAGE | \ DRM_XE_VM_CREATE_COMPUTE_MODE | \ - DRM_XE_VM_CREATE_ASYNC_BIND_OPS | \ + DRM_XE_VM_CREATE_ASYNC_DEFAULT | \ DRM_XE_VM_CREATE_FAULT_MODE) int xe_vm_create_ioctl(struct drm_device *dev, void *data, @@ -2051,12 +1943,15 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, xe_device_in_fault_mode(xe))) return -EINVAL; + if (XE_IOCTL_DBG(xe, args->extensions)) + return -EINVAL; + if (args->flags & DRM_XE_VM_CREATE_SCRATCH_PAGE) flags |= XE_VM_FLAG_SCRATCH_PAGE; if (args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE) flags |= XE_VM_FLAG_COMPUTE_MODE; - if (args->flags & DRM_XE_VM_CREATE_ASYNC_BIND_OPS) - flags |= XE_VM_FLAG_ASYNC_BIND_OPS; + if (args->flags & DRM_XE_VM_CREATE_ASYNC_DEFAULT) + flags |= XE_VM_FLAG_ASYNC_DEFAULT; if (args->flags & DRM_XE_VM_CREATE_FAULT_MODE) flags |= XE_VM_FLAG_FAULT_MODE; @@ -2139,8 +2034,7 @@ static const u32 region_to_mem_type[] = { static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q, u32 region, struct xe_sync_entry *syncs, u32 num_syncs, - struct async_op_fence *afence, bool first_op, - bool last_op) + bool first_op, bool last_op) { int err; @@ -2154,7 +2048,7 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, if (vma->tile_mask != (vma->tile_present & ~vma->usm.tile_invalidated)) { return xe_vm_bind(vm, vma, q, xe_vma_bo(vma), syncs, num_syncs, - afence, true, first_op, last_op); + true, first_op, last_op); } else { int i; @@ -2164,55 +2058,9 @@ static int xe_vm_prefetch(struct xe_vm *vm, struct xe_vma *vma, xe_sync_entry_signal(&syncs[i], NULL, dma_fence_get_stub()); } - if (afence) - dma_fence_signal(&afence->fence); - return 0; - } -} - -static void vm_set_async_error(struct xe_vm *vm, int err) -{ - lockdep_assert_held(&vm->lock); - vm->async_ops.error = err; -} - -static int vm_bind_ioctl_lookup_vma(struct xe_vm *vm, struct xe_bo *bo, - u64 addr, u64 range, u32 op, u32 flags) -{ - struct xe_device *xe = vm->xe; - struct xe_vma *vma; - bool async = !!(flags & XE_VM_BIND_FLAG_ASYNC); - - lockdep_assert_held(&vm->lock); - switch (op) { - case XE_VM_BIND_OP_MAP: - case XE_VM_BIND_OP_MAP_USERPTR: - vma = xe_vm_find_overlapping_vma(vm, addr, range); - if (XE_IOCTL_DBG(xe, vma && !async)) - return -EBUSY; - break; - case XE_VM_BIND_OP_UNMAP: - case XE_VM_BIND_OP_PREFETCH: - vma = xe_vm_find_overlapping_vma(vm, addr, range); - if (XE_IOCTL_DBG(xe, !vma)) - /* Not an actual error, IOCTL cleans up returns and 0 */ - return -ENODATA; - if (XE_IOCTL_DBG(xe, (xe_vma_start(vma) != addr || - xe_vma_end(vma) != addr + range) && !async)) - return -EINVAL; - break; - case XE_VM_BIND_OP_UNMAP_ALL: - if (XE_IOCTL_DBG(xe, list_empty(&bo->ttm.base.gpuva.list))) - /* Not an actual error, IOCTL cleans up returns and 0 */ - return -ENODATA; - break; - default: - drm_warn(&xe->drm, "NOT POSSIBLE"); - return -EINVAL; + return 0; } - - return 0; } static void prep_vma_destroy(struct xe_vm *vm, struct xe_vma *vma, @@ -2509,37 +2357,15 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, bool async) { struct xe_vma_op *last_op = NULL; - struct async_op_fence *fence = NULL; struct drm_gpuva_op *__op; int err = 0; lockdep_assert_held_write(&vm->lock); - if (last && num_syncs && async) { - u64 seqno; - - fence = kmalloc(sizeof(*fence), GFP_KERNEL); - if (!fence) - return -ENOMEM; - - seqno = q ? ++q->bind.fence_seqno : ++vm->async_ops.fence.seqno; - dma_fence_init(&fence->fence, &async_op_fence_ops, - &vm->async_ops.lock, q ? q->bind.fence_ctx : - vm->async_ops.fence.context, seqno); - - if (!xe_vm_no_dma_fences(vm)) { - fence->vm = vm; - fence->started = false; - init_waitqueue_head(&fence->wq); - } - } - drm_gpuva_for_each_op(__op, ops) { struct xe_vma_op *op = gpuva_op_to_vma_op(__op); bool first = list_empty(ops_list); - xe_assert(vm->xe, first || async); - INIT_LIST_HEAD(&op->link); list_add_tail(&op->link, ops_list); @@ -2559,10 +2385,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, vma = new_vma(vm, &op->base.map, op->tile_mask, op->map.read_only, op->map.is_null); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto free_fence; - } + if (IS_ERR(vma)) + return PTR_ERR(vma); op->map.vma = vma; break; @@ -2587,10 +2411,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, vma = new_vma(vm, op->base.remap.prev, op->tile_mask, read_only, is_null); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto free_fence; - } + if (IS_ERR(vma)) + return PTR_ERR(vma); op->remap.prev = vma; @@ -2623,10 +2445,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, vma = new_vma(vm, op->base.remap.next, op->tile_mask, read_only, is_null); - if (IS_ERR(vma)) { - err = PTR_ERR(vma); - goto free_fence; - } + if (IS_ERR(vma)) + return PTR_ERR(vma); op->remap.next = vma; @@ -2658,27 +2478,23 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, err = xe_vma_op_commit(vm, op); if (err) - goto free_fence; + return err; } /* FIXME: Unhandled corner case */ XE_WARN_ON(!last_op && last && !list_empty(ops_list)); if (!last_op) - goto free_fence; + return 0; + last_op->ops = ops; if (last) { last_op->flags |= XE_VMA_OP_LAST; last_op->num_syncs = num_syncs; last_op->syncs = syncs; - last_op->fence = fence; } return 0; - -free_fence: - kfree(fence); - return err; } static int op_execute(struct drm_exec *exec, struct xe_vm *vm, @@ -2698,7 +2514,7 @@ static int op_execute(struct drm_exec *exec, struct xe_vm *vm, switch (op->base.op) { case DRM_GPUVA_OP_MAP: err = xe_vm_bind(vm, vma, op->q, xe_vma_bo(vma), - op->syncs, op->num_syncs, op->fence, + op->syncs, op->num_syncs, op->map.immediate || !xe_vm_in_fault_mode(vm), op->flags & XE_VMA_OP_FIRST, op->flags & XE_VMA_OP_LAST); @@ -2709,16 +2525,13 @@ static int op_execute(struct drm_exec *exec, struct xe_vm *vm, bool next = !!op->remap.next; if (!op->remap.unmap_done) { - if (prev || next) { - vm->async_ops.munmap_rebind_inflight = true; + if (prev || next) vma->gpuva.flags |= XE_VMA_FIRST_REBIND; - } err = xe_vm_unbind(vm, vma, op->q, op->syncs, op->num_syncs, - !prev && !next ? op->fence : NULL, op->flags & XE_VMA_OP_FIRST, - op->flags & XE_VMA_OP_LAST && !prev && - !next); + op->flags & XE_VMA_OP_LAST && + !prev && !next); if (err) break; op->remap.unmap_done = true; @@ -2728,8 +2541,7 @@ static int op_execute(struct drm_exec *exec, struct xe_vm *vm, op->remap.prev->gpuva.flags |= XE_VMA_LAST_REBIND; err = xe_vm_bind(vm, op->remap.prev, op->q, xe_vma_bo(op->remap.prev), op->syncs, - op->num_syncs, - !next ? op->fence : NULL, true, false, + op->num_syncs, true, false, op->flags & XE_VMA_OP_LAST && !next); op->remap.prev->gpuva.flags &= ~XE_VMA_LAST_REBIND; if (err) @@ -2742,26 +2554,24 @@ static int op_execute(struct drm_exec *exec, struct xe_vm *vm, err = xe_vm_bind(vm, op->remap.next, op->q, xe_vma_bo(op->remap.next), op->syncs, op->num_syncs, - op->fence, true, false, + true, false, op->flags & XE_VMA_OP_LAST); op->remap.next->gpuva.flags &= ~XE_VMA_LAST_REBIND; if (err) break; op->remap.next = NULL; } - vm->async_ops.munmap_rebind_inflight = false; break; } case DRM_GPUVA_OP_UNMAP: err = xe_vm_unbind(vm, vma, op->q, op->syncs, - op->num_syncs, op->fence, - op->flags & XE_VMA_OP_FIRST, + op->num_syncs, op->flags & XE_VMA_OP_FIRST, op->flags & XE_VMA_OP_LAST); break; case DRM_GPUVA_OP_PREFETCH: err = xe_vm_prefetch(vm, vma, op->q, op->prefetch.region, - op->syncs, op->num_syncs, op->fence, + op->syncs, op->num_syncs, op->flags & XE_VMA_OP_FIRST, op->flags & XE_VMA_OP_LAST); break; @@ -2860,14 +2670,9 @@ static void xe_vma_op_cleanup(struct xe_vm *vm, struct xe_vma_op *op) kfree(op->syncs); if (op->q) xe_exec_queue_put(op->q); - if (op->fence) - dma_fence_put(&op->fence->fence); } - if (!list_empty(&op->link)) { - spin_lock_irq(&vm->async_ops.lock); + if (!list_empty(&op->link)) list_del(&op->link); - spin_unlock_irq(&vm->async_ops.lock); - } if (op->ops) drm_gpuva_ops_free(&vm->gpuvm, op->ops); if (last) @@ -2929,129 +2734,6 @@ static void xe_vma_op_unwind(struct xe_vm *vm, struct xe_vma_op *op, } } -static struct xe_vma_op *next_vma_op(struct xe_vm *vm) -{ - return list_first_entry_or_null(&vm->async_ops.pending, - struct xe_vma_op, link); -} - -static void xe_vma_op_work_func(struct work_struct *w) -{ - struct xe_vm *vm = container_of(w, struct xe_vm, async_ops.work); - - for (;;) { - struct xe_vma_op *op; - int err; - - if (vm->async_ops.error && !xe_vm_is_closed(vm)) - break; - - spin_lock_irq(&vm->async_ops.lock); - op = next_vma_op(vm); - spin_unlock_irq(&vm->async_ops.lock); - - if (!op) - break; - - if (!xe_vm_is_closed(vm)) { - down_write(&vm->lock); - err = xe_vma_op_execute(vm, op); - if (err) { - drm_warn(&vm->xe->drm, - "Async VM op(%d) failed with %d", - op->base.op, err); - vm_set_async_error(vm, err); - up_write(&vm->lock); - - break; - } - up_write(&vm->lock); - } else { - struct xe_vma *vma; - - switch (op->base.op) { - case DRM_GPUVA_OP_REMAP: - vma = gpuva_to_vma(op->base.remap.unmap->va); - trace_xe_vma_flush(vma); - - down_write(&vm->lock); - xe_vma_destroy_unlocked(vma); - up_write(&vm->lock); - break; - case DRM_GPUVA_OP_UNMAP: - vma = gpuva_to_vma(op->base.unmap.va); - trace_xe_vma_flush(vma); - - down_write(&vm->lock); - xe_vma_destroy_unlocked(vma); - up_write(&vm->lock); - break; - default: - /* Nothing to do */ - break; - } - - if (op->fence && !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, - &op->fence->fence.flags)) { - if (!xe_vm_no_dma_fences(vm)) { - op->fence->started = true; - wake_up_all(&op->fence->wq); - } - dma_fence_signal(&op->fence->fence); - } - } - - xe_vma_op_cleanup(vm, op); - } -} - -static int vm_bind_ioctl_ops_execute(struct xe_vm *vm, - struct list_head *ops_list, bool async) -{ - struct xe_vma_op *op, *last_op, *next; - int err; - - lockdep_assert_held_write(&vm->lock); - - last_op = list_last_entry(ops_list, struct xe_vma_op, link); - - if (!async) { - err = xe_vma_op_execute(vm, last_op); - if (err) - goto unwind; - xe_vma_op_cleanup(vm, last_op); - } else { - int i; - bool installed = false; - - for (i = 0; i < last_op->num_syncs; i++) - installed |= xe_sync_entry_signal(&last_op->syncs[i], - NULL, - &last_op->fence->fence); - if (!installed && last_op->fence) - dma_fence_signal(&last_op->fence->fence); - - spin_lock_irq(&vm->async_ops.lock); - list_splice_tail(ops_list, &vm->async_ops.pending); - spin_unlock_irq(&vm->async_ops.lock); - - if (!vm->async_ops.error) - queue_work(system_unbound_wq, &vm->async_ops.work); - } - - return 0; - -unwind: - list_for_each_entry_reverse(op, ops_list, link) - xe_vma_op_unwind(vm, op, op->flags & XE_VMA_OP_COMMITTED, - op->flags & XE_VMA_OP_PREV_COMMITTED, - op->flags & XE_VMA_OP_NEXT_COMMITTED); - list_for_each_entry_safe(op, next, ops_list, link) - xe_vma_op_cleanup(vm, op); - - return err; -} - static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, struct drm_gpuva_ops **ops, int num_ops_list) @@ -3078,6 +2760,31 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, } } +static int vm_bind_ioctl_ops_execute(struct xe_vm *vm, + struct list_head *ops_list) +{ + struct xe_vma_op *op, *next; + int err; + + lockdep_assert_held_write(&vm->lock); + + list_for_each_entry_safe(op, next, ops_list, link) { + err = xe_vma_op_execute(vm, op); + if (err) { + drm_warn(&vm->xe->drm, "VM op(%d) failed with %d", + op->base.op, err); + /* + * FIXME: Killing VM rather than proper error handling + */ + xe_vm_kill(vm); + return -ENOSPC; + } + xe_vma_op_cleanup(vm, op); + } + + return 0; +} + #ifdef TEST_VM_ASYNC_OPS_ERROR #define SUPPORTED_FLAGS \ (FORCE_ASYNC_OP_ERROR | XE_VM_BIND_FLAG_ASYNC | \ @@ -3086,7 +2793,8 @@ static void vm_bind_ioctl_ops_unwind(struct xe_vm *vm, #else #define SUPPORTED_FLAGS \ (XE_VM_BIND_FLAG_ASYNC | XE_VM_BIND_FLAG_READONLY | \ - XE_VM_BIND_FLAG_IMMEDIATE | XE_VM_BIND_FLAG_NULL | 0xffff) + XE_VM_BIND_FLAG_IMMEDIATE | XE_VM_BIND_FLAG_NULL | \ + 0xffff) #endif #define XE_64K_PAGE_MASK 0xffffull @@ -3137,21 +2845,12 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, if (i == 0) { *async = !!(flags & XE_VM_BIND_FLAG_ASYNC); - } else if (XE_IOCTL_DBG(xe, !*async) || - XE_IOCTL_DBG(xe, !(flags & XE_VM_BIND_FLAG_ASYNC)) || - XE_IOCTL_DBG(xe, op == XE_VM_BIND_OP_RESTART)) { - err = -EINVAL; - goto free_bind_ops; - } - - if (XE_IOCTL_DBG(xe, !*async && - op == XE_VM_BIND_OP_UNMAP_ALL)) { - err = -EINVAL; - goto free_bind_ops; - } - - if (XE_IOCTL_DBG(xe, !*async && - op == XE_VM_BIND_OP_PREFETCH)) { + if (XE_IOCTL_DBG(xe, !*async && args->num_syncs)) { + err = -EINVAL; + goto free_bind_ops; + } + } else if (XE_IOCTL_DBG(xe, *async != + !!(flags & XE_VM_BIND_FLAG_ASYNC))) { err = -EINVAL; goto free_bind_ops; } @@ -3188,8 +2887,7 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, if (XE_IOCTL_DBG(xe, obj_offset & ~PAGE_MASK) || XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) || XE_IOCTL_DBG(xe, range & ~PAGE_MASK) || - XE_IOCTL_DBG(xe, !range && op != - XE_VM_BIND_OP_RESTART && + XE_IOCTL_DBG(xe, !range && op != XE_VM_BIND_OP_UNMAP_ALL)) { err = -EINVAL; goto free_bind_ops; @@ -3237,6 +2935,12 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) err = -EINVAL; goto put_exec_queue; } + + if (XE_IOCTL_DBG(xe, async != + !!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) { + err = -EINVAL; + goto put_exec_queue; + } } vm = xe_vm_lookup(xef, args->vm_id); @@ -3245,6 +2949,14 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto put_exec_queue; } + if (!args->exec_queue_id) { + if (XE_IOCTL_DBG(xe, async != + !!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) { + err = -EINVAL; + goto put_vm; + } + } + err = down_write_killable(&vm->lock); if (err) goto put_vm; @@ -3254,34 +2966,6 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto release_vm_lock; } - if (bind_ops[0].op == XE_VM_BIND_OP_RESTART) { - if (XE_IOCTL_DBG(xe, !(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) - err = -EOPNOTSUPP; - if (XE_IOCTL_DBG(xe, !err && args->num_syncs)) - err = EINVAL; - if (XE_IOCTL_DBG(xe, !err && !vm->async_ops.error)) - err = -EPROTO; - - if (!err) { - trace_xe_vm_restart(vm); - vm_set_async_error(vm, 0); - - queue_work(system_unbound_wq, &vm->async_ops.work); - - /* Rebinds may have been blocked, give worker a kick */ - if (xe_vm_in_compute_mode(vm)) - xe_vm_queue_rebind_worker(vm); - } - - goto release_vm_lock; - } - - if (XE_IOCTL_DBG(xe, !vm->async_ops.error && - async != !!(vm->flags & XE_VM_FLAG_ASYNC_BIND_OPS))) { - err = -EOPNOTSUPP; - goto release_vm_lock; - } - for (i = 0; i < args->num_binds; ++i) { u64 range = bind_ops[i].range; u64 addr = bind_ops[i].addr; @@ -3367,18 +3051,6 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto free_syncs; } - /* Do some error checking first to make the unwind easier */ - for (i = 0; i < args->num_binds; ++i) { - u64 range = bind_ops[i].range; - u64 addr = bind_ops[i].addr; - u32 op = bind_ops[i].op; - u32 flags = bind_ops[i].flags; - - err = vm_bind_ioctl_lookup_vma(vm, bos[i], addr, range, op, flags); - if (err) - goto free_syncs; - } - for (i = 0; i < args->num_binds; ++i) { u64 range = bind_ops[i].range; u64 addr = bind_ops[i].addr; @@ -3411,10 +3083,19 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto unwind_ops; } - err = vm_bind_ioctl_ops_execute(vm, &ops_list, async); + xe_vm_get(vm); + if (q) + xe_exec_queue_get(q); + + err = vm_bind_ioctl_ops_execute(vm, &ops_list); + up_write(&vm->lock); - for (i = 0; i < args->num_binds; ++i) + if (q) + xe_exec_queue_put(q); + xe_vm_put(vm); + + for (i = 0; bos && i < args->num_binds; ++i) xe_bo_put(bos[i]); kfree(bos); diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index 59dcbd1adf15..45b70ba86553 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -177,8 +177,6 @@ struct dma_fence *xe_vm_rebind(struct xe_vm *vm, bool rebind_worker); int xe_vm_invalidate_vma(struct xe_vma *vma); -int xe_vm_async_fence_wait_start(struct dma_fence *fence); - extern struct ttm_device_funcs xe_ttm_funcs; static inline void xe_vm_queue_rebind_worker(struct xe_vm *vm) diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 828ed0fa7e60..97d779d8a7d3 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -17,7 +17,6 @@ #include "xe_pt_types.h" #include "xe_range_fence.h" -struct async_op_fence; struct xe_bo; struct xe_sync_entry; struct xe_vm; @@ -156,7 +155,7 @@ struct xe_vm { */ #define XE_VM_FLAG_64K BIT(0) #define XE_VM_FLAG_COMPUTE_MODE BIT(1) -#define XE_VM_FLAG_ASYNC_BIND_OPS BIT(2) +#define XE_VM_FLAG_ASYNC_DEFAULT BIT(2) #define XE_VM_FLAG_MIGRATION BIT(3) #define XE_VM_FLAG_SCRATCH_PAGE BIT(4) #define XE_VM_FLAG_FAULT_MODE BIT(5) @@ -394,10 +393,6 @@ struct xe_vma_op { u32 num_syncs; /** @link: async operation link */ struct list_head link; - /** - * @fence: async operation fence, signaled on last operation complete - */ - struct async_op_fence *fence; /** @tile_mask: gt mask for this operation */ u8 tile_mask; /** @flags: operation flags */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index f13974f17be9..4dc103aa00f1 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -134,10 +134,11 @@ struct drm_xe_engine_class_instance { #define DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE 3 #define DRM_XE_ENGINE_CLASS_COMPUTE 4 /* - * Kernel only class (not actual hardware engine class). Used for + * Kernel only classes (not actual hardware engine class). Used for * creating ordered queues of VM bind operations. */ -#define DRM_XE_ENGINE_CLASS_VM_BIND 5 +#define DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC 5 +#define DRM_XE_ENGINE_CLASS_VM_BIND_SYNC 6 __u16 engine_class; __u16 engine_instance; @@ -577,7 +578,7 @@ struct drm_xe_vm_create { #define DRM_XE_VM_CREATE_SCRATCH_PAGE (0x1 << 0) #define DRM_XE_VM_CREATE_COMPUTE_MODE (0x1 << 1) -#define DRM_XE_VM_CREATE_ASYNC_BIND_OPS (0x1 << 2) +#define DRM_XE_VM_CREATE_ASYNC_DEFAULT (0x1 << 2) #define DRM_XE_VM_CREATE_FAULT_MODE (0x1 << 3) /** @flags: Flags */ __u32 flags; @@ -637,34 +638,12 @@ struct drm_xe_vm_bind_op { #define XE_VM_BIND_OP_MAP 0x0 #define XE_VM_BIND_OP_UNMAP 0x1 #define XE_VM_BIND_OP_MAP_USERPTR 0x2 -#define XE_VM_BIND_OP_RESTART 0x3 -#define XE_VM_BIND_OP_UNMAP_ALL 0x4 -#define XE_VM_BIND_OP_PREFETCH 0x5 +#define XE_VM_BIND_OP_UNMAP_ALL 0x3 +#define XE_VM_BIND_OP_PREFETCH 0x4 /** @op: Bind operation to perform */ __u32 op; #define XE_VM_BIND_FLAG_READONLY (0x1 << 0) - /* - * A bind ops completions are always async, hence the support for out - * sync. This flag indicates the allocation of the memory for new page - * tables and the job to program the pages tables is asynchronous - * relative to the IOCTL. That part of a bind operation can fail under - * memory pressure, the job in practice can't fail unless the system is - * totally shot. - * - * If this flag is clear and the IOCTL doesn't return an error, in - * practice the bind op is good and will complete. - * - * If this flag is set and doesn't return an error, the bind op can - * still fail and recovery is needed. It should free memory - * via non-async unbinds, and then restart all queued async binds op via - * XE_VM_BIND_OP_RESTART. Or alternatively the user should destroy the - * VM. - * - * This flag is only allowed when DRM_XE_VM_CREATE_ASYNC_BIND_OPS is - * configured in the VM and must be set if the VM is configured with - * DRM_XE_VM_CREATE_ASYNC_BIND_OPS and not in an error state. - */ #define XE_VM_BIND_FLAG_ASYNC (0x1 << 1) /* * Valid on a faulting VM only, do the MAP operation immediately rather -- cgit v1.2.3 From 25f656f534f4b4eb95140efce37328efbda13af7 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 20 Sep 2023 15:29:33 -0400 Subject: drm/xe/uapi: Document drm_xe_query_gt Split drm_xe_query_gt out of the gt list one in order to better document it. No functional change at this point. Any actual change to the uapi should come in follow-up additions. v2: s/maks/mask Cc: Matt Roper Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi Reviewed-by: Matthew Brost --- include/uapi/drm/xe_drm.h | 65 +++++++++++++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 22 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 4dc103aa00f1..53b7b2ddf304 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -334,6 +334,47 @@ struct drm_xe_query_config { __u64 info[]; }; +/** + * struct drm_xe_query_gt - describe an individual GT. + * + * To be used with drm_xe_query_gts, which will return a list with all the + * existing GT individual descriptions. + * Graphics Technology (GT) is a subset of a GPU/tile that is responsible for + * implementing graphics and/or media operations. + */ +struct drm_xe_query_gt { +#define XE_QUERY_GT_TYPE_MAIN 0 +#define XE_QUERY_GT_TYPE_REMOTE 1 +#define XE_QUERY_GT_TYPE_MEDIA 2 + /** @type: GT type: Main, Remote, or Media */ + __u16 type; + /** @instance: Instance of this GT in the GT list */ + __u16 instance; + /** @clock_freq: A clock frequency for timestamp */ + __u32 clock_freq; + /** @features: Reserved for future information about GT features */ + __u64 features; + /** + * @native_mem_regions: Bit mask of instances from + * drm_xe_query_mem_usage that lives on the same GPU/Tile and have + * direct access. + */ + __u64 native_mem_regions; + /** + * @slow_mem_regions: Bit mask of instances from + * drm_xe_query_mem_usage that this GT can indirectly access, although + * they live on a different GPU/Tile. + */ + __u64 slow_mem_regions; + /** + * @inaccessible_mem_regions: Bit mask of instances from + * drm_xe_query_mem_usage that is not accessible by this GT at all. + */ + __u64 inaccessible_mem_regions; + /** @reserved: Reserved */ + __u64 reserved[8]; +}; + /** * struct drm_xe_query_gts - describe GTs * @@ -344,30 +385,10 @@ struct drm_xe_query_config { struct drm_xe_query_gts { /** @num_gt: number of GTs returned in gts */ __u32 num_gt; - /** @pad: MBZ */ __u32 pad; - - /** - * @gts: The GTs returned for this device - * - * TODO: convert drm_xe_query_gt to proper kernel-doc. - * TODO: Perhaps info about every mem region relative to this GT? e.g. - * bandwidth between this GT and remote region? - */ - struct drm_xe_query_gt { -#define XE_QUERY_GT_TYPE_MAIN 0 -#define XE_QUERY_GT_TYPE_REMOTE 1 -#define XE_QUERY_GT_TYPE_MEDIA 2 - __u16 type; - __u16 instance; - __u32 clock_freq; - __u64 features; - __u64 native_mem_regions; /* bit mask of instances from drm_xe_query_mem_usage */ - __u64 slow_mem_regions; /* bit mask of instances from drm_xe_query_mem_usage */ - __u64 inaccessible_mem_regions; /* bit mask of instances from drm_xe_query_mem_usage */ - __u64 reserved[8]; - } gts[]; + /** @gts: The GT list returned for this device */ + struct drm_xe_query_gt gts[]; }; /** -- cgit v1.2.3 From 2519450aaa31948d27db0715c24398b2590517f1 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 20 Sep 2023 15:29:34 -0400 Subject: drm/xe/uapi: Replace useless 'instance' per unique gt_id Let's have a single GT ID per GT within the PCI Device Card. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matthew Brost --- drivers/gpu/drm/xe/xe_gt_types.h | 2 +- drivers/gpu/drm/xe/xe_pci.c | 4 ---- drivers/gpu/drm/xe/xe_query.c | 2 +- include/uapi/drm/xe_drm.h | 4 ++-- 4 files changed, 4 insertions(+), 8 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index d4310be3e1e7..d3f2793684e2 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -105,7 +105,7 @@ struct xe_gt { struct { /** @type: type of GT */ enum xe_gt_type type; - /** @id: id of GT */ + /** @id: Unique ID of this GT within the PCI Device */ u8 id; /** @clock_freq: clock frequency */ u32 clock_freq; diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 9963772caabb..eec2b852c7aa 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -593,10 +593,6 @@ static int xe_info_init(struct xe_device *xe, return PTR_ERR(tile->primary_gt); gt = tile->primary_gt; - /* - * FIXME: GT numbering scheme may change depending on UAPI - * decisions. - */ gt->info.id = xe->info.gt_count++; gt->info.type = XE_GT_TYPE_MAIN; gt->info.__engine_mask = graphics_desc->hw_engine_mask; diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index cd3e0f3208a6..3bff06299e65 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -376,7 +376,7 @@ static int query_gts(struct xe_device *xe, struct drm_xe_device_query *query) gts->gts[id].type = XE_QUERY_GT_TYPE_REMOTE; else gts->gts[id].type = XE_QUERY_GT_TYPE_MAIN; - gts->gts[id].instance = id; + gts->gts[id].gt_id = gt->info.id; gts->gts[id].clock_freq = gt->info.clock_freq; if (!IS_DGFX(xe)) gts->gts[id].native_mem_regions = 0x1; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 53b7b2ddf304..11bc4dc2c78c 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -348,8 +348,8 @@ struct drm_xe_query_gt { #define XE_QUERY_GT_TYPE_MEDIA 2 /** @type: GT type: Main, Remote, or Media */ __u16 type; - /** @instance: Instance of this GT in the GT list */ - __u16 instance; + /** @gt_id: Unique ID of this GT within the PCI Device */ + __u16 gt_id; /** @clock_freq: A clock frequency for timestamp */ __u32 clock_freq; /** @features: Reserved for future information about GT features */ -- cgit v1.2.3 From 92296571546460bf9f4faf5e288d63f91d838968 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 20 Sep 2023 15:29:35 -0400 Subject: drm/xe/uapi: Remove unused field of drm_xe_query_gt We already have many bits reserved at the end already. Let's kill the unused ones. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matthew Brost --- include/uapi/drm/xe_drm.h | 2 -- 1 file changed, 2 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 11bc4dc2c78c..538873361d17 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -352,8 +352,6 @@ struct drm_xe_query_gt { __u16 gt_id; /** @clock_freq: A clock frequency for timestamp */ __u32 clock_freq; - /** @features: Reserved for future information about GT features */ - __u64 features; /** * @native_mem_regions: Bit mask of instances from * drm_xe_query_mem_usage that lives on the same GPU/Tile and have -- cgit v1.2.3 From e16b48378527dbe2f200b792922f59a2bf038507 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 20 Sep 2023 15:29:36 -0400 Subject: drm/xe/uapi: Rename gts to gt_list During the uapi review it was identified a possible confusion with the plural of acronym with a new acronym. So the recommendation is to go with gt_list instead. Suggested-by: Matt Roper Signed-off-by: Rodrigo Vivi Reviewed-by: Matt Roper Signed-off-by: Francois Dugast --- drivers/gpu/drm/xe/xe_query.c | 40 ++++++++++++++++++++-------------------- include/uapi/drm/xe_drm.h | 18 +++++++++--------- 2 files changed, 29 insertions(+), 29 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 3bff06299e65..d37c75a0b028 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -347,14 +347,14 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) return 0; } -static int query_gts(struct xe_device *xe, struct drm_xe_device_query *query) +static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query) { struct xe_gt *gt; - size_t size = sizeof(struct drm_xe_query_gts) + + size_t size = sizeof(struct drm_xe_query_gt_list) + xe->info.gt_count * sizeof(struct drm_xe_query_gt); - struct drm_xe_query_gts __user *query_ptr = + struct drm_xe_query_gt_list __user *query_ptr = u64_to_user_ptr(query->data); - struct drm_xe_query_gts *gts; + struct drm_xe_query_gt_list *gt_list; u8 id; if (query->size == 0) { @@ -364,34 +364,34 @@ static int query_gts(struct xe_device *xe, struct drm_xe_device_query *query) return -EINVAL; } - gts = kzalloc(size, GFP_KERNEL); - if (!gts) + gt_list = kzalloc(size, GFP_KERNEL); + if (!gt_list) return -ENOMEM; - gts->num_gt = xe->info.gt_count; + gt_list->num_gt = xe->info.gt_count; for_each_gt(gt, xe, id) { if (xe_gt_is_media_type(gt)) - gts->gts[id].type = XE_QUERY_GT_TYPE_MEDIA; + gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_MEDIA; else if (gt_to_tile(gt)->id > 0) - gts->gts[id].type = XE_QUERY_GT_TYPE_REMOTE; + gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_REMOTE; else - gts->gts[id].type = XE_QUERY_GT_TYPE_MAIN; - gts->gts[id].gt_id = gt->info.id; - gts->gts[id].clock_freq = gt->info.clock_freq; + gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_MAIN; + gt_list->gt_list[id].gt_id = gt->info.id; + gt_list->gt_list[id].clock_freq = gt->info.clock_freq; if (!IS_DGFX(xe)) - gts->gts[id].native_mem_regions = 0x1; + gt_list->gt_list[id].native_mem_regions = 0x1; else - gts->gts[id].native_mem_regions = + gt_list->gt_list[id].native_mem_regions = BIT(gt_to_tile(gt)->id) << 1; - gts->gts[id].slow_mem_regions = xe->info.mem_region_mask ^ - gts->gts[id].native_mem_regions; + gt_list->gt_list[id].slow_mem_regions = xe->info.mem_region_mask ^ + gt_list->gt_list[id].native_mem_regions; } - if (copy_to_user(query_ptr, gts, size)) { - kfree(gts); + if (copy_to_user(query_ptr, gt_list, size)) { + kfree(gt_list); return -EFAULT; } - kfree(gts); + kfree(gt_list); return 0; } @@ -503,7 +503,7 @@ static int (* const xe_query_funcs[])(struct xe_device *xe, query_engines, query_memory_usage, query_config, - query_gts, + query_gt_list, query_hwconfig, query_gt_topology, query_engine_cycles, diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 538873361d17..b02a63270972 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -337,7 +337,7 @@ struct drm_xe_query_config { /** * struct drm_xe_query_gt - describe an individual GT. * - * To be used with drm_xe_query_gts, which will return a list with all the + * To be used with drm_xe_query_gt_list, which will return a list with all the * existing GT individual descriptions. * Graphics Technology (GT) is a subset of a GPU/tile that is responsible for * implementing graphics and/or media operations. @@ -374,19 +374,19 @@ struct drm_xe_query_gt { }; /** - * struct drm_xe_query_gts - describe GTs + * struct drm_xe_query_gt_list - A list with GT description items. * * If a query is made with a struct drm_xe_device_query where .query - * is equal to DRM_XE_DEVICE_QUERY_GTS, then the reply uses struct - * drm_xe_query_gts in .data. + * is equal to DRM_XE_DEVICE_QUERY_GT_LIST, then the reply uses struct + * drm_xe_query_gt_list in .data. */ -struct drm_xe_query_gts { - /** @num_gt: number of GTs returned in gts */ +struct drm_xe_query_gt_list { + /** @num_gt: number of GT items returned in gt_list */ __u32 num_gt; /** @pad: MBZ */ __u32 pad; - /** @gts: The GT list returned for this device */ - struct drm_xe_query_gt gts[]; + /** @gt_list: The GT list returned for this device */ + struct drm_xe_query_gt gt_list[]; }; /** @@ -479,7 +479,7 @@ struct drm_xe_device_query { #define DRM_XE_DEVICE_QUERY_ENGINES 0 #define DRM_XE_DEVICE_QUERY_MEM_USAGE 1 #define DRM_XE_DEVICE_QUERY_CONFIG 2 -#define DRM_XE_DEVICE_QUERY_GTS 3 +#define DRM_XE_DEVICE_QUERY_GT_LIST 3 #define DRM_XE_DEVICE_QUERY_HWCONFIG 4 #define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5 #define DRM_XE_DEVICE_QUERY_ENGINE_CYCLES 6 -- cgit v1.2.3 From e48d146456e34625c6edafd6350bfaac5004727c Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 20 Sep 2023 15:29:37 -0400 Subject: drm/xe/uapi: Fix naming of XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY This is used for the priority of an exec queue (not an engine) and should be named accordingly. Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi Reviewed-by: Matthew Brost --- drivers/gpu/drm/xe/xe_query.c | 2 +- include/uapi/drm/xe_drm.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index d37c75a0b028..10b9878ec95a 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -335,7 +335,7 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) config->info[XE_QUERY_CONFIG_GT_COUNT] = xe->info.gt_count; config->info[XE_QUERY_CONFIG_MEM_REGION_COUNT] = hweight_long(xe->info.mem_region_mask); - config->info[XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY] = + config->info[XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = xe_exec_queue_device_get_max_priority(xe); if (copy_to_user(query_ptr, config, size)) { diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index b02a63270972..24bf8f0f52e8 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -328,8 +328,8 @@ struct drm_xe_query_config { #define XE_QUERY_CONFIG_VA_BITS 3 #define XE_QUERY_CONFIG_GT_COUNT 4 #define XE_QUERY_CONFIG_MEM_REGION_COUNT 5 -#define XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY 6 -#define XE_QUERY_CONFIG_NUM_PARAM (XE_QUERY_CONFIG_MAX_ENGINE_PRIORITY + 1) +#define XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 6 +#define XE_QUERY_CONFIG_NUM_PARAM (XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1) /** @info: array of elements containing the config info */ __u64 info[]; }; -- cgit v1.2.3 From b8d70702def26d7597eded092fe43cc584c0d064 Mon Sep 17 00:00:00 2001 From: Priyanka Dandamudi Date: Fri, 27 Oct 2023 10:55:07 +0530 Subject: drm/xe/xe_exec_queue: Add check for access counter granularity Add conditional check for access counter granularity. This check will return -EINVAL if granularity is beyond 64M which is a hardware limitation. v2: Defined XE_ACC_GRANULARITY_128K 0 XE_ACC_GRANULARITY_2M 1 XE_ACC_GRANULARITY_16M 2 XE_ACC_GRANULARITY_64M 3 as part of uAPI. So, that user can also use it.(Oak) v3: Move uAPI to proper location and give proper documentation.(Brian, Oak) Cc: Oak Zeng Cc: Janga Rahul Kumar Cc: Brian Welty Signed-off-by: Priyanka Dandamudi Reviewed-by: Oak Zeng Reviewed-by: Oak Zeng Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_exec_queue.c | 3 +++ include/uapi/drm/xe_drm.h | 14 ++++++++++++++ 2 files changed, 17 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 8e0620cb89e5..f67a6dee4a6f 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -393,6 +393,9 @@ static int exec_queue_set_acc_granularity(struct xe_device *xe, struct xe_exec_q if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) return -EINVAL; + if (value > XE_ACC_GRANULARITY_64M) + return -EINVAL; + q->usm.acc_granularity = value; return 0; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 24bf8f0f52e8..9bd7092a7ea4 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -731,6 +731,20 @@ struct drm_xe_vm_bind { __u64 reserved[2]; }; +/* For use with XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY */ + +/* Monitor 128KB contiguous region with 4K sub-granularity */ +#define XE_ACC_GRANULARITY_128K 0 + +/* Monitor 2MB contiguous region with 64KB sub-granularity */ +#define XE_ACC_GRANULARITY_2M 1 + +/* Monitor 16MB contiguous region with 512KB sub-granularity */ +#define XE_ACC_GRANULARITY_16M 2 + +/* Monitor 64MB contiguous region with 2M sub-granularity */ +#define XE_ACC_GRANULARITY_64M 3 + /** * struct drm_xe_exec_queue_set_property - exec queue set property * -- cgit v1.2.3 From de84aa96e4427125d00af1706b59584b2cbb0085 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 10 Nov 2023 15:41:50 +0000 Subject: drm/xe/uapi: Remove useless XE_QUERY_CONFIG_NUM_PARAM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit num_params can be used to retrieve the size of the info array for the specific version of the kernel being used. v2: Also remove XE_QUERY_CONFIG_NUM_PARAM (José Roberto de Souza) Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_query.c | 2 +- include/uapi/drm/xe_drm.h | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 10b9878ec95a..58fb06a63db2 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -305,7 +305,7 @@ static int query_memory_usage(struct xe_device *xe, static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) { - u32 num_params = XE_QUERY_CONFIG_NUM_PARAM; + const u32 num_params = XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1; size_t size = sizeof(struct drm_xe_query_config) + num_params * sizeof(u64); struct drm_xe_query_config __user *query_ptr = diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 9bd7092a7ea4..b9a68f8b69f3 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -329,7 +329,6 @@ struct drm_xe_query_config { #define XE_QUERY_CONFIG_GT_COUNT 4 #define XE_QUERY_CONFIG_MEM_REGION_COUNT 5 #define XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 6 -#define XE_QUERY_CONFIG_NUM_PARAM (XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1) /** @info: array of elements containing the config info */ __u64 info[]; }; -- cgit v1.2.3 From 1a912c90a278177423128e5b82673575821d0c35 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Fri, 10 Nov 2023 15:41:51 +0000 Subject: drm/xe/uapi: Remove GT_TYPE_REMOTE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the split between tile and gt, this is currently unused. Also it is bringing confusion because main vs remote would be more a concept of the tile itself and not about GT. So, the MAIN one is the traditional GT used for every operation in older platforms, and for render/graphics and compute on platforms that contains the stand-alone Media GT. Cc: Matt Roper Cc: Francois Dugast Cc: Carl Zhang Cc: José Roberto de Souza Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_query.c | 2 -- include/uapi/drm/xe_drm.h | 5 ++--- 2 files changed, 2 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 58fb06a63db2..cb3461971dc9 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -372,8 +372,6 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query for_each_gt(gt, xe, id) { if (xe_gt_is_media_type(gt)) gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_MEDIA; - else if (gt_to_tile(gt)->id > 0) - gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_REMOTE; else gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_MAIN; gt_list->gt_list[id].gt_id = gt->info.id; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index b9a68f8b69f3..8154cecf6f0d 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -343,9 +343,8 @@ struct drm_xe_query_config { */ struct drm_xe_query_gt { #define XE_QUERY_GT_TYPE_MAIN 0 -#define XE_QUERY_GT_TYPE_REMOTE 1 -#define XE_QUERY_GT_TYPE_MEDIA 2 - /** @type: GT type: Main, Remote, or Media */ +#define XE_QUERY_GT_TYPE_MEDIA 1 + /** @type: GT type: Main or Media */ __u16 type; /** @gt_id: Unique ID of this GT within the PCI Device */ __u16 gt_id; -- cgit v1.2.3 From ddfa2d6a846a571edb4dc6ed29d94b38558ae088 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Fri, 10 Nov 2023 15:41:52 +0000 Subject: drm/xe/uapi: Kill VM_MADVISE IOCTL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove unused IOCTL. Without any userspace using it we need to remove before we can be accepted upstream. At this point we are breaking the compatibility for good, so we don't need to break when we are in-tree. So, let's also use this breakage to sort out the IOCTL entries and fix all the small indentation and line issues. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/Makefile | 1 - drivers/gpu/drm/xe/xe_bo.c | 2 +- drivers/gpu/drm/xe/xe_bo_types.h | 3 + drivers/gpu/drm/xe/xe_device.c | 8 +- drivers/gpu/drm/xe/xe_vm_madvise.c | 299 ------------------------------------- drivers/gpu/drm/xe/xe_vm_madvise.h | 15 -- include/uapi/drm/xe_drm.h | 92 ++---------- 7 files changed, 18 insertions(+), 402 deletions(-) delete mode 100644 drivers/gpu/drm/xe/xe_vm_madvise.c delete mode 100644 drivers/gpu/drm/xe/xe_vm_madvise.h (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 41d92014a45c..a29b92080c85 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -115,7 +115,6 @@ xe-y += xe_bb.o \ xe_uc_debugfs.o \ xe_uc_fw.o \ xe_vm.o \ - xe_vm_madvise.o \ xe_wait_user_fence.o \ xe_wa.o \ xe_wopcm.o diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index c23a5694a788..5b5f764586fe 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1239,7 +1239,7 @@ struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, bo->props.preferred_mem_class = XE_BO_PROPS_INVALID; bo->props.preferred_gt = XE_BO_PROPS_INVALID; bo->props.preferred_mem_type = XE_BO_PROPS_INVALID; - bo->ttm.priority = DRM_XE_VMA_PRIORITY_NORMAL; + bo->ttm.priority = XE_BO_PRIORITY_NORMAL; INIT_LIST_HEAD(&bo->pinned_link); #ifdef CONFIG_PROC_FS INIT_LIST_HEAD(&bo->client_link); diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index 051fe990c133..4bff60996168 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -19,6 +19,9 @@ struct xe_vm; #define XE_BO_MAX_PLACEMENTS 3 +/* TODO: To be selected with VM_MADVISE */ +#define XE_BO_PRIORITY_NORMAL 1 + /** @xe_bo: XE buffer object */ struct xe_bo { /** @ttm: TTM base buffer object */ diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 1202f8007f79..8be765adf702 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -36,7 +36,6 @@ #include "xe_ttm_stolen_mgr.h" #include "xe_ttm_sys_mgr.h" #include "xe_vm.h" -#include "xe_vm_madvise.h" #include "xe_wait_user_fence.h" #include "xe_hwmon.h" @@ -117,18 +116,17 @@ static const struct drm_ioctl_desc xe_ioctls[] = { DRM_IOCTL_DEF_DRV(XE_VM_CREATE, xe_vm_create_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_VM_DESTROY, xe_vm_destroy_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_VM_BIND, xe_vm_bind_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_CREATE, xe_exec_queue_create_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, - DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(XE_EXEC, xe_exec_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, DRM_RENDER_ALLOW), + DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, + DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(XE_VM_MADVISE, xe_vm_madvise_ioctl, DRM_RENDER_ALLOW), }; static const struct file_operations xe_driver_fops = { diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.c b/drivers/gpu/drm/xe/xe_vm_madvise.c deleted file mode 100644 index 0ef7d483d050..000000000000 --- a/drivers/gpu/drm/xe/xe_vm_madvise.c +++ /dev/null @@ -1,299 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2021 Intel Corporation - */ - -#include "xe_vm_madvise.h" - -#include - -#include -#include - -#include "xe_bo.h" -#include "xe_vm.h" - -static int madvise_preferred_mem_class(struct xe_device *xe, struct xe_vm *vm, - struct xe_vma **vmas, int num_vmas, - u64 value) -{ - int i, err; - - if (XE_IOCTL_DBG(xe, value > XE_MEM_REGION_CLASS_VRAM)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, value == XE_MEM_REGION_CLASS_VRAM && - !xe->info.is_dgfx)) - return -EINVAL; - - for (i = 0; i < num_vmas; ++i) { - struct xe_bo *bo; - - bo = xe_vma_bo(vmas[i]); - - err = xe_bo_lock(bo, true); - if (err) - return err; - bo->props.preferred_mem_class = value; - xe_bo_placement_for_flags(xe, bo, bo->flags); - xe_bo_unlock(bo); - } - - return 0; -} - -static int madvise_preferred_gt(struct xe_device *xe, struct xe_vm *vm, - struct xe_vma **vmas, int num_vmas, u64 value) -{ - int i, err; - - if (XE_IOCTL_DBG(xe, value > xe->info.tile_count)) - return -EINVAL; - - for (i = 0; i < num_vmas; ++i) { - struct xe_bo *bo; - - bo = xe_vma_bo(vmas[i]); - - err = xe_bo_lock(bo, true); - if (err) - return err; - bo->props.preferred_gt = value; - xe_bo_placement_for_flags(xe, bo, bo->flags); - xe_bo_unlock(bo); - } - - return 0; -} - -static int madvise_preferred_mem_class_gt(struct xe_device *xe, - struct xe_vm *vm, - struct xe_vma **vmas, int num_vmas, - u64 value) -{ - int i, err; - u32 gt_id = upper_32_bits(value); - u32 mem_class = lower_32_bits(value); - - if (XE_IOCTL_DBG(xe, mem_class > XE_MEM_REGION_CLASS_VRAM)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, mem_class == XE_MEM_REGION_CLASS_VRAM && - !xe->info.is_dgfx)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, gt_id > xe->info.tile_count)) - return -EINVAL; - - for (i = 0; i < num_vmas; ++i) { - struct xe_bo *bo; - - bo = xe_vma_bo(vmas[i]); - - err = xe_bo_lock(bo, true); - if (err) - return err; - bo->props.preferred_mem_class = mem_class; - bo->props.preferred_gt = gt_id; - xe_bo_placement_for_flags(xe, bo, bo->flags); - xe_bo_unlock(bo); - } - - return 0; -} - -static int madvise_cpu_atomic(struct xe_device *xe, struct xe_vm *vm, - struct xe_vma **vmas, int num_vmas, u64 value) -{ - int i, err; - - for (i = 0; i < num_vmas; ++i) { - struct xe_bo *bo; - - bo = xe_vma_bo(vmas[i]); - if (XE_IOCTL_DBG(xe, !(bo->flags & XE_BO_CREATE_SYSTEM_BIT))) - return -EINVAL; - - err = xe_bo_lock(bo, true); - if (err) - return err; - bo->props.cpu_atomic = !!value; - - /* - * All future CPU accesses must be from system memory only, we - * just invalidate the CPU page tables which will trigger a - * migration on next access. - */ - if (bo->props.cpu_atomic) - ttm_bo_unmap_virtual(&bo->ttm); - xe_bo_unlock(bo); - } - - return 0; -} - -static int madvise_device_atomic(struct xe_device *xe, struct xe_vm *vm, - struct xe_vma **vmas, int num_vmas, u64 value) -{ - int i, err; - - for (i = 0; i < num_vmas; ++i) { - struct xe_bo *bo; - - bo = xe_vma_bo(vmas[i]); - if (XE_IOCTL_DBG(xe, !(bo->flags & XE_BO_CREATE_VRAM0_BIT) && - !(bo->flags & XE_BO_CREATE_VRAM1_BIT))) - return -EINVAL; - - err = xe_bo_lock(bo, true); - if (err) - return err; - bo->props.device_atomic = !!value; - xe_bo_unlock(bo); - } - - return 0; -} - -static int madvise_priority(struct xe_device *xe, struct xe_vm *vm, - struct xe_vma **vmas, int num_vmas, u64 value) -{ - int i, err; - - if (XE_IOCTL_DBG(xe, value > DRM_XE_VMA_PRIORITY_HIGH)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, value == DRM_XE_VMA_PRIORITY_HIGH && - !capable(CAP_SYS_NICE))) - return -EPERM; - - for (i = 0; i < num_vmas; ++i) { - struct xe_bo *bo; - - bo = xe_vma_bo(vmas[i]); - - err = xe_bo_lock(bo, true); - if (err) - return err; - bo->ttm.priority = value; - ttm_bo_move_to_lru_tail(&bo->ttm); - xe_bo_unlock(bo); - } - - return 0; -} - -static int madvise_pin(struct xe_device *xe, struct xe_vm *vm, - struct xe_vma **vmas, int num_vmas, u64 value) -{ - drm_warn(&xe->drm, "NIY"); - return 0; -} - -typedef int (*madvise_func)(struct xe_device *xe, struct xe_vm *vm, - struct xe_vma **vmas, int num_vmas, u64 value); - -static const madvise_func madvise_funcs[] = { - [DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS] = madvise_preferred_mem_class, - [DRM_XE_VM_MADVISE_PREFERRED_GT] = madvise_preferred_gt, - [DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS_GT] = - madvise_preferred_mem_class_gt, - [DRM_XE_VM_MADVISE_CPU_ATOMIC] = madvise_cpu_atomic, - [DRM_XE_VM_MADVISE_DEVICE_ATOMIC] = madvise_device_atomic, - [DRM_XE_VM_MADVISE_PRIORITY] = madvise_priority, - [DRM_XE_VM_MADVISE_PIN] = madvise_pin, -}; - -static struct xe_vma ** -get_vmas(struct xe_vm *vm, int *num_vmas, u64 addr, u64 range) -{ - struct xe_vma **vmas, **__vmas; - struct drm_gpuva *gpuva; - int max_vmas = 8; - - lockdep_assert_held(&vm->lock); - - vmas = kmalloc(max_vmas * sizeof(*vmas), GFP_KERNEL); - if (!vmas) - return NULL; - - drm_gpuvm_for_each_va_range(gpuva, &vm->gpuvm, addr, addr + range) { - struct xe_vma *vma = gpuva_to_vma(gpuva); - - if (xe_vma_is_userptr(vma)) - continue; - - if (*num_vmas == max_vmas) { - max_vmas <<= 1; - __vmas = krealloc(vmas, max_vmas * sizeof(*vmas), - GFP_KERNEL); - if (!__vmas) - return NULL; - vmas = __vmas; - } - - vmas[*num_vmas] = vma; - *num_vmas += 1; - } - - return vmas; -} - -int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct xe_device *xe = to_xe_device(dev); - struct xe_file *xef = to_xe_file(file); - struct drm_xe_vm_madvise *args = data; - struct xe_vm *vm; - struct xe_vma **vmas = NULL; - int num_vmas = 0, err = 0, idx; - - if (XE_IOCTL_DBG(xe, args->extensions) || - XE_IOCTL_DBG(xe, args->pad || args->pad2) || - XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, args->property > ARRAY_SIZE(madvise_funcs))) - return -EINVAL; - - vm = xe_vm_lookup(xef, args->vm_id); - if (XE_IOCTL_DBG(xe, !vm)) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, !xe_vm_in_fault_mode(vm))) { - err = -EINVAL; - goto put_vm; - } - - down_read(&vm->lock); - - if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { - err = -ENOENT; - goto unlock_vm; - } - - vmas = get_vmas(vm, &num_vmas, args->addr, args->range); - if (XE_IOCTL_DBG(xe, err)) - goto unlock_vm; - - if (XE_IOCTL_DBG(xe, !vmas)) { - err = -ENOMEM; - goto unlock_vm; - } - - if (XE_IOCTL_DBG(xe, !num_vmas)) { - err = -EINVAL; - goto unlock_vm; - } - - idx = array_index_nospec(args->property, ARRAY_SIZE(madvise_funcs)); - err = madvise_funcs[idx](xe, vm, vmas, num_vmas, args->value); - -unlock_vm: - up_read(&vm->lock); -put_vm: - xe_vm_put(vm); - kfree(vmas); - return err; -} diff --git a/drivers/gpu/drm/xe/xe_vm_madvise.h b/drivers/gpu/drm/xe/xe_vm_madvise.h deleted file mode 100644 index eecd33acd248..000000000000 --- a/drivers/gpu/drm/xe/xe_vm_madvise.h +++ /dev/null @@ -1,15 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2021 Intel Corporation - */ - -#ifndef _XE_VM_MADVISE_H_ -#define _XE_VM_MADVISE_H_ - -struct drm_device; -struct drm_file; - -int xe_vm_madvise_ioctl(struct drm_device *dev, void *data, - struct drm_file *file); - -#endif diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 8154cecf6f0d..808d92262bcd 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -103,28 +103,26 @@ struct xe_user_extension { #define DRM_XE_VM_CREATE 0x03 #define DRM_XE_VM_DESTROY 0x04 #define DRM_XE_VM_BIND 0x05 -#define DRM_XE_EXEC_QUEUE_CREATE 0x06 -#define DRM_XE_EXEC_QUEUE_DESTROY 0x07 -#define DRM_XE_EXEC 0x08 +#define DRM_XE_EXEC 0x06 +#define DRM_XE_EXEC_QUEUE_CREATE 0x07 +#define DRM_XE_EXEC_QUEUE_DESTROY 0x08 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x09 -#define DRM_XE_WAIT_USER_FENCE 0x0a -#define DRM_XE_VM_MADVISE 0x0b -#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x0c - +#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x0a +#define DRM_XE_WAIT_USER_FENCE 0x0b /* Must be kept compact -- no holes */ + #define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query) #define DRM_IOCTL_XE_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_CREATE, struct drm_xe_gem_create) #define DRM_IOCTL_XE_GEM_MMAP_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_MMAP_OFFSET, struct drm_xe_gem_mmap_offset) #define DRM_IOCTL_XE_VM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_CREATE, struct drm_xe_vm_create) -#define DRM_IOCTL_XE_VM_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) -#define DRM_IOCTL_XE_VM_BIND DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) +#define DRM_IOCTL_XE_VM_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) +#define DRM_IOCTL_XE_VM_BIND DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) +#define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) #define DRM_IOCTL_XE_EXEC_QUEUE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_CREATE, struct drm_xe_exec_queue_create) +#define DRM_IOCTL_XE_EXEC_QUEUE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_DESTROY, struct drm_xe_exec_queue_destroy) +#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property) #define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property) -#define DRM_IOCTL_XE_EXEC_QUEUE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_DESTROY, struct drm_xe_exec_queue_destroy) -#define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) -#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property) #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) -#define DRM_IOCTL_XE_VM_MADVISE DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_MADVISE, struct drm_xe_vm_madvise) /** struct drm_xe_engine_class_instance - instance of an engine class */ struct drm_xe_engine_class_instance { @@ -978,74 +976,6 @@ struct drm_xe_wait_user_fence { __u64 reserved[2]; }; -struct drm_xe_vm_madvise { - /** @extensions: Pointer to the first extension struct, if any */ - __u64 extensions; - - /** @vm_id: The ID VM in which the VMA exists */ - __u32 vm_id; - - /** @pad: MBZ */ - __u32 pad; - - /** @range: Number of bytes in the VMA */ - __u64 range; - - /** @addr: Address of the VMA to operation on */ - __u64 addr; - - /* - * Setting the preferred location will trigger a migrate of the VMA - * backing store to new location if the backing store is already - * allocated. - * - * For DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS usage, see enum - * drm_xe_memory_class. - */ -#define DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS 0 -#define DRM_XE_VM_MADVISE_PREFERRED_GT 1 - /* - * In this case lower 32 bits are mem class, upper 32 are GT. - * Combination provides a single IOCTL plus migrate VMA to preferred - * location. - */ -#define DRM_XE_VM_MADVISE_PREFERRED_MEM_CLASS_GT 2 - /* - * The CPU will do atomic memory operations to this VMA. Must be set on - * some devices for atomics to behave correctly. - */ -#define DRM_XE_VM_MADVISE_CPU_ATOMIC 3 - /* - * The device will do atomic memory operations to this VMA. Must be set - * on some devices for atomics to behave correctly. - */ -#define DRM_XE_VM_MADVISE_DEVICE_ATOMIC 4 - /* - * Priority WRT to eviction (moving from preferred memory location due - * to memory pressure). The lower the priority, the more likely to be - * evicted. - */ -#define DRM_XE_VM_MADVISE_PRIORITY 5 -#define DRM_XE_VMA_PRIORITY_LOW 0 - /* Default */ -#define DRM_XE_VMA_PRIORITY_NORMAL 1 - /* Must be user with elevated privileges */ -#define DRM_XE_VMA_PRIORITY_HIGH 2 - /* Pin the VMA in memory, must be user with elevated privileges */ -#define DRM_XE_VM_MADVISE_PIN 6 - /** @property: property to set */ - __u32 property; - - /** @pad2: MBZ */ - __u32 pad2; - - /** @value: property value */ - __u64 value; - - /** @reserved: Reserved */ - __u64 reserved[2]; -}; - /** * DOC: XE PMU event config IDs * -- cgit v1.2.3 From 34f0cf6dc4c79a915c7e1022f232f592bfa6c078 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 10 Nov 2023 15:41:53 +0000 Subject: drm/xe/uapi: Remove unused inaccessible memory region MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is not used and also the negative of the other 2 regions: native_mem_regions and slow_mem_regions. Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 808d92262bcd..0f8c5afd3584 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -360,11 +360,6 @@ struct drm_xe_query_gt { * they live on a different GPU/Tile. */ __u64 slow_mem_regions; - /** - * @inaccessible_mem_regions: Bit mask of instances from - * drm_xe_query_mem_usage that is not accessible by this GT at all. - */ - __u64 inaccessible_mem_regions; /** @reserved: Reserved */ __u64 reserved[8]; }; -- cgit v1.2.3 From 4195e5e5e3d544a90a1edac1e21cd53a5117bd1f Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 10 Nov 2023 15:41:54 +0000 Subject: drm/xe/uapi: Remove unused QUERY_CONFIG_MEM_REGION_COUNT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As part of uAPI cleanup, remove this constant which is not used. Memory regions can be queried with DRM_XE_DEVICE_QUERY_MEM_USAGE. Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_query.c | 2 -- include/uapi/drm/xe_drm.h | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index cb3461971dc9..b5cd980f81f9 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -333,8 +333,6 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; config->info[XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; config->info[XE_QUERY_CONFIG_GT_COUNT] = xe->info.gt_count; - config->info[XE_QUERY_CONFIG_MEM_REGION_COUNT] = - hweight_long(xe->info.mem_region_mask); config->info[XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = xe_exec_queue_device_get_max_priority(xe); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 0f8c5afd3584..1ac9ae0591de 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -311,6 +311,7 @@ struct drm_xe_query_mem_usage { * If a query is made with a struct drm_xe_device_query where .query * is equal to DRM_XE_DEVICE_QUERY_CONFIG, then the reply uses * struct drm_xe_query_config in .data. + * */ struct drm_xe_query_config { /** @num_params: number of parameters returned in info */ @@ -325,8 +326,7 @@ struct drm_xe_query_config { #define XE_QUERY_CONFIG_MIN_ALIGNMENT 2 #define XE_QUERY_CONFIG_VA_BITS 3 #define XE_QUERY_CONFIG_GT_COUNT 4 -#define XE_QUERY_CONFIG_MEM_REGION_COUNT 5 -#define XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 6 +#define XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 5 /** @info: array of elements containing the config info */ __u64 info[]; }; -- cgit v1.2.3 From 60f3c7fc5c2464f73a7d64a4cc2dd4707a0d1831 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 10 Nov 2023 15:41:55 +0000 Subject: drm/xe/uapi: Remove unused QUERY_CONFIG_GT_COUNT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As part of uAPI cleanup, remove this constant which is not used. Number of GTs are provided as num_gt in drm_xe_query_gt_list. Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_query.c | 1 - include/uapi/drm/xe_drm.h | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index b5cd980f81f9..e9c8c97a030f 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -332,7 +332,6 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) config->info[XE_QUERY_CONFIG_MIN_ALIGNMENT] = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; config->info[XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; - config->info[XE_QUERY_CONFIG_GT_COUNT] = xe->info.gt_count; config->info[XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = xe_exec_queue_device_get_max_priority(xe); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 1ac9ae0591de..097d045d0444 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -325,8 +325,7 @@ struct drm_xe_query_config { #define XE_QUERY_CONFIG_FLAGS_HAS_VRAM (0x1 << 0) #define XE_QUERY_CONFIG_MIN_ALIGNMENT 2 #define XE_QUERY_CONFIG_VA_BITS 3 -#define XE_QUERY_CONFIG_GT_COUNT 4 -#define XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 5 +#define XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4 /** @info: array of elements containing the config info */ __u64 info[]; }; -- cgit v1.2.3 From be13336e07b5cc26c8b971a50ff6dc60d7050417 Mon Sep 17 00:00:00 2001 From: Aravind Iddamsetty Date: Fri, 10 Nov 2023 15:41:56 +0000 Subject: drm/xe/pmu: Drop interrupt pmu event Drop interrupt event from PMU as that is not useful and not being used by any UMD. Cc: Rodrigo Vivi Cc: Tvrtko Ursulin Cc: Francois Dugast Signed-off-by: Aravind Iddamsetty Reviewed-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_irq.c | 18 ------------------ drivers/gpu/drm/xe/xe_pmu.c | 19 +++++-------------- drivers/gpu/drm/xe/xe_pmu_types.h | 8 -------- include/uapi/drm/xe_drm.h | 13 ++++++------- 4 files changed, 11 insertions(+), 47 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_irq.c b/drivers/gpu/drm/xe/xe_irq.c index c5315e02fc5b..25ba5167c1b9 100644 --- a/drivers/gpu/drm/xe/xe_irq.c +++ b/drivers/gpu/drm/xe/xe_irq.c @@ -27,20 +27,6 @@ #define IIR(offset) XE_REG(offset + 0x8) #define IER(offset) XE_REG(offset + 0xc) -/* - * Interrupt statistic for PMU. Increments the counter only if the - * interrupt originated from the GPU so interrupts from a device which - * shares the interrupt line are not accounted. - */ -static __always_inline void xe_pmu_irq_stats(struct xe_device *xe) -{ - /* - * A clever compiler translates that into INC. A not so clever one - * should at least prevent store tearing. - */ - WRITE_ONCE(xe->pmu.irq_count, xe->pmu.irq_count + 1); -} - static void assert_iir_is_zero(struct xe_gt *mmio, struct xe_reg reg) { u32 val = xe_mmio_read32(mmio, reg); @@ -360,8 +346,6 @@ static irqreturn_t xelp_irq_handler(int irq, void *arg) xe_display_irq_enable(xe, gu_misc_iir); - xe_pmu_irq_stats(xe); - return IRQ_HANDLED; } @@ -458,8 +442,6 @@ static irqreturn_t dg1_irq_handler(int irq, void *arg) dg1_intr_enable(xe, false); xe_display_irq_enable(xe, gu_misc_iir); - xe_pmu_irq_stats(xe); - return IRQ_HANDLED; } diff --git a/drivers/gpu/drm/xe/xe_pmu.c b/drivers/gpu/drm/xe/xe_pmu.c index abfc0b3aeac4..b843259578fd 100644 --- a/drivers/gpu/drm/xe/xe_pmu.c +++ b/drivers/gpu/drm/xe/xe_pmu.c @@ -61,7 +61,7 @@ static u64 __engine_group_busyness_read(struct xe_gt *gt, int sample_type) static u64 engine_group_busyness_read(struct xe_gt *gt, u64 config) { - int sample_type = config_counter(config) - 1; + int sample_type = config_counter(config); const unsigned int gt_id = gt->info.id; struct xe_device *xe = gt->tile->xe; struct xe_pmu *pmu = &xe->pmu; @@ -114,10 +114,6 @@ config_status(struct xe_device *xe, u64 config) return -ENOENT; switch (config_counter(config)) { - case XE_PMU_INTERRUPTS(0): - if (gt_id) - return -ENOENT; - break; case XE_PMU_RENDER_GROUP_BUSY(0): case XE_PMU_COPY_GROUP_BUSY(0): case XE_PMU_ANY_ENGINE_GROUP_BUSY(0): @@ -181,13 +177,9 @@ static u64 __xe_pmu_event_read(struct perf_event *event) const unsigned int gt_id = config_gt_id(event->attr.config); const u64 config = event->attr.config; struct xe_gt *gt = xe_device_get_gt(xe, gt_id); - struct xe_pmu *pmu = &xe->pmu; u64 val; switch (config_counter(config)) { - case XE_PMU_INTERRUPTS(0): - val = READ_ONCE(pmu->irq_count); - break; case XE_PMU_RENDER_GROUP_BUSY(0): case XE_PMU_COPY_GROUP_BUSY(0): case XE_PMU_ANY_ENGINE_GROUP_BUSY(0): @@ -361,11 +353,10 @@ create_event_attributes(struct xe_pmu *pmu) const char *unit; bool global; } events[] = { - __global_event(0, "interrupts", NULL), - __event(1, "render-group-busy", "ns"), - __event(2, "copy-group-busy", "ns"), - __event(3, "media-group-busy", "ns"), - __event(4, "any-engine-group-busy", "ns"), + __event(0, "render-group-busy", "ns"), + __event(1, "copy-group-busy", "ns"), + __event(2, "media-group-busy", "ns"), + __event(3, "any-engine-group-busy", "ns"), }; struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter; diff --git a/drivers/gpu/drm/xe/xe_pmu_types.h b/drivers/gpu/drm/xe/xe_pmu_types.h index 4ccc7e9042f6..9cadbd243f57 100644 --- a/drivers/gpu/drm/xe/xe_pmu_types.h +++ b/drivers/gpu/drm/xe/xe_pmu_types.h @@ -51,14 +51,6 @@ struct xe_pmu { * */ u64 sample[XE_PMU_MAX_GT][__XE_NUM_PMU_SAMPLERS]; - /** - * @irq_count: Number of interrupts - * - * Intentionally unsigned long to avoid atomics or heuristics on 32bit. - * 4e9 interrupts are a lot and postprocessing can really deal with an - * occasional wraparound easily. It's 32bit after all. - */ - unsigned long irq_count; /** * @events_attr_group: Device events attribute group. */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 097d045d0444..e007dbefd627 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -977,7 +977,7 @@ struct drm_xe_wait_user_fence { * in 'struct perf_event_attr' as part of perf_event_open syscall to read a * particular event. * - * For example to open the XE_PMU_INTERRUPTS(0): + * For example to open the XE_PMU_RENDER_GROUP_BUSY(0): * * .. code-block:: C * @@ -991,7 +991,7 @@ struct drm_xe_wait_user_fence { * attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED; * attr.use_clockid = 1; * attr.clockid = CLOCK_MONOTONIC; - * attr.config = XE_PMU_INTERRUPTS(0); + * attr.config = XE_PMU_RENDER_GROUP_BUSY(0); * * fd = syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0); */ @@ -1004,11 +1004,10 @@ struct drm_xe_wait_user_fence { #define ___XE_PMU_OTHER(gt, x) \ (((__u64)(x)) | ((__u64)(gt) << __XE_PMU_GT_SHIFT)) -#define XE_PMU_INTERRUPTS(gt) ___XE_PMU_OTHER(gt, 0) -#define XE_PMU_RENDER_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 1) -#define XE_PMU_COPY_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 2) -#define XE_PMU_MEDIA_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 3) -#define XE_PMU_ANY_ENGINE_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 4) +#define XE_PMU_RENDER_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 0) +#define XE_PMU_COPY_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 1) +#define XE_PMU_MEDIA_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 2) +#define XE_PMU_ANY_ENGINE_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 3) #if defined(__cplusplus) } -- cgit v1.2.3 From d5dc73dbd148ef38dbe35f18d2908d2ff343c208 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Tue, 14 Nov 2023 13:34:27 +0000 Subject: drm/xe/uapi: Add missing DRM_ prefix in uAPI constants MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most constants defined in xe_drm.h use DRM_XE_ as prefix which is helpful to identify the name space. Make this systematic and add this prefix where it was missing. v2: - fix vertical alignment of define values - remove double DRM_ in some variables (José Roberto de Souza) v3: Rebase Signed-off-by: Francois Dugast Reviewed-by: Matthew Brost Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_bo.c | 14 ++--- drivers/gpu/drm/xe/xe_exec_queue.c | 22 +++---- drivers/gpu/drm/xe/xe_gt.c | 2 +- drivers/gpu/drm/xe/xe_pmu.c | 24 +++---- drivers/gpu/drm/xe/xe_query.c | 28 ++++----- drivers/gpu/drm/xe/xe_vm.c | 54 ++++++++-------- drivers/gpu/drm/xe/xe_vm_doc.h | 12 ++-- include/uapi/drm/xe_drm.h | 124 ++++++++++++++++++------------------- 8 files changed, 140 insertions(+), 140 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 5b5f764586fe..e8c89b6e06dc 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -209,7 +209,7 @@ static int __xe_bo_placement_for_flags(struct xe_device *xe, struct xe_bo *bo, /* The order of placements should indicate preferred location */ - if (bo->props.preferred_mem_class == XE_MEM_REGION_CLASS_SYSMEM) { + if (bo->props.preferred_mem_class == DRM_XE_MEM_REGION_CLASS_SYSMEM) { try_add_system(bo, places, bo_flags, &c); try_add_vram(xe, bo, places, bo_flags, &c); } else { @@ -1814,9 +1814,9 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, return -EINVAL; if (XE_IOCTL_DBG(xe, args->flags & - ~(XE_GEM_CREATE_FLAG_DEFER_BACKING | - XE_GEM_CREATE_FLAG_SCANOUT | - XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM | + ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING | + DRM_XE_GEM_CREATE_FLAG_SCANOUT | + DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM | xe->info.mem_region_mask))) return -EINVAL; @@ -1836,15 +1836,15 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, args->size & ~PAGE_MASK)) return -EINVAL; - if (args->flags & XE_GEM_CREATE_FLAG_DEFER_BACKING) + if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING) bo_flags |= XE_BO_DEFER_BACKING; - if (args->flags & XE_GEM_CREATE_FLAG_SCANOUT) + if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT) bo_flags |= XE_BO_SCANOUT_BIT; bo_flags |= args->flags << (ffs(XE_BO_CREATE_SYSTEM_BIT) - 1); - if (args->flags & XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) { + if (args->flags & DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) { if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_CREATE_VRAM_MASK))) return -EINVAL; diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index f67a6dee4a6f..fbb4d3cca9f6 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -393,7 +393,7 @@ static int exec_queue_set_acc_granularity(struct xe_device *xe, struct xe_exec_q if (XE_IOCTL_DBG(xe, !xe->info.supports_usm)) return -EINVAL; - if (value > XE_ACC_GRANULARITY_64M) + if (value > DRM_XE_ACC_GRANULARITY_64M) return -EINVAL; q->usm.acc_granularity = value; @@ -406,14 +406,14 @@ typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, u64 value, bool create); static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { - [XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, - [XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, - [XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT] = exec_queue_set_preemption_timeout, - [XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE] = exec_queue_set_persistence, - [XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT] = exec_queue_set_job_timeout, - [XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER] = exec_queue_set_acc_trigger, - [XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY] = exec_queue_set_acc_notify, - [XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY] = exec_queue_set_acc_granularity, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT] = exec_queue_set_preemption_timeout, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE] = exec_queue_set_persistence, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT] = exec_queue_set_job_timeout, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER] = exec_queue_set_acc_trigger, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY] = exec_queue_set_acc_notify, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY] = exec_queue_set_acc_granularity, }; static int exec_queue_user_ext_set_property(struct xe_device *xe, @@ -445,7 +445,7 @@ typedef int (*xe_exec_queue_user_extension_fn)(struct xe_device *xe, bool create); static const xe_exec_queue_set_property_fn exec_queue_user_extension_funcs[] = { - [XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, + [DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY] = exec_queue_user_ext_set_property, }; #define MAX_USER_EXTENSIONS 16 @@ -764,7 +764,7 @@ int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, return -ENOENT; switch (args->property) { - case XE_EXEC_QUEUE_GET_PROPERTY_BAN: + case DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN: args->value = !!(q->flags & EXEC_QUEUE_FLAG_BANNED); ret = 0; break; diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index 6c885dde5d59..53b39fe91601 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -560,7 +560,7 @@ static void xe_uevent_gt_reset_failure(struct pci_dev *pdev, u8 tile_id, u8 gt_i { char *reset_event[4]; - reset_event[0] = XE_RESET_FAILED_UEVENT "=NEEDS_RESET"; + reset_event[0] = DRM_XE_RESET_FAILED_UEVENT "=NEEDS_RESET"; reset_event[1] = kasprintf(GFP_KERNEL, "TILE_ID=%d", tile_id); reset_event[2] = kasprintf(GFP_KERNEL, "GT_ID=%d", gt_id); reset_event[3] = NULL; diff --git a/drivers/gpu/drm/xe/xe_pmu.c b/drivers/gpu/drm/xe/xe_pmu.c index b843259578fd..9d0b7887cfc4 100644 --- a/drivers/gpu/drm/xe/xe_pmu.c +++ b/drivers/gpu/drm/xe/xe_pmu.c @@ -17,12 +17,12 @@ static unsigned int xe_pmu_target_cpu = -1; static unsigned int config_gt_id(const u64 config) { - return config >> __XE_PMU_GT_SHIFT; + return config >> __DRM_XE_PMU_GT_SHIFT; } static u64 config_counter(const u64 config) { - return config & ~(~0ULL << __XE_PMU_GT_SHIFT); + return config & ~(~0ULL << __DRM_XE_PMU_GT_SHIFT); } static void xe_pmu_event_destroy(struct perf_event *event) @@ -114,13 +114,13 @@ config_status(struct xe_device *xe, u64 config) return -ENOENT; switch (config_counter(config)) { - case XE_PMU_RENDER_GROUP_BUSY(0): - case XE_PMU_COPY_GROUP_BUSY(0): - case XE_PMU_ANY_ENGINE_GROUP_BUSY(0): + case DRM_XE_PMU_RENDER_GROUP_BUSY(0): + case DRM_XE_PMU_COPY_GROUP_BUSY(0): + case DRM_XE_PMU_ANY_ENGINE_GROUP_BUSY(0): if (gt->info.type == XE_GT_TYPE_MEDIA) return -ENOENT; break; - case XE_PMU_MEDIA_GROUP_BUSY(0): + case DRM_XE_PMU_MEDIA_GROUP_BUSY(0): if (!(gt->info.engine_mask & (BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VECS0)))) return -ENOENT; break; @@ -180,10 +180,10 @@ static u64 __xe_pmu_event_read(struct perf_event *event) u64 val; switch (config_counter(config)) { - case XE_PMU_RENDER_GROUP_BUSY(0): - case XE_PMU_COPY_GROUP_BUSY(0): - case XE_PMU_ANY_ENGINE_GROUP_BUSY(0): - case XE_PMU_MEDIA_GROUP_BUSY(0): + case DRM_XE_PMU_RENDER_GROUP_BUSY(0): + case DRM_XE_PMU_COPY_GROUP_BUSY(0): + case DRM_XE_PMU_ANY_ENGINE_GROUP_BUSY(0): + case DRM_XE_PMU_MEDIA_GROUP_BUSY(0): val = engine_group_busyness_read(gt, config); break; default: @@ -369,7 +369,7 @@ create_event_attributes(struct xe_pmu *pmu) /* Count how many counters we will be exposing. */ for_each_gt(gt, xe, j) { for (i = 0; i < ARRAY_SIZE(events); i++) { - u64 config = ___XE_PMU_OTHER(j, events[i].counter); + u64 config = ___DRM_XE_PMU_OTHER(j, events[i].counter); if (!config_status(xe, config)) count++; @@ -396,7 +396,7 @@ create_event_attributes(struct xe_pmu *pmu) for_each_gt(gt, xe, j) { for (i = 0; i < ARRAY_SIZE(events); i++) { - u64 config = ___XE_PMU_OTHER(j, events[i].counter); + u64 config = ___DRM_XE_PMU_OTHER(j, events[i].counter); char *str; if (config_status(xe, config)) diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index e9c8c97a030f..565a716302bb 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -261,7 +261,7 @@ static int query_memory_usage(struct xe_device *xe, return -ENOMEM; man = ttm_manager_type(&xe->ttm, XE_PL_TT); - usage->regions[0].mem_class = XE_MEM_REGION_CLASS_SYSMEM; + usage->regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; usage->regions[0].instance = 0; usage->regions[0].min_page_size = PAGE_SIZE; usage->regions[0].total_size = man->size << PAGE_SHIFT; @@ -273,7 +273,7 @@ static int query_memory_usage(struct xe_device *xe, man = ttm_manager_type(&xe->ttm, i); if (man) { usage->regions[usage->num_regions].mem_class = - XE_MEM_REGION_CLASS_VRAM; + DRM_XE_MEM_REGION_CLASS_VRAM; usage->regions[usage->num_regions].instance = usage->num_regions; usage->regions[usage->num_regions].min_page_size = @@ -305,7 +305,7 @@ static int query_memory_usage(struct xe_device *xe, static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) { - const u32 num_params = XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1; + const u32 num_params = DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY + 1; size_t size = sizeof(struct drm_xe_query_config) + num_params * sizeof(u64); struct drm_xe_query_config __user *query_ptr = @@ -324,15 +324,15 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) return -ENOMEM; config->num_params = num_params; - config->info[XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = + config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = xe->info.devid | (xe->info.revid << 16); if (xe_device_get_root_tile(xe)->mem.vram.usable_size) - config->info[XE_QUERY_CONFIG_FLAGS] = - XE_QUERY_CONFIG_FLAGS_HAS_VRAM; - config->info[XE_QUERY_CONFIG_MIN_ALIGNMENT] = + config->info[DRM_XE_QUERY_CONFIG_FLAGS] = + DRM_XE_QUERY_CONFIG_FLAGS_HAS_VRAM; + config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; - config->info[XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; - config->info[XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = + config->info[DRM_XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; + config->info[DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY] = xe_exec_queue_device_get_max_priority(xe); if (copy_to_user(query_ptr, config, size)) { @@ -368,9 +368,9 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query gt_list->num_gt = xe->info.gt_count; for_each_gt(gt, xe, id) { if (xe_gt_is_media_type(gt)) - gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_MEDIA; + gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MEDIA; else - gt_list->gt_list[id].type = XE_QUERY_GT_TYPE_MAIN; + gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MAIN; gt_list->gt_list[id].gt_id = gt->info.id; gt_list->gt_list[id].clock_freq = gt->info.clock_freq; if (!IS_DGFX(xe)) @@ -468,21 +468,21 @@ static int query_gt_topology(struct xe_device *xe, for_each_gt(gt, xe, id) { topo.gt_id = id; - topo.type = XE_TOPO_DSS_GEOMETRY; + topo.type = DRM_XE_TOPO_DSS_GEOMETRY; query_ptr = copy_mask(query_ptr, &topo, gt->fuse_topo.g_dss_mask, sizeof(gt->fuse_topo.g_dss_mask)); if (IS_ERR(query_ptr)) return PTR_ERR(query_ptr); - topo.type = XE_TOPO_DSS_COMPUTE; + topo.type = DRM_XE_TOPO_DSS_COMPUTE; query_ptr = copy_mask(query_ptr, &topo, gt->fuse_topo.c_dss_mask, sizeof(gt->fuse_topo.c_dss_mask)); if (IS_ERR(query_ptr)) return PTR_ERR(query_ptr); - topo.type = XE_TOPO_EU_PER_DSS; + topo.type = DRM_XE_TOPO_EU_PER_DSS; query_ptr = copy_mask(query_ptr, &topo, gt->fuse_topo.eu_mask_per_dss, sizeof(gt->fuse_topo.eu_mask_per_dss)); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index b4a4ed28019c..66d878bc464a 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2177,8 +2177,8 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, (ULL)bo_offset_or_userptr); switch (operation) { - case XE_VM_BIND_OP_MAP: - case XE_VM_BIND_OP_MAP_USERPTR: + case DRM_XE_VM_BIND_OP_MAP: + case DRM_XE_VM_BIND_OP_MAP_USERPTR: ops = drm_gpuvm_sm_map_ops_create(&vm->gpuvm, addr, range, obj, bo_offset_or_userptr); if (IS_ERR(ops)) @@ -2189,13 +2189,13 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, op->tile_mask = tile_mask; op->map.immediate = - flags & XE_VM_BIND_FLAG_IMMEDIATE; + flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE; op->map.read_only = - flags & XE_VM_BIND_FLAG_READONLY; - op->map.is_null = flags & XE_VM_BIND_FLAG_NULL; + flags & DRM_XE_VM_BIND_FLAG_READONLY; + op->map.is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; } break; - case XE_VM_BIND_OP_UNMAP: + case DRM_XE_VM_BIND_OP_UNMAP: ops = drm_gpuvm_sm_unmap_ops_create(&vm->gpuvm, addr, range); if (IS_ERR(ops)) return ops; @@ -2206,7 +2206,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, op->tile_mask = tile_mask; } break; - case XE_VM_BIND_OP_PREFETCH: + case DRM_XE_VM_BIND_OP_PREFETCH: ops = drm_gpuvm_prefetch_ops_create(&vm->gpuvm, addr, range); if (IS_ERR(ops)) return ops; @@ -2218,7 +2218,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, op->prefetch.region = region; } break; - case XE_VM_BIND_OP_UNMAP_ALL: + case DRM_XE_VM_BIND_OP_UNMAP_ALL: xe_assert(vm->xe, bo); err = xe_bo_lock(bo, true); @@ -2828,13 +2828,13 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm, #ifdef TEST_VM_ASYNC_OPS_ERROR #define SUPPORTED_FLAGS \ - (FORCE_ASYNC_OP_ERROR | XE_VM_BIND_FLAG_ASYNC | \ - XE_VM_BIND_FLAG_READONLY | XE_VM_BIND_FLAG_IMMEDIATE | \ - XE_VM_BIND_FLAG_NULL | 0xffff) + (FORCE_ASYNC_OP_ERROR | DRM_XE_VM_BIND_FLAG_ASYNC | \ + DRM_XE_VM_BIND_FLAG_READONLY | DRM_XE_VM_BIND_FLAG_IMMEDIATE | \ + DRM_XE_VM_BIND_FLAG_NULL | 0xffff) #else #define SUPPORTED_FLAGS \ - (XE_VM_BIND_FLAG_ASYNC | XE_VM_BIND_FLAG_READONLY | \ - XE_VM_BIND_FLAG_IMMEDIATE | XE_VM_BIND_FLAG_NULL | \ + (DRM_XE_VM_BIND_FLAG_ASYNC | DRM_XE_VM_BIND_FLAG_READONLY | \ + DRM_XE_VM_BIND_FLAG_IMMEDIATE | DRM_XE_VM_BIND_FLAG_NULL | \ 0xffff) #endif #define XE_64K_PAGE_MASK 0xffffull @@ -2882,45 +2882,45 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, u32 obj = (*bind_ops)[i].obj; u64 obj_offset = (*bind_ops)[i].obj_offset; u32 region = (*bind_ops)[i].region; - bool is_null = flags & XE_VM_BIND_FLAG_NULL; + bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; if (i == 0) { - *async = !!(flags & XE_VM_BIND_FLAG_ASYNC); + *async = !!(flags & DRM_XE_VM_BIND_FLAG_ASYNC); if (XE_IOCTL_DBG(xe, !*async && args->num_syncs)) { err = -EINVAL; goto free_bind_ops; } } else if (XE_IOCTL_DBG(xe, *async != - !!(flags & XE_VM_BIND_FLAG_ASYNC))) { + !!(flags & DRM_XE_VM_BIND_FLAG_ASYNC))) { err = -EINVAL; goto free_bind_ops; } - if (XE_IOCTL_DBG(xe, op > XE_VM_BIND_OP_PREFETCH) || + if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) || XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) || XE_IOCTL_DBG(xe, obj && is_null) || XE_IOCTL_DBG(xe, obj_offset && is_null) || - XE_IOCTL_DBG(xe, op != XE_VM_BIND_OP_MAP && + XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP && is_null) || XE_IOCTL_DBG(xe, !obj && - op == XE_VM_BIND_OP_MAP && + op == DRM_XE_VM_BIND_OP_MAP && !is_null) || XE_IOCTL_DBG(xe, !obj && - op == XE_VM_BIND_OP_UNMAP_ALL) || + op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || XE_IOCTL_DBG(xe, addr && - op == XE_VM_BIND_OP_UNMAP_ALL) || + op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || XE_IOCTL_DBG(xe, range && - op == XE_VM_BIND_OP_UNMAP_ALL) || + op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || XE_IOCTL_DBG(xe, obj && - op == XE_VM_BIND_OP_MAP_USERPTR) || + op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || XE_IOCTL_DBG(xe, obj && - op == XE_VM_BIND_OP_PREFETCH) || + op == DRM_XE_VM_BIND_OP_PREFETCH) || XE_IOCTL_DBG(xe, region && - op != XE_VM_BIND_OP_PREFETCH) || + op != DRM_XE_VM_BIND_OP_PREFETCH) || XE_IOCTL_DBG(xe, !(BIT(region) & xe->info.mem_region_mask)) || XE_IOCTL_DBG(xe, obj && - op == XE_VM_BIND_OP_UNMAP)) { + op == DRM_XE_VM_BIND_OP_UNMAP)) { err = -EINVAL; goto free_bind_ops; } @@ -2929,7 +2929,7 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, XE_IOCTL_DBG(xe, addr & ~PAGE_MASK) || XE_IOCTL_DBG(xe, range & ~PAGE_MASK) || XE_IOCTL_DBG(xe, !range && - op != XE_VM_BIND_OP_UNMAP_ALL)) { + op != DRM_XE_VM_BIND_OP_UNMAP_ALL)) { err = -EINVAL; goto free_bind_ops; } diff --git a/drivers/gpu/drm/xe/xe_vm_doc.h b/drivers/gpu/drm/xe/xe_vm_doc.h index b1b2dc4a6089..516f4dc97223 100644 --- a/drivers/gpu/drm/xe/xe_vm_doc.h +++ b/drivers/gpu/drm/xe/xe_vm_doc.h @@ -32,9 +32,9 @@ * Operations * ---------- * - * XE_VM_BIND_OP_MAP - Create mapping for a BO - * XE_VM_BIND_OP_UNMAP - Destroy mapping for a BO / userptr - * XE_VM_BIND_OP_MAP_USERPTR - Create mapping for userptr + * DRM_XE_VM_BIND_OP_MAP - Create mapping for a BO + * DRM_XE_VM_BIND_OP_UNMAP - Destroy mapping for a BO / userptr + * DRM_XE_VM_BIND_OP_MAP_USERPTR - Create mapping for userptr * * Implementation details * ~~~~~~~~~~~~~~~~~~~~~~ @@ -113,7 +113,7 @@ * VM uses to report errors to. The ufence wait interface can be used to wait on * a VM going into an error state. Once an error is reported the VM's async * worker is paused. While the VM's async worker is paused sync, - * XE_VM_BIND_OP_UNMAP operations are allowed (this can free memory). Once the + * DRM_XE_VM_BIND_OP_UNMAP operations are allowed (this can free memory). Once the * uses believe the error state is fixed, the async worker can be resumed via * XE_VM_BIND_OP_RESTART operation. When VM async bind work is restarted, the * first operation processed is the operation that caused the original error. @@ -193,7 +193,7 @@ * In a VM is in fault mode (TODO: link to fault mode), new bind operations that * create mappings are by default are deferred to the page fault handler (first * use). This behavior can be overriden by setting the flag - * XE_VM_BIND_FLAG_IMMEDIATE which indicates to creating the mapping + * DRM_XE_VM_BIND_FLAG_IMMEDIATE which indicates to creating the mapping * immediately. * * User pointer @@ -322,7 +322,7 @@ * * By default, on a faulting VM binds just allocate the VMA and the actual * updating of the page tables is defered to the page fault handler. This - * behavior can be overridden by setting the flag XE_VM_BIND_FLAG_IMMEDIATE in + * behavior can be overridden by setting the flag DRM_XE_VM_BIND_FLAG_IMMEDIATE in * the VM bind which will then do the bind immediately. * * Page fault handler diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index e007dbefd627..3ef49e3baaed 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -19,12 +19,12 @@ extern "C" { /** * DOC: uevent generated by xe on it's pci node. * - * XE_RESET_FAILED_UEVENT - Event is generated when attempt to reset gt + * DRM_XE_RESET_FAILED_UEVENT - Event is generated when attempt to reset gt * fails. The value supplied with the event is always "NEEDS_RESET". * Additional information supplied is tile id and gt id of the gt unit for * which reset has failed. */ -#define XE_RESET_FAILED_UEVENT "DEVICE_STATUS" +#define DRM_XE_RESET_FAILED_UEVENT "DEVICE_STATUS" /** * struct xe_user_extension - Base class for defining a chain of extensions @@ -148,14 +148,14 @@ struct drm_xe_engine_class_instance { * enum drm_xe_memory_class - Supported memory classes. */ enum drm_xe_memory_class { - /** @XE_MEM_REGION_CLASS_SYSMEM: Represents system memory. */ - XE_MEM_REGION_CLASS_SYSMEM = 0, + /** @DRM_XE_MEM_REGION_CLASS_SYSMEM: Represents system memory. */ + DRM_XE_MEM_REGION_CLASS_SYSMEM = 0, /** - * @XE_MEM_REGION_CLASS_VRAM: On discrete platforms, this + * @DRM_XE_MEM_REGION_CLASS_VRAM: On discrete platforms, this * represents the memory that is local to the device, which we * call VRAM. Not valid on integrated platforms. */ - XE_MEM_REGION_CLASS_VRAM + DRM_XE_MEM_REGION_CLASS_VRAM }; /** @@ -215,7 +215,7 @@ struct drm_xe_query_mem_region { * always equal the @total_size, since all of it will be CPU * accessible. * - * Note this is only tracked for XE_MEM_REGION_CLASS_VRAM + * Note this is only tracked for DRM_XE_MEM_REGION_CLASS_VRAM * regions (for other types the value here will always equal * zero). */ @@ -227,7 +227,7 @@ struct drm_xe_query_mem_region { * Requires CAP_PERFMON or CAP_SYS_ADMIN to get reliable * accounting. Without this the value here will always equal * zero. Note this is only currently tracked for - * XE_MEM_REGION_CLASS_VRAM regions (for other types the value + * DRM_XE_MEM_REGION_CLASS_VRAM regions (for other types the value * here will always be zero). */ __u64 cpu_visible_used; @@ -320,12 +320,12 @@ struct drm_xe_query_config { /** @pad: MBZ */ __u32 pad; -#define XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 -#define XE_QUERY_CONFIG_FLAGS 1 - #define XE_QUERY_CONFIG_FLAGS_HAS_VRAM (0x1 << 0) -#define XE_QUERY_CONFIG_MIN_ALIGNMENT 2 -#define XE_QUERY_CONFIG_VA_BITS 3 -#define XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4 +#define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 +#define DRM_XE_QUERY_CONFIG_FLAGS 1 + #define DRM_XE_QUERY_CONFIG_FLAGS_HAS_VRAM (0x1 << 0) +#define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2 +#define DRM_XE_QUERY_CONFIG_VA_BITS 3 +#define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4 /** @info: array of elements containing the config info */ __u64 info[]; }; @@ -339,8 +339,8 @@ struct drm_xe_query_config { * implementing graphics and/or media operations. */ struct drm_xe_query_gt { -#define XE_QUERY_GT_TYPE_MAIN 0 -#define XE_QUERY_GT_TYPE_MEDIA 1 +#define DRM_XE_QUERY_GT_TYPE_MAIN 0 +#define DRM_XE_QUERY_GT_TYPE_MEDIA 1 /** @type: GT type: Main or Media */ __u16 type; /** @gt_id: Unique ID of this GT within the PCI Device */ @@ -400,7 +400,7 @@ struct drm_xe_query_topology_mask { * DSS_GEOMETRY ff ff ff ff 00 00 00 00 * means 32 DSS are available for geometry. */ -#define XE_TOPO_DSS_GEOMETRY (1 << 0) +#define DRM_XE_TOPO_DSS_GEOMETRY (1 << 0) /* * To query the mask of Dual Sub Slices (DSS) available for compute * operations. For example a query response containing the following @@ -408,7 +408,7 @@ struct drm_xe_query_topology_mask { * DSS_COMPUTE ff ff ff ff 00 00 00 00 * means 32 DSS are available for compute. */ -#define XE_TOPO_DSS_COMPUTE (1 << 1) +#define DRM_XE_TOPO_DSS_COMPUTE (1 << 1) /* * To query the mask of Execution Units (EU) available per Dual Sub * Slices (DSS). For example a query response containing the following @@ -416,7 +416,7 @@ struct drm_xe_query_topology_mask { * EU_PER_DSS ff ff 00 00 00 00 00 00 * means each DSS has 16 EU. */ -#define XE_TOPO_EU_PER_DSS (1 << 2) +#define DRM_XE_TOPO_EU_PER_DSS (1 << 2) /** @type: type of mask */ __u16 type; @@ -497,8 +497,8 @@ struct drm_xe_gem_create { */ __u64 size; -#define XE_GEM_CREATE_FLAG_DEFER_BACKING (0x1 << 24) -#define XE_GEM_CREATE_FLAG_SCANOUT (0x1 << 25) +#define DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING (0x1 << 24) +#define DRM_XE_GEM_CREATE_FLAG_SCANOUT (0x1 << 25) /* * When using VRAM as a possible placement, ensure that the corresponding VRAM * allocation will always use the CPU accessible part of VRAM. This is important @@ -514,7 +514,7 @@ struct drm_xe_gem_create { * display surfaces, therefore the kernel requires setting this flag for such * objects, otherwise an error is thrown on small-bar systems. */ -#define XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM (0x1 << 26) +#define DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM (0x1 << 26) /** * @flags: Flags, currently a mask of memory instances of where BO can * be placed @@ -581,14 +581,14 @@ struct drm_xe_ext_set_property { }; struct drm_xe_vm_create { -#define XE_VM_EXTENSION_SET_PROPERTY 0 +#define DRM_XE_VM_EXTENSION_SET_PROPERTY 0 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; -#define DRM_XE_VM_CREATE_SCRATCH_PAGE (0x1 << 0) -#define DRM_XE_VM_CREATE_COMPUTE_MODE (0x1 << 1) -#define DRM_XE_VM_CREATE_ASYNC_DEFAULT (0x1 << 2) -#define DRM_XE_VM_CREATE_FAULT_MODE (0x1 << 3) +#define DRM_XE_VM_CREATE_SCRATCH_PAGE (0x1 << 0) +#define DRM_XE_VM_CREATE_COMPUTE_MODE (0x1 << 1) +#define DRM_XE_VM_CREATE_ASYNC_DEFAULT (0x1 << 2) +#define DRM_XE_VM_CREATE_FAULT_MODE (0x1 << 3) /** @flags: Flags */ __u32 flags; @@ -644,29 +644,29 @@ struct drm_xe_vm_bind_op { */ __u64 tile_mask; -#define XE_VM_BIND_OP_MAP 0x0 -#define XE_VM_BIND_OP_UNMAP 0x1 -#define XE_VM_BIND_OP_MAP_USERPTR 0x2 -#define XE_VM_BIND_OP_UNMAP_ALL 0x3 -#define XE_VM_BIND_OP_PREFETCH 0x4 +#define DRM_XE_VM_BIND_OP_MAP 0x0 +#define DRM_XE_VM_BIND_OP_UNMAP 0x1 +#define DRM_XE_VM_BIND_OP_MAP_USERPTR 0x2 +#define DRM_XE_VM_BIND_OP_UNMAP_ALL 0x3 +#define DRM_XE_VM_BIND_OP_PREFETCH 0x4 /** @op: Bind operation to perform */ __u32 op; -#define XE_VM_BIND_FLAG_READONLY (0x1 << 0) -#define XE_VM_BIND_FLAG_ASYNC (0x1 << 1) +#define DRM_XE_VM_BIND_FLAG_READONLY (0x1 << 0) +#define DRM_XE_VM_BIND_FLAG_ASYNC (0x1 << 1) /* * Valid on a faulting VM only, do the MAP operation immediately rather * than deferring the MAP to the page fault handler. */ -#define XE_VM_BIND_FLAG_IMMEDIATE (0x1 << 2) +#define DRM_XE_VM_BIND_FLAG_IMMEDIATE (0x1 << 2) /* * When the NULL flag is set, the page tables are setup with a special * bit which indicates writes are dropped and all reads return zero. In - * the future, the NULL flags will only be valid for XE_VM_BIND_OP_MAP + * the future, the NULL flags will only be valid for DRM_XE_VM_BIND_OP_MAP * operations, the BO handle MBZ, and the BO offset MBZ. This flag is * intended to implement VK sparse bindings. */ -#define XE_VM_BIND_FLAG_NULL (0x1 << 3) +#define DRM_XE_VM_BIND_FLAG_NULL (0x1 << 3) /** @flags: Bind flags */ __u32 flags; @@ -721,19 +721,19 @@ struct drm_xe_vm_bind { __u64 reserved[2]; }; -/* For use with XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY */ +/* For use with DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY */ /* Monitor 128KB contiguous region with 4K sub-granularity */ -#define XE_ACC_GRANULARITY_128K 0 +#define DRM_XE_ACC_GRANULARITY_128K 0 /* Monitor 2MB contiguous region with 64KB sub-granularity */ -#define XE_ACC_GRANULARITY_2M 1 +#define DRM_XE_ACC_GRANULARITY_2M 1 /* Monitor 16MB contiguous region with 512KB sub-granularity */ -#define XE_ACC_GRANULARITY_16M 2 +#define DRM_XE_ACC_GRANULARITY_16M 2 /* Monitor 64MB contiguous region with 2M sub-granularity */ -#define XE_ACC_GRANULARITY_64M 3 +#define DRM_XE_ACC_GRANULARITY_64M 3 /** * struct drm_xe_exec_queue_set_property - exec queue set property @@ -747,14 +747,14 @@ struct drm_xe_exec_queue_set_property { /** @exec_queue_id: Exec queue ID */ __u32 exec_queue_id; -#define XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 -#define XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1 -#define XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT 2 -#define XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE 3 -#define XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT 4 -#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER 5 -#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY 6 -#define XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY 7 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT 2 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE 3 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT 4 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER 5 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY 6 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY 7 /** @property: property to set */ __u32 property; @@ -766,7 +766,7 @@ struct drm_xe_exec_queue_set_property { }; struct drm_xe_exec_queue_create { -#define XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 +#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -805,7 +805,7 @@ struct drm_xe_exec_queue_get_property { /** @exec_queue_id: Exec queue ID */ __u32 exec_queue_id; -#define XE_EXEC_QUEUE_GET_PROPERTY_BAN 0 +#define DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN 0 /** @property: property to get */ __u32 property; @@ -973,11 +973,11 @@ struct drm_xe_wait_user_fence { /** * DOC: XE PMU event config IDs * - * Check 'man perf_event_open' to use the ID's XE_PMU_XXXX listed in xe_drm.h + * Check 'man perf_event_open' to use the ID's DRM_XE_PMU_XXXX listed in xe_drm.h * in 'struct perf_event_attr' as part of perf_event_open syscall to read a * particular event. * - * For example to open the XE_PMU_RENDER_GROUP_BUSY(0): + * For example to open the DRMXE_PMU_RENDER_GROUP_BUSY(0): * * .. code-block:: C * @@ -991,7 +991,7 @@ struct drm_xe_wait_user_fence { * attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED; * attr.use_clockid = 1; * attr.clockid = CLOCK_MONOTONIC; - * attr.config = XE_PMU_RENDER_GROUP_BUSY(0); + * attr.config = DRM_XE_PMU_RENDER_GROUP_BUSY(0); * * fd = syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0); */ @@ -999,15 +999,15 @@ struct drm_xe_wait_user_fence { /* * Top bits of every counter are GT id. */ -#define __XE_PMU_GT_SHIFT (56) +#define __DRM_XE_PMU_GT_SHIFT (56) -#define ___XE_PMU_OTHER(gt, x) \ - (((__u64)(x)) | ((__u64)(gt) << __XE_PMU_GT_SHIFT)) +#define ___DRM_XE_PMU_OTHER(gt, x) \ + (((__u64)(x)) | ((__u64)(gt) << __DRM_XE_PMU_GT_SHIFT)) -#define XE_PMU_RENDER_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 0) -#define XE_PMU_COPY_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 1) -#define XE_PMU_MEDIA_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 2) -#define XE_PMU_ANY_ENGINE_GROUP_BUSY(gt) ___XE_PMU_OTHER(gt, 3) +#define DRM_XE_PMU_RENDER_GROUP_BUSY(gt) ___DRM_XE_PMU_OTHER(gt, 0) +#define DRM_XE_PMU_COPY_GROUP_BUSY(gt) ___DRM_XE_PMU_OTHER(gt, 1) +#define DRM_XE_PMU_MEDIA_GROUP_BUSY(gt) ___DRM_XE_PMU_OTHER(gt, 2) +#define DRM_XE_PMU_ANY_ENGINE_GROUP_BUSY(gt) ___DRM_XE_PMU_OTHER(gt, 3) #if defined(__cplusplus) } -- cgit v1.2.3 From 3ac4a7896d1c02918ee76acaf7e8160f3d11fa75 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Tue, 14 Nov 2023 13:34:28 +0000 Subject: drm/xe/uapi: Add _FLAG to uAPI constants usable for flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Most constants defined in xe_drm.h which can be used for flags are named DRM_XE_*_FLAG_*, which is helpful to identify them. Make this systematic and add _FLAG where it was missing. Signed-off-by: Francois Dugast Reviewed-by: Rodrigo Vivi Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_sync.c | 16 ++++++++-------- drivers/gpu/drm/xe/xe_vm.c | 32 ++++++++++++++++---------------- drivers/gpu/drm/xe/xe_vm_doc.h | 2 +- drivers/gpu/drm/xe/xe_wait_user_fence.c | 10 +++++----- include/uapi/drm/xe_drm.h | 30 +++++++++++++++--------------- 5 files changed, 45 insertions(+), 45 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index 73ef259aa387..eafe53c2f55d 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -110,14 +110,14 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, return -EFAULT; if (XE_IOCTL_DBG(xe, sync_in.flags & - ~(SYNC_FLAGS_TYPE_MASK | DRM_XE_SYNC_SIGNAL)) || + ~(SYNC_FLAGS_TYPE_MASK | DRM_XE_SYNC_FLAG_SIGNAL)) || XE_IOCTL_DBG(xe, sync_in.pad) || XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) return -EINVAL; - signal = sync_in.flags & DRM_XE_SYNC_SIGNAL; + signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; switch (sync_in.flags & SYNC_FLAGS_TYPE_MASK) { - case DRM_XE_SYNC_SYNCOBJ: + case DRM_XE_SYNC_FLAG_SYNCOBJ: if (XE_IOCTL_DBG(xe, no_dma_fences && signal)) return -EOPNOTSUPP; @@ -135,7 +135,7 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, } break; - case DRM_XE_SYNC_TIMELINE_SYNCOBJ: + case DRM_XE_SYNC_FLAG_TIMELINE_SYNCOBJ: if (XE_IOCTL_DBG(xe, no_dma_fences && signal)) return -EOPNOTSUPP; @@ -165,12 +165,12 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, } break; - case DRM_XE_SYNC_DMA_BUF: + case DRM_XE_SYNC_FLAG_DMA_BUF: if (XE_IOCTL_DBG(xe, "TODO")) return -EINVAL; break; - case DRM_XE_SYNC_USER_FENCE: + case DRM_XE_SYNC_FLAG_USER_FENCE: if (XE_IOCTL_DBG(xe, !signal)) return -EOPNOTSUPP; @@ -225,7 +225,7 @@ int xe_sync_entry_add_deps(struct xe_sync_entry *sync, struct xe_sched_job *job) void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, struct dma_fence *fence) { - if (!(sync->flags & DRM_XE_SYNC_SIGNAL)) + if (!(sync->flags & DRM_XE_SYNC_FLAG_SIGNAL)) return; if (sync->chain_fence) { @@ -253,7 +253,7 @@ void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, dma_fence_put(fence); } } else if ((sync->flags & SYNC_FLAGS_TYPE_MASK) == - DRM_XE_SYNC_USER_FENCE) { + DRM_XE_SYNC_FLAG_USER_FENCE) { job->user_fence.used = true; job->user_fence.addr = sync->addr; job->user_fence.value = sync->timeline_value; diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 66d878bc464a..e8dd46789537 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1920,10 +1920,10 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, return 0; } -#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_SCRATCH_PAGE | \ - DRM_XE_VM_CREATE_COMPUTE_MODE | \ - DRM_XE_VM_CREATE_ASYNC_DEFAULT | \ - DRM_XE_VM_CREATE_FAULT_MODE) +#define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \ + DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE | \ + DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT | \ + DRM_XE_VM_CREATE_FLAG_FAULT_MODE) int xe_vm_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) @@ -1941,9 +1941,9 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, return -EINVAL; if (XE_WA(xe_root_mmio_gt(xe), 14016763929)) - args->flags |= DRM_XE_VM_CREATE_SCRATCH_PAGE; + args->flags |= DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE; - if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FAULT_MODE && + if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && !xe->info.supports_usm)) return -EINVAL; @@ -1953,32 +1953,32 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, args->flags & ~ALL_DRM_XE_VM_CREATE_FLAGS)) return -EINVAL; - if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_SCRATCH_PAGE && - args->flags & DRM_XE_VM_CREATE_FAULT_MODE)) + if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE && + args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) return -EINVAL; - if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE && - args->flags & DRM_XE_VM_CREATE_FAULT_MODE)) + if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE && + args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) return -EINVAL; - if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FAULT_MODE && + if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE && xe_device_in_non_fault_mode(xe))) return -EINVAL; - if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FAULT_MODE) && + if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) && xe_device_in_fault_mode(xe))) return -EINVAL; if (XE_IOCTL_DBG(xe, args->extensions)) return -EINVAL; - if (args->flags & DRM_XE_VM_CREATE_SCRATCH_PAGE) + if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE) flags |= XE_VM_FLAG_SCRATCH_PAGE; - if (args->flags & DRM_XE_VM_CREATE_COMPUTE_MODE) + if (args->flags & DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE) flags |= XE_VM_FLAG_COMPUTE_MODE; - if (args->flags & DRM_XE_VM_CREATE_ASYNC_DEFAULT) + if (args->flags & DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT) flags |= XE_VM_FLAG_ASYNC_DEFAULT; - if (args->flags & DRM_XE_VM_CREATE_FAULT_MODE) + if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) flags |= XE_VM_FLAG_FAULT_MODE; vm = xe_vm_create(xe, flags); diff --git a/drivers/gpu/drm/xe/xe_vm_doc.h b/drivers/gpu/drm/xe/xe_vm_doc.h index 516f4dc97223..bdc6659891a5 100644 --- a/drivers/gpu/drm/xe/xe_vm_doc.h +++ b/drivers/gpu/drm/xe/xe_vm_doc.h @@ -18,7 +18,7 @@ * Scratch page * ------------ * - * If the VM is created with the flag, DRM_XE_VM_CREATE_SCRATCH_PAGE, set the + * If the VM is created with the flag, DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE, set the * entire page table structure defaults pointing to blank page allocated by the * VM. Invalid memory access rather than fault just read / write to this page. * diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.c b/drivers/gpu/drm/xe/xe_wait_user_fence.c index 78686908f7fb..13562db6c07f 100644 --- a/drivers/gpu/drm/xe/xe_wait_user_fence.c +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.c @@ -79,8 +79,8 @@ static int check_hw_engines(struct xe_device *xe, return 0; } -#define VALID_FLAGS (DRM_XE_UFENCE_WAIT_SOFT_OP | \ - DRM_XE_UFENCE_WAIT_ABSTIME) +#define VALID_FLAGS (DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP | \ + DRM_XE_UFENCE_WAIT_FLAG_ABSTIME) #define MAX_OP DRM_XE_UFENCE_WAIT_LTE static long to_jiffies_timeout(struct xe_device *xe, @@ -107,7 +107,7 @@ static long to_jiffies_timeout(struct xe_device *xe, * Save the timeout to an u64 variable because nsecs_to_jiffies * might return a value that overflows s32 variable. */ - if (args->flags & DRM_XE_UFENCE_WAIT_ABSTIME) + if (args->flags & DRM_XE_UFENCE_WAIT_FLAG_ABSTIME) t = drm_timeout_abs_to_jiffies(args->timeout); else t = nsecs_to_jiffies(args->timeout); @@ -137,7 +137,7 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, u64_to_user_ptr(args->instances); u64 addr = args->addr; int err; - bool no_engines = args->flags & DRM_XE_UFENCE_WAIT_SOFT_OP; + bool no_engines = args->flags & DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP; long timeout; ktime_t start; @@ -206,7 +206,7 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, } remove_wait_queue(&xe->ufence_wq, &w_wait); - if (!(args->flags & DRM_XE_UFENCE_WAIT_ABSTIME)) { + if (!(args->flags & DRM_XE_UFENCE_WAIT_FLAG_ABSTIME)) { args->timeout -= ktime_to_ns(ktime_sub(ktime_get(), start)); if (args->timeout < 0) args->timeout = 0; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 3ef49e3baaed..f6346a8351e4 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -585,10 +585,10 @@ struct drm_xe_vm_create { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; -#define DRM_XE_VM_CREATE_SCRATCH_PAGE (0x1 << 0) -#define DRM_XE_VM_CREATE_COMPUTE_MODE (0x1 << 1) -#define DRM_XE_VM_CREATE_ASYNC_DEFAULT (0x1 << 2) -#define DRM_XE_VM_CREATE_FAULT_MODE (0x1 << 3) +#define DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE (0x1 << 0) +#define DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE (0x1 << 1) +#define DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT (0x1 << 2) +#define DRM_XE_VM_CREATE_FLAG_FAULT_MODE (0x1 << 3) /** @flags: Flags */ __u32 flags; @@ -831,11 +831,11 @@ struct drm_xe_sync { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; -#define DRM_XE_SYNC_SYNCOBJ 0x0 -#define DRM_XE_SYNC_TIMELINE_SYNCOBJ 0x1 -#define DRM_XE_SYNC_DMA_BUF 0x2 -#define DRM_XE_SYNC_USER_FENCE 0x3 -#define DRM_XE_SYNC_SIGNAL 0x10 +#define DRM_XE_SYNC_FLAG_SYNCOBJ 0x0 +#define DRM_XE_SYNC_FLAG_TIMELINE_SYNCOBJ 0x1 +#define DRM_XE_SYNC_FLAG_DMA_BUF 0x2 +#define DRM_XE_SYNC_FLAG_USER_FENCE 0x3 +#define DRM_XE_SYNC_FLAG_SIGNAL 0x10 __u32 flags; /** @pad: MBZ */ @@ -921,8 +921,8 @@ struct drm_xe_wait_user_fence { /** @op: wait operation (type of comparison) */ __u16 op; -#define DRM_XE_UFENCE_WAIT_SOFT_OP (1 << 0) /* e.g. Wait on VM bind */ -#define DRM_XE_UFENCE_WAIT_ABSTIME (1 << 1) +#define DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP (1 << 0) /* e.g. Wait on VM bind */ +#define DRM_XE_UFENCE_WAIT_FLAG_ABSTIME (1 << 1) /** @flags: wait flags */ __u16 flags; @@ -940,10 +940,10 @@ struct drm_xe_wait_user_fence { __u64 mask; /** * @timeout: how long to wait before bailing, value in nanoseconds. - * Without DRM_XE_UFENCE_WAIT_ABSTIME flag set (relative timeout) + * Without DRM_XE_UFENCE_WAIT_FLAG_ABSTIME flag set (relative timeout) * it contains timeout expressed in nanoseconds to wait (fence will * expire at now() + timeout). - * When DRM_XE_UFENCE_WAIT_ABSTIME flat is set (absolute timeout) wait + * When DRM_XE_UFENCE_WAIT_FLAG_ABSTIME flat is set (absolute timeout) wait * will end at timeout (uses system MONOTONIC_CLOCK). * Passing negative timeout leads to neverending wait. * @@ -956,13 +956,13 @@ struct drm_xe_wait_user_fence { /** * @num_engines: number of engine instances to wait on, must be zero - * when DRM_XE_UFENCE_WAIT_SOFT_OP set + * when DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP set */ __u64 num_engines; /** * @instances: user pointer to array of drm_xe_engine_class_instance to - * wait on, must be NULL when DRM_XE_UFENCE_WAIT_SOFT_OP set + * wait on, must be NULL when DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP set */ __u64 instances; -- cgit v1.2.3 From 5ca2c4b800194b55a863882273b8ca34b56afb35 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Tue, 14 Nov 2023 13:34:29 +0000 Subject: drm/xe/uapi: Change rsvd to pad in struct drm_xe_class_instance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change rsvd to pad in struct drm_xe_class_instance to prevent the field from being used in future. v2: Change from fixup to regular commit because this touches the uAPI (Francois Dugast) Signed-off-by: Umesh Nerlige Ramappa Signed-off-by: Francois Dugast Reviewed-by: Rodrigo Vivi Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_query.c | 5 ++++- include/uapi/drm/xe_drm.h | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 565a716302bb..48befd9f0812 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -215,7 +215,10 @@ static int query_engines(struct xe_device *xe, xe_to_user_engine_class[hwe->class]; hw_engine_info[i].engine_instance = hwe->logical_instance; - hw_engine_info[i++].gt_id = gt->info.id; + hw_engine_info[i].gt_id = gt->info.id; + hw_engine_info[i].pad = 0; + + i++; } if (copy_to_user(query_ptr, hw_engine_info, size)) { diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index f6346a8351e4..a8d351c9fa7c 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -141,7 +141,8 @@ struct drm_xe_engine_class_instance { __u16 engine_instance; __u16 gt_id; - __u16 rsvd; + /** @pad: MBZ */ + __u16 pad; }; /** -- cgit v1.2.3 From 45c30d80008264d55915f4b87c6f9bbb3261071c Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Tue, 14 Nov 2023 13:34:30 +0000 Subject: drm/xe/uapi: Rename *_mem_regions masks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 'native' doesn't make much sense on integrated devices. - 'slow' is not necessarily true and doesn't go well with opposition to 'native'. Instead, let's use 'near' vs 'far'. It makes sense with all the current Intel GPUs and it is future proof. Right now, there's absolutely no need to define among the 'far' memory, which ones are slower, either in terms of latency, nunmber of hops or bandwidth. In case of this might become a requirement in the future, a new query could be added to indicate the certain 'distance' between a given engine and a memory_region. But for now, this fulfill all of the current requirements in the most straightforward way for the userspace drivers. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matt Roper Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_query.c | 8 ++++---- include/uapi/drm/xe_drm.h | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 48befd9f0812..8b5136460ea6 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -377,12 +377,12 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query gt_list->gt_list[id].gt_id = gt->info.id; gt_list->gt_list[id].clock_freq = gt->info.clock_freq; if (!IS_DGFX(xe)) - gt_list->gt_list[id].native_mem_regions = 0x1; + gt_list->gt_list[id].near_mem_regions = 0x1; else - gt_list->gt_list[id].native_mem_regions = + gt_list->gt_list[id].near_mem_regions = BIT(gt_to_tile(gt)->id) << 1; - gt_list->gt_list[id].slow_mem_regions = xe->info.mem_region_mask ^ - gt_list->gt_list[id].native_mem_regions; + gt_list->gt_list[id].far_mem_regions = xe->info.mem_region_mask ^ + gt_list->gt_list[id].near_mem_regions; } if (copy_to_user(query_ptr, gt_list, size)) { diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index a8d351c9fa7c..30567500e6cd 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -349,17 +349,19 @@ struct drm_xe_query_gt { /** @clock_freq: A clock frequency for timestamp */ __u32 clock_freq; /** - * @native_mem_regions: Bit mask of instances from - * drm_xe_query_mem_usage that lives on the same GPU/Tile and have - * direct access. + * @near_mem_regions: Bit mask of instances from + * drm_xe_query_mem_usage that are nearest to the current engines + * of this GT. */ - __u64 native_mem_regions; + __u64 near_mem_regions; /** - * @slow_mem_regions: Bit mask of instances from - * drm_xe_query_mem_usage that this GT can indirectly access, although - * they live on a different GPU/Tile. + * @far_mem_regions: Bit mask of instances from + * drm_xe_query_mem_usage that are far from the engines of this GT. + * In general, they have extra indirections when compared to the + * @near_mem_regions. For a discrete device this could mean system + * memory and memory living in a different tile. */ - __u64 slow_mem_regions; + __u64 far_mem_regions; /** @reserved: Reserved */ __u64 reserved[8]; }; -- cgit v1.2.3 From b02606d32376b8d51b33211f8c069b16165390eb Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Tue, 14 Nov 2023 13:34:31 +0000 Subject: drm/xe/uapi: Rename query's mem_usage to mem_regions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 'Usage' gives an impression of telemetry information where someone would query to see how the memory is currently used and available size, etc. However this API is more than this. It is about a global view of all the memory regions available in the system and user space needs to have this information so they can then use the mem_region masks that are returned for the engine access. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matt Roper Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_query.c | 16 ++++++++-------- include/uapi/drm/xe_drm.h | 14 +++++++------- 2 files changed, 15 insertions(+), 15 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 8b5136460ea6..d495716b2c96 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -230,7 +230,7 @@ static int query_engines(struct xe_device *xe, return 0; } -static size_t calc_memory_usage_size(struct xe_device *xe) +static size_t calc_mem_regions_size(struct xe_device *xe) { u32 num_managers = 1; int i; @@ -239,15 +239,15 @@ static size_t calc_memory_usage_size(struct xe_device *xe) if (ttm_manager_type(&xe->ttm, i)) num_managers++; - return offsetof(struct drm_xe_query_mem_usage, regions[num_managers]); + return offsetof(struct drm_xe_query_mem_regions, regions[num_managers]); } -static int query_memory_usage(struct xe_device *xe, - struct drm_xe_device_query *query) +static int query_mem_regions(struct xe_device *xe, + struct drm_xe_device_query *query) { - size_t size = calc_memory_usage_size(xe); - struct drm_xe_query_mem_usage *usage; - struct drm_xe_query_mem_usage __user *query_ptr = + size_t size = calc_mem_regions_size(xe); + struct drm_xe_query_mem_regions *usage; + struct drm_xe_query_mem_regions __user *query_ptr = u64_to_user_ptr(query->data); struct ttm_resource_manager *man; int ret, i; @@ -499,7 +499,7 @@ static int query_gt_topology(struct xe_device *xe, static int (* const xe_query_funcs[])(struct xe_device *xe, struct drm_xe_device_query *query) = { query_engines, - query_memory_usage, + query_mem_regions, query_config, query_gt_list, query_hwconfig, diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 30567500e6cd..8ec12f9f4132 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -291,13 +291,13 @@ struct drm_xe_query_engine_cycles { }; /** - * struct drm_xe_query_mem_usage - describe memory regions and usage + * struct drm_xe_query_mem_regions - describe memory regions * * If a query is made with a struct drm_xe_device_query where .query - * is equal to DRM_XE_DEVICE_QUERY_MEM_USAGE, then the reply uses - * struct drm_xe_query_mem_usage in .data. + * is equal to DRM_XE_DEVICE_QUERY_MEM_REGIONS, then the reply uses + * struct drm_xe_query_mem_regions in .data. */ -struct drm_xe_query_mem_usage { +struct drm_xe_query_mem_regions { /** @num_regions: number of memory regions returned in @regions */ __u32 num_regions; /** @pad: MBZ */ @@ -350,13 +350,13 @@ struct drm_xe_query_gt { __u32 clock_freq; /** * @near_mem_regions: Bit mask of instances from - * drm_xe_query_mem_usage that are nearest to the current engines + * drm_xe_query_mem_regions that are nearest to the current engines * of this GT. */ __u64 near_mem_regions; /** * @far_mem_regions: Bit mask of instances from - * drm_xe_query_mem_usage that are far from the engines of this GT. + * drm_xe_query_mem_regions that are far from the engines of this GT. * In general, they have extra indirections when compared to the * @near_mem_regions. For a discrete device this could mean system * memory and memory living in a different tile. @@ -470,7 +470,7 @@ struct drm_xe_device_query { __u64 extensions; #define DRM_XE_DEVICE_QUERY_ENGINES 0 -#define DRM_XE_DEVICE_QUERY_MEM_USAGE 1 +#define DRM_XE_DEVICE_QUERY_MEM_REGIONS 1 #define DRM_XE_DEVICE_QUERY_CONFIG 2 #define DRM_XE_DEVICE_QUERY_GT_LIST 3 #define DRM_XE_DEVICE_QUERY_HWCONFIG 4 -- cgit v1.2.3 From 9ad743515cc59275653f719886d1b93fa7a824ab Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Tue, 14 Nov 2023 13:34:32 +0000 Subject: drm/xe/uapi: Standardize the FLAG naming and assignment Only cosmetic things. No functional change on this patch. Define every flag with (1 << n) and use singular FLAG name. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matthew Brost --- drivers/gpu/drm/xe/xe_query.c | 2 +- include/uapi/drm/xe_drm.h | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index d495716b2c96..61a7d92b7e88 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -331,7 +331,7 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) xe->info.devid | (xe->info.revid << 16); if (xe_device_get_root_tile(xe)->mem.vram.usable_size) config->info[DRM_XE_QUERY_CONFIG_FLAGS] = - DRM_XE_QUERY_CONFIG_FLAGS_HAS_VRAM; + DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM; config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; config->info[DRM_XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 8ec12f9f4132..236e643be69a 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -323,7 +323,7 @@ struct drm_xe_query_config { #define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 #define DRM_XE_QUERY_CONFIG_FLAGS 1 - #define DRM_XE_QUERY_CONFIG_FLAGS_HAS_VRAM (0x1 << 0) + #define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) #define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2 #define DRM_XE_QUERY_CONFIG_VA_BITS 3 #define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4 @@ -588,10 +588,10 @@ struct drm_xe_vm_create { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; -#define DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE (0x1 << 0) -#define DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE (0x1 << 1) -#define DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT (0x1 << 2) -#define DRM_XE_VM_CREATE_FLAG_FAULT_MODE (0x1 << 3) +#define DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE (1 << 0) +#define DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE (1 << 1) +#define DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT (1 << 2) +#define DRM_XE_VM_CREATE_FLAG_FAULT_MODE (1 << 3) /** @flags: Flags */ __u32 flags; @@ -655,13 +655,13 @@ struct drm_xe_vm_bind_op { /** @op: Bind operation to perform */ __u32 op; -#define DRM_XE_VM_BIND_FLAG_READONLY (0x1 << 0) -#define DRM_XE_VM_BIND_FLAG_ASYNC (0x1 << 1) +#define DRM_XE_VM_BIND_FLAG_READONLY (1 << 0) +#define DRM_XE_VM_BIND_FLAG_ASYNC (1 << 1) /* * Valid on a faulting VM only, do the MAP operation immediately rather * than deferring the MAP to the page fault handler. */ -#define DRM_XE_VM_BIND_FLAG_IMMEDIATE (0x1 << 2) +#define DRM_XE_VM_BIND_FLAG_IMMEDIATE (1 << 2) /* * When the NULL flag is set, the page tables are setup with a special * bit which indicates writes are dropped and all reads return zero. In @@ -669,7 +669,7 @@ struct drm_xe_vm_bind_op { * operations, the BO handle MBZ, and the BO offset MBZ. This flag is * intended to implement VK sparse bindings. */ -#define DRM_XE_VM_BIND_FLAG_NULL (0x1 << 3) +#define DRM_XE_VM_BIND_FLAG_NULL (1 << 3) /** @flags: Bind flags */ __u32 flags; -- cgit v1.2.3 From 4a349c86110a6fab26ce5f4fcb545acf214efed5 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Tue, 14 Nov 2023 13:34:33 +0000 Subject: drm/xe/uapi: Differentiate WAIT_OP from WAIT_MASK On one hand the WAIT_OP represents the operation use for waiting such as ==, !=, > and so on. On the other hand, the mask is applied to the value used for comparision. Split those two to bring clarity to the uapi. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matt Roper --- drivers/gpu/drm/xe/xe_wait_user_fence.c | 14 +++++++------- include/uapi/drm/xe_drm.h | 21 +++++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.c b/drivers/gpu/drm/xe/xe_wait_user_fence.c index 13562db6c07f..4d5c2555ce41 100644 --- a/drivers/gpu/drm/xe/xe_wait_user_fence.c +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.c @@ -25,22 +25,22 @@ static int do_compare(u64 addr, u64 value, u64 mask, u16 op) return -EFAULT; switch (op) { - case DRM_XE_UFENCE_WAIT_EQ: + case DRM_XE_UFENCE_WAIT_OP_EQ: passed = (rvalue & mask) == (value & mask); break; - case DRM_XE_UFENCE_WAIT_NEQ: + case DRM_XE_UFENCE_WAIT_OP_NEQ: passed = (rvalue & mask) != (value & mask); break; - case DRM_XE_UFENCE_WAIT_GT: + case DRM_XE_UFENCE_WAIT_OP_GT: passed = (rvalue & mask) > (value & mask); break; - case DRM_XE_UFENCE_WAIT_GTE: + case DRM_XE_UFENCE_WAIT_OP_GTE: passed = (rvalue & mask) >= (value & mask); break; - case DRM_XE_UFENCE_WAIT_LT: + case DRM_XE_UFENCE_WAIT_OP_LT: passed = (rvalue & mask) < (value & mask); break; - case DRM_XE_UFENCE_WAIT_LTE: + case DRM_XE_UFENCE_WAIT_OP_LTE: passed = (rvalue & mask) <= (value & mask); break; default: @@ -81,7 +81,7 @@ static int check_hw_engines(struct xe_device *xe, #define VALID_FLAGS (DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP | \ DRM_XE_UFENCE_WAIT_FLAG_ABSTIME) -#define MAX_OP DRM_XE_UFENCE_WAIT_LTE +#define MAX_OP DRM_XE_UFENCE_WAIT_OP_LTE static long to_jiffies_timeout(struct xe_device *xe, struct drm_xe_wait_user_fence *args) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 236e643be69a..b2bd76efd940 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -915,12 +915,12 @@ struct drm_xe_wait_user_fence { */ __u64 addr; -#define DRM_XE_UFENCE_WAIT_EQ 0 -#define DRM_XE_UFENCE_WAIT_NEQ 1 -#define DRM_XE_UFENCE_WAIT_GT 2 -#define DRM_XE_UFENCE_WAIT_GTE 3 -#define DRM_XE_UFENCE_WAIT_LT 4 -#define DRM_XE_UFENCE_WAIT_LTE 5 +#define DRM_XE_UFENCE_WAIT_OP_EQ 0x0 +#define DRM_XE_UFENCE_WAIT_OP_NEQ 0x1 +#define DRM_XE_UFENCE_WAIT_OP_GT 0x2 +#define DRM_XE_UFENCE_WAIT_OP_GTE 0x3 +#define DRM_XE_UFENCE_WAIT_OP_LT 0x4 +#define DRM_XE_UFENCE_WAIT_OP_LTE 0x5 /** @op: wait operation (type of comparison) */ __u16 op; @@ -935,12 +935,13 @@ struct drm_xe_wait_user_fence { /** @value: compare value */ __u64 value; -#define DRM_XE_UFENCE_WAIT_U8 0xffu -#define DRM_XE_UFENCE_WAIT_U16 0xffffu -#define DRM_XE_UFENCE_WAIT_U32 0xffffffffu -#define DRM_XE_UFENCE_WAIT_U64 0xffffffffffffffffu +#define DRM_XE_UFENCE_WAIT_MASK_U8 0xffu +#define DRM_XE_UFENCE_WAIT_MASK_U16 0xffffu +#define DRM_XE_UFENCE_WAIT_MASK_U32 0xffffffffu +#define DRM_XE_UFENCE_WAIT_MASK_U64 0xffffffffffffffffu /** @mask: comparison mask */ __u64 mask; + /** * @timeout: how long to wait before bailing, value in nanoseconds. * Without DRM_XE_UFENCE_WAIT_FLAG_ABSTIME flag set (relative timeout) -- cgit v1.2.3 From aaa115ffaa467782b01cfa81711424315823bdb5 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Tue, 14 Nov 2023 13:34:34 +0000 Subject: drm/xe/uapi: Be more specific about the vm_bind prefetch region Let's bring a bit of clarity on this 'region' field that is part of vm_bind operation struct. Rename and document to make it more than obvious that it is a region instance and not a mask and also that it should only be used with the prefetch operation itself. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matt Roper --- drivers/gpu/drm/xe/xe_vm.c | 15 ++++++++------- include/uapi/drm/xe_drm.h | 8 ++++++-- 2 files changed, 14 insertions(+), 9 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index e8dd46789537..174441c4ca5a 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2160,7 +2160,8 @@ static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) static struct drm_gpuva_ops * vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, u64 bo_offset_or_userptr, u64 addr, u64 range, - u32 operation, u32 flags, u8 tile_mask, u32 region) + u32 operation, u32 flags, u8 tile_mask, + u32 prefetch_region) { struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; struct drm_gpuva_ops *ops; @@ -2215,7 +2216,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, struct xe_vma_op *op = gpuva_op_to_vma_op(__op); op->tile_mask = tile_mask; - op->prefetch.region = region; + op->prefetch.region = prefetch_region; } break; case DRM_XE_VM_BIND_OP_UNMAP_ALL: @@ -2881,7 +2882,7 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, u32 flags = (*bind_ops)[i].flags; u32 obj = (*bind_ops)[i].obj; u64 obj_offset = (*bind_ops)[i].obj_offset; - u32 region = (*bind_ops)[i].region; + u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance; bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; if (i == 0) { @@ -2915,9 +2916,9 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || XE_IOCTL_DBG(xe, obj && op == DRM_XE_VM_BIND_OP_PREFETCH) || - XE_IOCTL_DBG(xe, region && + XE_IOCTL_DBG(xe, prefetch_region && op != DRM_XE_VM_BIND_OP_PREFETCH) || - XE_IOCTL_DBG(xe, !(BIT(region) & + XE_IOCTL_DBG(xe, !(BIT(prefetch_region) & xe->info.mem_region_mask)) || XE_IOCTL_DBG(xe, obj && op == DRM_XE_VM_BIND_OP_UNMAP)) { @@ -3099,11 +3100,11 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) u32 flags = bind_ops[i].flags; u64 obj_offset = bind_ops[i].obj_offset; u8 tile_mask = bind_ops[i].tile_mask; - u32 region = bind_ops[i].region; + u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance; ops[i] = vm_bind_ioctl_ops_create(vm, bos[i], obj_offset, addr, range, op, flags, - tile_mask, region); + tile_mask, prefetch_region); if (IS_ERR(ops[i])) { err = PTR_ERR(ops[i]); ops[i] = NULL; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index b2bd76efd940..88f3aca02b08 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -673,8 +673,12 @@ struct drm_xe_vm_bind_op { /** @flags: Bind flags */ __u32 flags; - /** @mem_region: Memory region to prefetch VMA to, instance not a mask */ - __u32 region; + /** + * @prefetch_mem_region_instance: Memory region to prefetch VMA to. + * It is a region instance, not a mask. + * To be used only with %DRM_XE_VM_BIND_OP_PREFETCH operation. + */ + __u32 prefetch_mem_region_instance; /** @reserved: Reserved */ __u64 reserved[2]; -- cgit v1.2.3 From 622f709ca6297d838d9bd8b33196b388909d5951 Mon Sep 17 00:00:00 2001 From: Pallavi Mishra Date: Fri, 11 Aug 2023 01:36:43 +0530 Subject: drm/xe/uapi: Add support for CPU caching mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow userspace to specify the CPU caching mode at object creation. Modify gem create handler and introduce xe_bo_create_user to replace xe_bo_create. In a later patch we will support setting the pat_index as part of vm_bind, where expectation is that the coherency mode extracted from the pat_index must be least 1way coherent if using cpu_caching=wb. v2 - s/smem_caching/smem_cpu_caching/ and s/XE_GEM_CACHING/XE_GEM_CPU_CACHING/. (Matt Roper) - Drop COH_2WAY and just use COH_NONE + COH_AT_LEAST_1WAY; KMD mostly just cares that zeroing/swap-in can't be bypassed with the given smem_caching mode. (Matt Roper) - Fix broken range check for coh_mode and smem_cpu_caching and also don't use constant value, but the already defined macros. (José) - Prefer switch statement for smem_cpu_caching -> ttm_caching. (José) - Add note in kernel-doc for dgpu and coherency modes for system memory. (José) v3 (José): - Make sure to reject coh_mode == 0 for VRAM-only. - Also make sure to actually pass along the (start, end) for __xe_bo_create_locked. v4 - Drop UC caching mode. Can be added back if we need it. (Matt Roper) - s/smem_cpu_caching/cpu_caching. Idea is that VRAM is always WC, but that is currently implicit and KMD controlled. Make it explicit in the uapi with the limitation that it currently must be WC. For VRAM + SYS objects userspace must now select WC. (José) - Make sure to initialize bo_flags. (José) v5 - Make to align with the other uapi and prefix uapi constants with DRM_ (José) v6: - Make it clear that zero cpu_caching is only allowed for kernel objects. (José) v7: (Oak) - With all the changes from the original design, it looks we can further simplify here and drop the explicit coh_mode. We can just infer the coh_mode from the cpu_caching. i.e reject cpu_caching=wb + coh_none. It's one less thing for userspace to maintain so seems worth it. v8: - Make sure to also update the kselftests. Testcase: igt@xe_mmap@cpu-caching Signed-off-by: Pallavi Mishra Co-developed-by: Matthew Auld Signed-off-by: Matthew Auld Cc: Thomas Hellström Cc: Joonas Lahtinen Cc: Lucas De Marchi Cc: Matt Roper Cc: José Roberto de Souza Cc: Filip Hazubski Cc: Carl Zhang Cc: Effie Yu Cc: Zhengguo Xu Cc: Francois Dugast Cc: Oak Zeng Reviewed-by: José Roberto de Souza Acked-by: Zhengguo Xu Acked-by: Bartosz Dunajski Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/tests/xe_bo.c | 14 +++-- drivers/gpu/drm/xe/tests/xe_dma_buf.c | 4 +- drivers/gpu/drm/xe/xe_bo.c | 100 ++++++++++++++++++++++++++-------- drivers/gpu/drm/xe/xe_bo.h | 14 +++-- drivers/gpu/drm/xe/xe_bo_types.h | 5 ++ drivers/gpu/drm/xe/xe_dma_buf.c | 5 +- include/uapi/drm/xe_drm.h | 19 ++++++- 7 files changed, 122 insertions(+), 39 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/tests/xe_bo.c b/drivers/gpu/drm/xe/tests/xe_bo.c index 2c04357377ab..549ab343de80 100644 --- a/drivers/gpu/drm/xe/tests/xe_bo.c +++ b/drivers/gpu/drm/xe/tests/xe_bo.c @@ -177,8 +177,7 @@ EXPORT_SYMBOL_IF_KUNIT(xe_ccs_migrate_kunit); static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struct kunit *test) { struct xe_bo *bo, *external; - unsigned int bo_flags = XE_BO_CREATE_USER_BIT | - XE_BO_CREATE_VRAM_IF_DGFX(tile); + unsigned int bo_flags = XE_BO_CREATE_VRAM_IF_DGFX(tile); struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->migrate); struct xe_gt *__gt; int err, i, id; @@ -188,16 +187,19 @@ static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struc for (i = 0; i < 2; ++i) { xe_vm_lock(vm, false); - bo = xe_bo_create(xe, NULL, vm, 0x10000, ttm_bo_type_device, - bo_flags); + bo = xe_bo_create_user(xe, NULL, vm, 0x10000, + DRM_XE_GEM_CPU_CACHING_WC, + ttm_bo_type_device, + bo_flags); xe_vm_unlock(vm); if (IS_ERR(bo)) { KUNIT_FAIL(test, "bo create err=%pe\n", bo); break; } - external = xe_bo_create(xe, NULL, NULL, 0x10000, - ttm_bo_type_device, bo_flags); + external = xe_bo_create_user(xe, NULL, NULL, 0x10000, + DRM_XE_GEM_CPU_CACHING_WC, + ttm_bo_type_device, bo_flags); if (IS_ERR(external)) { KUNIT_FAIL(test, "external bo create err=%pe\n", external); goto cleanup_bo; diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c index 18c00bc03024..81f12422a587 100644 --- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c @@ -116,8 +116,8 @@ static void xe_test_dmabuf_import_same_driver(struct xe_device *xe) return; kunit_info(test, "running %s\n", __func__); - bo = xe_bo_create(xe, NULL, NULL, PAGE_SIZE, ttm_bo_type_device, - XE_BO_CREATE_USER_BIT | params->mem_mask); + bo = xe_bo_create_user(xe, NULL, NULL, PAGE_SIZE, DRM_XE_GEM_CPU_CACHING_WC, + ttm_bo_type_device, params->mem_mask); if (IS_ERR(bo)) { KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n", PTR_ERR(bo)); diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index e19337390812..dc1ad3b4dc2a 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -332,7 +332,7 @@ static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo, struct xe_device *xe = xe_bo_device(bo); struct xe_ttm_tt *tt; unsigned long extra_pages; - enum ttm_caching caching = ttm_cached; + enum ttm_caching caching; int err; tt = kzalloc(sizeof(*tt), GFP_KERNEL); @@ -346,13 +346,24 @@ static struct ttm_tt *xe_ttm_tt_create(struct ttm_buffer_object *ttm_bo, extra_pages = DIV_ROUND_UP(xe_device_ccs_bytes(xe, bo->size), PAGE_SIZE); + switch (bo->cpu_caching) { + case DRM_XE_GEM_CPU_CACHING_WC: + caching = ttm_write_combined; + break; + default: + caching = ttm_cached; + break; + } + + WARN_ON((bo->flags & XE_BO_CREATE_USER_BIT) && !bo->cpu_caching); + /* * Display scanout is always non-coherent with the CPU cache. * * For Xe_LPG and beyond, PPGTT PTE lookups are also non-coherent and * require a CPU:WC mapping. */ - if (bo->flags & XE_BO_SCANOUT_BIT || + if ((!bo->cpu_caching && bo->flags & XE_BO_SCANOUT_BIT) || (xe->info.graphics_verx100 >= 1270 && bo->flags & XE_BO_PAGETABLE)) caching = ttm_write_combined; @@ -1198,10 +1209,11 @@ void xe_bo_free(struct xe_bo *bo) kfree(bo); } -struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, - struct xe_tile *tile, struct dma_resv *resv, - struct ttm_lru_bulk_move *bulk, size_t size, - enum ttm_bo_type type, u32 flags) +struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, + struct xe_tile *tile, struct dma_resv *resv, + struct ttm_lru_bulk_move *bulk, size_t size, + u16 cpu_caching, enum ttm_bo_type type, + u32 flags) { struct ttm_operation_ctx ctx = { .interruptible = true, @@ -1239,6 +1251,7 @@ struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, bo->tile = tile; bo->size = size; bo->flags = flags; + bo->cpu_caching = cpu_caching; bo->ttm.base.funcs = &xe_gem_object_funcs; bo->props.preferred_mem_class = XE_BO_PROPS_INVALID; bo->props.preferred_gt = XE_BO_PROPS_INVALID; @@ -1354,11 +1367,11 @@ static int __xe_bo_fixed_placement(struct xe_device *xe, return 0; } -struct xe_bo * -xe_bo_create_locked_range(struct xe_device *xe, - struct xe_tile *tile, struct xe_vm *vm, - size_t size, u64 start, u64 end, - enum ttm_bo_type type, u32 flags) +static struct xe_bo * +__xe_bo_create_locked(struct xe_device *xe, + struct xe_tile *tile, struct xe_vm *vm, + size_t size, u64 start, u64 end, + u16 cpu_caching, enum ttm_bo_type type, u32 flags) { struct xe_bo *bo = NULL; int err; @@ -1379,11 +1392,11 @@ xe_bo_create_locked_range(struct xe_device *xe, } } - bo = __xe_bo_create_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL, - vm && !xe_vm_in_fault_mode(vm) && - flags & XE_BO_CREATE_USER_BIT ? - &vm->lru_bulk_move : NULL, size, - type, flags); + bo = ___xe_bo_create_locked(xe, bo, tile, vm ? xe_vm_resv(vm) : NULL, + vm && !xe_vm_in_fault_mode(vm) && + flags & XE_BO_CREATE_USER_BIT ? + &vm->lru_bulk_move : NULL, size, + cpu_caching, type, flags); if (IS_ERR(bo)) return bo; @@ -1423,11 +1436,35 @@ err_unlock_put_bo: return ERR_PTR(err); } +struct xe_bo * +xe_bo_create_locked_range(struct xe_device *xe, + struct xe_tile *tile, struct xe_vm *vm, + size_t size, u64 start, u64 end, + enum ttm_bo_type type, u32 flags) +{ + return __xe_bo_create_locked(xe, tile, vm, size, start, end, 0, type, flags); +} + struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags) { - return xe_bo_create_locked_range(xe, tile, vm, size, 0, ~0ULL, type, flags); + return __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, 0, type, flags); +} + +struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile, + struct xe_vm *vm, size_t size, + u16 cpu_caching, + enum ttm_bo_type type, + u32 flags) +{ + struct xe_bo *bo = __xe_bo_create_locked(xe, tile, vm, size, 0, ~0ULL, + cpu_caching, type, + flags | XE_BO_CREATE_USER_BIT); + if (!IS_ERR(bo)) + xe_bo_unlock_vm_held(bo); + + return bo; } struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile, @@ -1809,7 +1846,7 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_xe_gem_create *args = data; struct xe_vm *vm = NULL; struct xe_bo *bo; - unsigned int bo_flags = XE_BO_CREATE_USER_BIT; + unsigned int bo_flags; u32 handle; int err; @@ -1840,6 +1877,7 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, args->size & ~PAGE_MASK)) return -EINVAL; + bo_flags = 0; if (args->flags & DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING) bo_flags |= XE_BO_DEFER_BACKING; @@ -1855,6 +1893,18 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, bo_flags |= XE_BO_NEEDS_CPU_ACCESS; } + if (XE_IOCTL_DBG(xe, !args->cpu_caching || + args->cpu_caching > DRM_XE_GEM_CPU_CACHING_WC)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_CREATE_VRAM_MASK && + args->cpu_caching != DRM_XE_GEM_CPU_CACHING_WC)) + return -EINVAL; + + if (XE_IOCTL_DBG(xe, bo_flags & XE_BO_SCANOUT_BIT && + args->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) + return -EINVAL; + if (args->vm_id) { vm = xe_vm_lookup(xef, args->vm_id); if (XE_IOCTL_DBG(xe, !vm)) @@ -1864,8 +1914,8 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, goto out_vm; } - bo = xe_bo_create(xe, NULL, vm, args->size, ttm_bo_type_device, - bo_flags); + bo = xe_bo_create_user(xe, NULL, vm, args->size, args->cpu_caching, + ttm_bo_type_device, bo_flags); if (vm) xe_vm_unlock(vm); @@ -2163,10 +2213,12 @@ int xe_bo_dumb_create(struct drm_file *file_priv, args->size = ALIGN(mul_u32_u32(args->pitch, args->height), page_size); - bo = xe_bo_create(xe, NULL, NULL, args->size, ttm_bo_type_device, - XE_BO_CREATE_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | - XE_BO_CREATE_USER_BIT | XE_BO_SCANOUT_BIT | - XE_BO_NEEDS_CPU_ACCESS); + bo = xe_bo_create_user(xe, NULL, NULL, args->size, + DRM_XE_GEM_CPU_CACHING_WC, + ttm_bo_type_device, + XE_BO_CREATE_VRAM_IF_DGFX(xe_device_get_root_tile(xe)) | + XE_BO_CREATE_USER_BIT | XE_BO_SCANOUT_BIT | + XE_BO_NEEDS_CPU_ACCESS); if (IS_ERR(bo)) return PTR_ERR(bo); diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index f8bae873418d..6f183568f76d 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -94,10 +94,11 @@ struct sg_table; struct xe_bo *xe_bo_alloc(void); void xe_bo_free(struct xe_bo *bo); -struct xe_bo *__xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, - struct xe_tile *tile, struct dma_resv *resv, - struct ttm_lru_bulk_move *bulk, size_t size, - enum ttm_bo_type type, u32 flags); +struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, + struct xe_tile *tile, struct dma_resv *resv, + struct ttm_lru_bulk_move *bulk, size_t size, + u16 cpu_caching, enum ttm_bo_type type, + u32 flags); struct xe_bo * xe_bo_create_locked_range(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, @@ -109,6 +110,11 @@ struct xe_bo *xe_bo_create_locked(struct xe_device *xe, struct xe_tile *tile, struct xe_bo *xe_bo_create(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags); +struct xe_bo *xe_bo_create_user(struct xe_device *xe, struct xe_tile *tile, + struct xe_vm *vm, size_t size, + u16 cpu_caching, + enum ttm_bo_type type, + u32 flags); struct xe_bo *xe_bo_create_pin_map(struct xe_device *xe, struct xe_tile *tile, struct xe_vm *vm, size_t size, enum ttm_bo_type type, u32 flags); diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index 4bff60996168..f71dbc518958 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -79,6 +79,11 @@ struct xe_bo { struct llist_node freed; /** @created: Whether the bo has passed initial creation */ bool created; + /** + * @cpu_caching: CPU caching mode. Currently only used for userspace + * objects. + */ + u16 cpu_caching; }; #define intel_bo_to_drm_bo(bo) (&(bo)->ttm.base) diff --git a/drivers/gpu/drm/xe/xe_dma_buf.c b/drivers/gpu/drm/xe/xe_dma_buf.c index cfde3be3b0dc..64ed303728fd 100644 --- a/drivers/gpu/drm/xe/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/xe_dma_buf.c @@ -214,8 +214,9 @@ xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage, int ret; dma_resv_lock(resv, NULL); - bo = __xe_bo_create_locked(xe, storage, NULL, resv, NULL, dma_buf->size, - ttm_bo_type_sg, XE_BO_CREATE_SYSTEM_BIT); + bo = ___xe_bo_create_locked(xe, storage, NULL, resv, NULL, dma_buf->size, + 0, /* Will require 1way or 2way for vm_bind */ + ttm_bo_type_sg, XE_BO_CREATE_SYSTEM_BIT); if (IS_ERR(bo)) { ret = PTR_ERR(bo); goto error; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 88f3aca02b08..ab7d1b26c773 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -541,8 +541,25 @@ struct drm_xe_gem_create { */ __u32 handle; + /** + * @cpu_caching: The CPU caching mode to select for this object. If + * mmaping the object the mode selected here will also be used. + * + * Supported values: + * + * DRM_XE_GEM_CPU_CACHING_WB: Allocate the pages with write-back + * caching. On iGPU this can't be used for scanout surfaces. Currently + * not allowed for objects placed in VRAM. + * + * DRM_XE_GEM_CPU_CACHING_WC: Allocate the pages as write-combined. This + * is uncached. Scanout surfaces should likely use this. All objects + * that can be placed in VRAM must use this. + */ +#define DRM_XE_GEM_CPU_CACHING_WB 1 +#define DRM_XE_GEM_CPU_CACHING_WC 2 + __u16 cpu_caching; /** @pad: MBZ */ - __u32 pad; + __u16 pad; /** @reserved: Reserved */ __u64 reserved[2]; -- cgit v1.2.3 From e1fbc4f18d5b4405271e964670b9b054c4397127 Mon Sep 17 00:00:00 2001 From: Matthew Auld Date: Mon, 25 Sep 2023 12:42:18 +0100 Subject: drm/xe/uapi: support pat_index selection with vm_bind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow userspace to directly control the pat_index for a given vm binding. This should allow directly controlling the coherency, caching behaviour, compression and potentially other stuff in the future for the ppGTT binding. The exact meaning behind the pat_index is very platform specific (see BSpec or PRMs) but effectively maps to some predefined memory attributes. From the KMD pov we only care about the coherency that is provided by the pat_index, which falls into either NONE, 1WAY or 2WAY. The vm_bind coherency mode for the given pat_index needs to be at least 1way coherent when using cpu_caching with DRM_XE_GEM_CPU_CACHING_WB. For platforms that lack the explicit coherency mode attribute, we treat UC/WT/WC as NONE and WB as AT_LEAST_1WAY. For userptr mappings we lack a corresponding gem object, so the expected coherency mode is instead implicit and must fall into either 1WAY or 2WAY. Trying to use NONE will be rejected by the kernel. For imported dma-buf (from a different device) the coherency mode is also implicit and must also be either 1WAY or 2WAY. v2: - Undefined coh_mode(pat_index) can now be treated as programmer error. (Matt Roper) - We now allow gem_create.coh_mode <= coh_mode(pat_index), rather than having to match exactly. This ensures imported dma-buf can always just use 1way (or even 2way), now that we also bundle 1way/2way into at_least_1way. We still require 1way/2way for external dma-buf, but the policy can now be the same for self-import, if desired. - Use u16 for pat_index in uapi. u32 is massive overkill. (José) - Move as much of the pat_index validation as we can into vm_bind_ioctl_check_args. (José) v3 (Matt Roper): - Split the pte_encode() refactoring into separate patch. v4: - Rebase v5: - Check for and reject !coh_mode which would indicate hw reserved pat_index on xe2. v6: - Rebase on removal of coh_mode from uapi. We just need to reject cpu_caching=wb + pat_index with coh_none. Testcase: igt@xe_pat Bspec: 45101, 44235 #xe Bspec: 70552, 71582, 59400 #xe2 Signed-off-by: Matthew Auld Cc: Pallavi Mishra Cc: Thomas Hellström Cc: Joonas Lahtinen Cc: Lucas De Marchi Cc: Matt Roper Cc: José Roberto de Souza Cc: Filip Hazubski Cc: Carl Zhang Cc: Effie Yu Cc: Zhengguo Xu Cc: Francois Dugast Tested-by: José Roberto de Souza Reviewed-by: José Roberto de Souza Acked-by: Zhengguo Xu Acked-by: Bartosz Dunajski Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_pt.c | 11 ++----- drivers/gpu/drm/xe/xe_vm.c | 67 +++++++++++++++++++++++++++++++++++----- drivers/gpu/drm/xe/xe_vm_types.h | 7 +++++ include/uapi/drm/xe_drm.h | 48 +++++++++++++++++++++++++++- 4 files changed, 115 insertions(+), 18 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index c6c9b723db5a..3b485313804a 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -290,8 +290,6 @@ struct xe_pt_stage_bind_walk { struct xe_vm *vm; /** @tile: The tile we're building for. */ struct xe_tile *tile; - /** @cache: Desired cache level for the ptes */ - enum xe_cache_level cache; /** @default_pte: PTE flag only template. No address is associated */ u64 default_pte; /** @dma_offset: DMA offset to add to the PTE. */ @@ -511,7 +509,7 @@ xe_pt_stage_bind_entry(struct xe_ptw *parent, pgoff_t offset, { struct xe_pt_stage_bind_walk *xe_walk = container_of(walk, typeof(*xe_walk), base); - u16 pat_index = tile_to_xe(xe_walk->tile)->pat.idx[xe_walk->cache]; + u16 pat_index = xe_walk->vma->pat_index; struct xe_pt *xe_parent = container_of(parent, typeof(*xe_parent), base); struct xe_vm *vm = xe_walk->vm; struct xe_pt *xe_child; @@ -657,13 +655,8 @@ xe_pt_stage_bind(struct xe_tile *tile, struct xe_vma *vma, if (is_devmem) { xe_walk.default_pte |= XE_PPGTT_PTE_DM; xe_walk.dma_offset = vram_region_gpu_offset(bo->ttm.resource); - xe_walk.cache = XE_CACHE_WB; - } else { - if (!xe_vma_has_no_bo(vma) && bo->flags & XE_BO_SCANOUT_BIT) - xe_walk.cache = XE_CACHE_WT; - else - xe_walk.cache = XE_CACHE_WB; } + if (!xe_vma_has_no_bo(vma) && xe_bo_is_stolen(bo)) xe_walk.dma_offset = xe_ttm_stolen_gpu_offset(xe_bo_device(bo)); diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index c33ae4db4e02..a97a310123fc 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -6,6 +6,7 @@ #include "xe_vm.h" #include +#include #include #include @@ -26,6 +27,7 @@ #include "xe_gt_pagefault.h" #include "xe_gt_tlb_invalidation.h" #include "xe_migrate.h" +#include "xe_pat.h" #include "xe_pm.h" #include "xe_preempt_fence.h" #include "xe_pt.h" @@ -868,7 +870,8 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, u64 start, u64 end, bool read_only, bool is_null, - u8 tile_mask) + u8 tile_mask, + u16 pat_index) { struct xe_vma *vma; struct xe_tile *tile; @@ -910,6 +913,8 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, if (GRAPHICS_VER(vm->xe) >= 20 || vm->xe->info.platform == XE_PVC) vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; + vma->pat_index = pat_index; + if (bo) { struct drm_gpuvm_bo *vm_bo; @@ -2162,7 +2167,7 @@ static struct drm_gpuva_ops * vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, u64 bo_offset_or_userptr, u64 addr, u64 range, u32 operation, u32 flags, u8 tile_mask, - u32 prefetch_region) + u32 prefetch_region, u16 pat_index) { struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; struct drm_gpuva_ops *ops; @@ -2231,6 +2236,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, op->map.read_only = flags & DRM_XE_VM_BIND_FLAG_READONLY; op->map.is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; + op->map.pat_index = pat_index; } else if (__op->op == DRM_GPUVA_OP_PREFETCH) { op->prefetch.region = prefetch_region; } @@ -2242,7 +2248,8 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, } static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, - u8 tile_mask, bool read_only, bool is_null) + u8 tile_mask, bool read_only, bool is_null, + u16 pat_index) { struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL; struct xe_vma *vma; @@ -2258,7 +2265,7 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, vma = xe_vma_create(vm, bo, op->gem.offset, op->va.addr, op->va.addr + op->va.range - 1, read_only, is_null, - tile_mask); + tile_mask, pat_index); if (bo) xe_bo_unlock(bo); @@ -2404,7 +2411,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, vma = new_vma(vm, &op->base.map, op->tile_mask, op->map.read_only, - op->map.is_null); + op->map.is_null, op->map.pat_index); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -2430,7 +2437,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, vma = new_vma(vm, op->base.remap.prev, op->tile_mask, read_only, - is_null); + is_null, old->pat_index); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -2464,7 +2471,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, vma = new_vma(vm, op->base.remap.next, op->tile_mask, read_only, - is_null); + is_null, old->pat_index); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -2862,6 +2869,26 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, u64 obj_offset = (*bind_ops)[i].obj_offset; u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance; bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; + u16 pat_index = (*bind_ops)[i].pat_index; + u16 coh_mode; + + if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) { + err = -EINVAL; + goto free_bind_ops; + } + + pat_index = array_index_nospec(pat_index, xe->pat.n_entries); + (*bind_ops)[i].pat_index = pat_index; + coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); + if (XE_IOCTL_DBG(xe, !coh_mode)) { /* hw reserved */ + err = -EINVAL; + goto free_bind_ops; + } + + if (XE_WARN_ON(coh_mode > XE_COH_AT_LEAST_1WAY)) { + err = -EINVAL; + goto free_bind_ops; + } if (i == 0) { *async = !!(flags & DRM_XE_VM_BIND_FLAG_ASYNC); @@ -2892,6 +2919,8 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || XE_IOCTL_DBG(xe, obj && op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || + XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && + op == DRM_XE_VM_BIND_OP_MAP_USERPTR) || XE_IOCTL_DBG(xe, obj && op == DRM_XE_VM_BIND_OP_PREFETCH) || XE_IOCTL_DBG(xe, prefetch_region && @@ -3025,6 +3054,8 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) u64 addr = bind_ops[i].addr; u32 obj = bind_ops[i].obj; u64 obj_offset = bind_ops[i].obj_offset; + u16 pat_index = bind_ops[i].pat_index; + u16 coh_mode; if (!obj) continue; @@ -3052,6 +3083,24 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto put_obj; } } + + coh_mode = xe_pat_index_get_coh_mode(xe, pat_index); + if (bos[i]->cpu_caching) { + if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE && + bos[i]->cpu_caching == DRM_XE_GEM_CPU_CACHING_WB)) { + err = -EINVAL; + goto put_obj; + } + } else if (XE_IOCTL_DBG(xe, coh_mode == XE_COH_NONE)) { + /* + * Imported dma-buf from a different device should + * require 1way or 2way coherency since we don't know + * how it was mapped on the CPU. Just assume is it + * potentially cached on CPU side. + */ + err = -EINVAL; + goto put_obj; + } } if (args->num_syncs) { @@ -3079,10 +3128,12 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) u64 obj_offset = bind_ops[i].obj_offset; u8 tile_mask = bind_ops[i].tile_mask; u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance; + u16 pat_index = bind_ops[i].pat_index; ops[i] = vm_bind_ioctl_ops_create(vm, bos[i], obj_offset, addr, range, op, flags, - tile_mask, prefetch_region); + tile_mask, prefetch_region, + pat_index); if (IS_ERR(ops[i])) { err = PTR_ERR(ops[i]); ops[i] = NULL; diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index fc2645e07578..74cdf16a42ad 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -110,6 +110,11 @@ struct xe_vma { */ u8 tile_present; + /** + * @pat_index: The pat index to use when encoding the PTEs for this vma. + */ + u16 pat_index; + struct { struct list_head rebind_link; } notifier; @@ -333,6 +338,8 @@ struct xe_vma_op_map { bool read_only; /** @is_null: is NULL binding */ bool is_null; + /** @pat_index: The pat index to use for this operation. */ + u16 pat_index; }; /** struct xe_vma_op_remap - VMA remap operation */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index ab7d1b26c773..1a844fa7af8a 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -636,8 +636,54 @@ struct drm_xe_vm_bind_op { */ __u32 obj; + /** + * @pat_index: The platform defined @pat_index to use for this mapping. + * The index basically maps to some predefined memory attributes, + * including things like caching, coherency, compression etc. The exact + * meaning of the pat_index is platform specific and defined in the + * Bspec and PRMs. When the KMD sets up the binding the index here is + * encoded into the ppGTT PTE. + * + * For coherency the @pat_index needs to be at least 1way coherent when + * drm_xe_gem_create.cpu_caching is DRM_XE_GEM_CPU_CACHING_WB. The KMD + * will extract the coherency mode from the @pat_index and reject if + * there is a mismatch (see note below for pre-MTL platforms). + * + * Note: On pre-MTL platforms there is only a caching mode and no + * explicit coherency mode, but on such hardware there is always a + * shared-LLC (or is dgpu) so all GT memory accesses are coherent with + * CPU caches even with the caching mode set as uncached. It's only the + * display engine that is incoherent (on dgpu it must be in VRAM which + * is always mapped as WC on the CPU). However to keep the uapi somewhat + * consistent with newer platforms the KMD groups the different cache + * levels into the following coherency buckets on all pre-MTL platforms: + * + * ppGTT UC -> COH_NONE + * ppGTT WC -> COH_NONE + * ppGTT WT -> COH_NONE + * ppGTT WB -> COH_AT_LEAST_1WAY + * + * In practice UC/WC/WT should only ever used for scanout surfaces on + * such platforms (or perhaps in general for dma-buf if shared with + * another device) since it is only the display engine that is actually + * incoherent. Everything else should typically use WB given that we + * have a shared-LLC. On MTL+ this completely changes and the HW + * defines the coherency mode as part of the @pat_index, where + * incoherent GT access is possible. + * + * Note: For userptr and externally imported dma-buf the kernel expects + * either 1WAY or 2WAY for the @pat_index. + * + * For DRM_XE_VM_BIND_FLAG_NULL bindings there are no KMD restrictions + * on the @pat_index. For such mappings there is no actual memory being + * mapped (the address in the PTE is invalid), so the various PAT memory + * attributes likely do not apply. Simply leaving as zero is one + * option (still a valid pat_index). + */ + __u16 pat_index; + /** @pad: MBZ */ - __u32 pad; + __u16 pad; union { /** -- cgit v1.2.3 From c4ad3710f51e8f0f2e169315e07e9e0c62dcded3 Mon Sep 17 00:00:00 2001 From: Mika Kuoppala Date: Wed, 22 Nov 2023 14:38:20 +0000 Subject: drm/xe: Extend drm_xe_vm_bind_op MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bind api is extensible but for a single bind op, there is not a mechanism to extend. Add extensions field to struct drm_xe_vm_bind_op. Cc: Rodrigo Vivi Cc: Matthew Brost Cc: Lucas De Marchi Cc: Francois Dugast Cc: Joonas Lahtinen Cc: Dominik Grzegorzek Signed-off-by: Mika Kuoppala Signed-off-by: Francois Dugast Reviewed-by: Rodrigo Vivi Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 3 +++ 1 file changed, 3 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 1a844fa7af8a..4c906ff2429e 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -631,6 +631,9 @@ struct drm_xe_vm_destroy { }; struct drm_xe_vm_bind_op { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + /** * @obj: GEM object to operate on, MBZ for MAP_USERPTR, MBZ for UNMAP */ -- cgit v1.2.3 From 6b8c1edc4f698d7e7e3cd5852bb5b20e93ab01b8 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 22 Nov 2023 14:38:21 +0000 Subject: drm/xe/uapi: Separate bo_create placement from flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although the flags are about the creation, the memory placement of the BO deserves a proper dedicated field in the uapi. Besides getting more clear, it also allows to remove the 'magic' shifts from the flags that was a concern during the uapi reviews. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matthew Brost Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_bo.c | 14 +++++++------- include/uapi/drm/xe_drm.h | 9 ++++++--- 2 files changed, 13 insertions(+), 10 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 5e3493f21b59..fd516ad7478c 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1890,15 +1890,15 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) return -EINVAL; + /* at least one valid memory placement must be specified */ + if (XE_IOCTL_DBG(xe, (args->placement & ~xe->info.mem_region_mask) || + !args->placement)) + return -EINVAL; + if (XE_IOCTL_DBG(xe, args->flags & ~(DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING | DRM_XE_GEM_CREATE_FLAG_SCANOUT | - DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM | - xe->info.mem_region_mask))) - return -EINVAL; - - /* at least one memory type must be specified */ - if (XE_IOCTL_DBG(xe, !(args->flags & xe->info.mem_region_mask))) + DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM))) return -EINVAL; if (XE_IOCTL_DBG(xe, args->handle)) @@ -1920,7 +1920,7 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, if (args->flags & DRM_XE_GEM_CREATE_FLAG_SCANOUT) bo_flags |= XE_BO_SCANOUT_BIT; - bo_flags |= args->flags << (ffs(XE_BO_CREATE_SYSTEM_BIT) - 1); + bo_flags |= args->placement << (ffs(XE_BO_CREATE_SYSTEM_BIT) - 1); if (args->flags & DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM) { if (XE_IOCTL_DBG(xe, !(bo_flags & XE_BO_CREATE_VRAM_MASK))) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 4c906ff2429e..6edbcd81c195 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -500,8 +500,11 @@ struct drm_xe_gem_create { */ __u64 size; -#define DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING (0x1 << 24) -#define DRM_XE_GEM_CREATE_FLAG_SCANOUT (0x1 << 25) + /** @placement: A mask of memory instances of where BO can be placed. */ + __u32 placement; + +#define DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING (1 << 0) +#define DRM_XE_GEM_CREATE_FLAG_SCANOUT (1 << 1) /* * When using VRAM as a possible placement, ensure that the corresponding VRAM * allocation will always use the CPU accessible part of VRAM. This is important @@ -517,7 +520,7 @@ struct drm_xe_gem_create { * display surfaces, therefore the kernel requires setting this flag for such * objects, otherwise an error is thrown on small-bar systems. */ -#define DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM (0x1 << 26) +#define DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM (1 << 2) /** * @flags: Flags, currently a mask of memory instances of where BO can * be placed -- cgit v1.2.3 From 2bec30715435824c2ea03714038f0ee7a4b5c698 Mon Sep 17 00:00:00 2001 From: José Roberto de Souza Date: Wed, 22 Nov 2023 14:38:22 +0000 Subject: drm/xe: Make DRM_XE_DEVICE_QUERY_ENGINES future proof MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have at least 2 future features(OA and future media engines capabilities) that will require Xe to provide more information about engines to UMDs. But this information should not just be added to drm_xe_engine_class_instance for a couple of reasons: - drm_xe_engine_class_instance is used as input to other structs/uAPIs and those uAPIs don't care about any of these future new engine fields - those new fields are useless information after initialization for some UMDs, so it should not need to carry that around So here my proposal is to make DRM_XE_DEVICE_QUERY_ENGINES return an array of drm_xe_query_engine_info that contain drm_xe_engine_class_instance and 3 u64s to be used for future features. Reference OA: https://patchwork.freedesktop.org/patch/558362/?series=121084&rev=6 v2: Reduce reserved[] to 3 u64 (Matthew Brost) Cc: Francois Dugast Cc: Rodrigo Vivi Signed-off-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi [Rodrigo Rebased] Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_query.c | 15 ++++++++------- include/uapi/drm/xe_drm.h | 27 +++++++++++++++++++++++++-- 2 files changed, 33 insertions(+), 9 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 61a7d92b7e88..0cbfeaeb1330 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -53,7 +53,7 @@ static size_t calc_hw_engine_info_size(struct xe_device *xe) i++; } - return i * sizeof(struct drm_xe_engine_class_instance); + return i * sizeof(struct drm_xe_query_engine_info); } typedef u64 (*__ktime_func_t)(void); @@ -186,9 +186,9 @@ static int query_engines(struct xe_device *xe, struct drm_xe_device_query *query) { size_t size = calc_hw_engine_info_size(xe); - struct drm_xe_engine_class_instance __user *query_ptr = + struct drm_xe_query_engine_info __user *query_ptr = u64_to_user_ptr(query->data); - struct drm_xe_engine_class_instance *hw_engine_info; + struct drm_xe_query_engine_info *hw_engine_info; struct xe_hw_engine *hwe; enum xe_hw_engine_id id; struct xe_gt *gt; @@ -211,12 +211,13 @@ static int query_engines(struct xe_device *xe, if (xe_hw_engine_is_reserved(hwe)) continue; - hw_engine_info[i].engine_class = + hw_engine_info[i].instance.engine_class = xe_to_user_engine_class[hwe->class]; - hw_engine_info[i].engine_instance = + hw_engine_info[i].instance.engine_instance = hwe->logical_instance; - hw_engine_info[i].gt_id = gt->info.id; - hw_engine_info[i].pad = 0; + hw_engine_info[i].instance.gt_id = gt->info.id; + hw_engine_info[i].instance.pad = 0; + memset(hw_engine_info->reserved, 0, sizeof(hw_engine_info->reserved)); i++; } diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 6edbcd81c195..dc657ae9db18 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -124,7 +124,13 @@ struct xe_user_extension { #define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property) #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) -/** struct drm_xe_engine_class_instance - instance of an engine class */ +/** + * struct drm_xe_engine_class_instance - instance of an engine class + * + * It is returned as part of the @drm_xe_query_engine_info, but it also is + * used as the input of engine selection for both @drm_xe_exec_queue_create + * and @drm_xe_query_engine_cycles + */ struct drm_xe_engine_class_instance { #define DRM_XE_ENGINE_CLASS_RENDER 0 #define DRM_XE_ENGINE_CLASS_COPY 1 @@ -137,14 +143,31 @@ struct drm_xe_engine_class_instance { */ #define DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC 5 #define DRM_XE_ENGINE_CLASS_VM_BIND_SYNC 6 + /** @engine_class: engine class id */ __u16 engine_class; - + /** @engine_instance: engine instance id */ __u16 engine_instance; + /** @gt_id: Unique ID of this GT within the PCI Device */ __u16 gt_id; /** @pad: MBZ */ __u16 pad; }; +/** + * struct drm_xe_query_engine_info - describe hardware engine + * + * If a query is made with a struct @drm_xe_device_query where .query + * is equal to %DRM_XE_DEVICE_QUERY_ENGINES, then the reply uses an array of + * struct @drm_xe_query_engine_info in .data. + */ +struct drm_xe_query_engine_info { + /** @instance: The @drm_xe_engine_class_instance */ + struct drm_xe_engine_class_instance instance; + + /** @reserved: Reserved */ + __u64 reserved[3]; +}; + /** * enum drm_xe_memory_class - Supported memory classes. */ -- cgit v1.2.3 From 4e03b584143e18eabd091061a1716515da928dcb Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 22 Nov 2023 14:38:23 +0000 Subject: drm/xe/uapi: Reject bo creation of unaligned size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For xe bo creation we request passing size which matches system or vram minimum page alignment. This way we want to ensure userspace is aware of region constraints and not aligned allocations will be rejected returning EINVAL. v2: - Rebase, Update uAPI documentation. (Thomas) v3: - Adjust the dma-buf kunit test accordingly. (Thomas) v4: - Fixed rebase conflicts and updated commit message. (Francois) Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Zbigniew Kempczyński Signed-off-by: Thomas Hellström Reviewed-by: Maarten Lankhorst Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/tests/xe_dma_buf.c | 10 ++++++++-- drivers/gpu/drm/xe/xe_bo.c | 26 +++++++++++++++++--------- include/uapi/drm/xe_drm.h | 17 +++++++++-------- 3 files changed, 34 insertions(+), 19 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c index 81f12422a587..bb6f6424e06f 100644 --- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c @@ -109,15 +109,21 @@ static void xe_test_dmabuf_import_same_driver(struct xe_device *xe) struct drm_gem_object *import; struct dma_buf *dmabuf; struct xe_bo *bo; + size_t size; /* No VRAM on this device? */ if (!ttm_manager_type(&xe->ttm, XE_PL_VRAM0) && (params->mem_mask & XE_BO_CREATE_VRAM0_BIT)) return; + size = PAGE_SIZE; + if ((params->mem_mask & XE_BO_CREATE_VRAM0_BIT) && + xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) + size = SZ_64K; + kunit_info(test, "running %s\n", __func__); - bo = xe_bo_create_user(xe, NULL, NULL, PAGE_SIZE, DRM_XE_GEM_CPU_CACHING_WC, - ttm_bo_type_device, params->mem_mask); + bo = xe_bo_create_user(xe, NULL, NULL, size, DRM_XE_GEM_CPU_CACHING_WC, + ttm_bo_type_device, XE_BO_CREATE_USER_BIT | params->mem_mask); if (IS_ERR(bo)) { KUNIT_FAIL(test, "xe_bo_create() failed with err=%ld\n", PTR_ERR(bo)); diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index fd516ad7478c..0bd1b3581945 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1222,6 +1222,7 @@ struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, }; struct ttm_placement *placement; uint32_t alignment; + size_t aligned_size; int err; /* Only kernel objects should set GT */ @@ -1232,23 +1233,30 @@ struct xe_bo *___xe_bo_create_locked(struct xe_device *xe, struct xe_bo *bo, return ERR_PTR(-EINVAL); } - if (!bo) { - bo = xe_bo_alloc(); - if (IS_ERR(bo)) - return bo; - } - if (flags & (XE_BO_CREATE_VRAM_MASK | XE_BO_CREATE_STOLEN_BIT) && !(flags & XE_BO_CREATE_IGNORE_MIN_PAGE_SIZE_BIT) && xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K) { - size = ALIGN(size, SZ_64K); + aligned_size = ALIGN(size, SZ_64K); + if (type != ttm_bo_type_device) + size = ALIGN(size, SZ_64K); flags |= XE_BO_INTERNAL_64K; alignment = SZ_64K >> PAGE_SHIFT; + } else { - size = ALIGN(size, PAGE_SIZE); + aligned_size = ALIGN(size, SZ_4K); + flags &= ~XE_BO_INTERNAL_64K; alignment = SZ_4K >> PAGE_SHIFT; } + if (type == ttm_bo_type_device && aligned_size != size) + return ERR_PTR(-EINVAL); + + if (!bo) { + bo = xe_bo_alloc(); + if (IS_ERR(bo)) + return bo; + } + bo->tile = tile; bo->size = size; bo->flags = flags; @@ -1566,7 +1574,7 @@ struct xe_bo *xe_managed_bo_create_pin_map(struct xe_device *xe, struct xe_tile struct xe_bo *xe_managed_bo_create_from_data(struct xe_device *xe, struct xe_tile *tile, const void *data, size_t size, u32 flags) { - struct xe_bo *bo = xe_managed_bo_create_pin_map(xe, tile, size, flags); + struct xe_bo *bo = xe_managed_bo_create_pin_map(xe, tile, ALIGN(size, PAGE_SIZE), flags); if (IS_ERR(bo)) return bo; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index dc657ae9db18..d7918f6e760f 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -207,11 +207,13 @@ struct drm_xe_query_mem_region { * * When the kernel allocates memory for this region, the * underlying pages will be at least @min_page_size in size. - * - * Important note: When userspace allocates a GTT address which - * can point to memory allocated from this region, it must also - * respect this minimum alignment. This is enforced by the - * kernel. + * Buffer objects with an allowable placement in this region must be + * created with a size aligned to this value. + * GPU virtual address mappings of (parts of) buffer objects that + * may be placed in this region must also have their GPU virtual + * address and range aligned to this value. + * Affected IOCTLS will return %-EINVAL if alignment restrictions are + * not met. */ __u32 min_page_size; /** @@ -517,9 +519,8 @@ struct drm_xe_gem_create { __u64 extensions; /** - * @size: Requested size for the object - * - * The (page-aligned) allocated size for the object will be returned. + * @size: Size of the object to be created, must match region + * (system or vram) minimum alignment (&min_page_size). */ __u64 size; -- cgit v1.2.3 From 4bc9dd98e0a7e8a14386fc8341379ee09e594987 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 22 Nov 2023 14:38:24 +0000 Subject: drm/xe/uapi: Align on a common way to return arrays (memory regions) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The uAPI provides queries which return arrays of elements. As of now the format used in the struct is different depending on which element is queried. Fix this for memory regions by applying the pattern below: struct drm_xe_query_Xs { __u32 num_Xs; struct drm_xe_X Xs[]; ... } This removes "query" in the name of struct drm_xe_query_mem_region as it is not returned from the query IOCTL. There is no functional change. v2: Only rename drm_xe_query_mem_region to drm_xe_mem_region (José Roberto de Souza) v3: Rename usage to mem_regions in xe_query.c (José Roberto de Souza) Signed-off-by: Francois Dugast Reviewed-by: Rodrigo Vivi Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_query.c | 46 ++++++++++++++++++++++--------------------- include/uapi/drm/xe_drm.h | 12 +++++------ 2 files changed, 30 insertions(+), 28 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 0cbfeaeb1330..34474f8b97f6 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -240,14 +240,14 @@ static size_t calc_mem_regions_size(struct xe_device *xe) if (ttm_manager_type(&xe->ttm, i)) num_managers++; - return offsetof(struct drm_xe_query_mem_regions, regions[num_managers]); + return offsetof(struct drm_xe_query_mem_regions, mem_regions[num_managers]); } static int query_mem_regions(struct xe_device *xe, - struct drm_xe_device_query *query) + struct drm_xe_device_query *query) { size_t size = calc_mem_regions_size(xe); - struct drm_xe_query_mem_regions *usage; + struct drm_xe_query_mem_regions *mem_regions; struct drm_xe_query_mem_regions __user *query_ptr = u64_to_user_ptr(query->data); struct ttm_resource_manager *man; @@ -260,50 +260,52 @@ static int query_mem_regions(struct xe_device *xe, return -EINVAL; } - usage = kzalloc(size, GFP_KERNEL); - if (XE_IOCTL_DBG(xe, !usage)) + mem_regions = kzalloc(size, GFP_KERNEL); + if (XE_IOCTL_DBG(xe, !mem_regions)) return -ENOMEM; man = ttm_manager_type(&xe->ttm, XE_PL_TT); - usage->regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; - usage->regions[0].instance = 0; - usage->regions[0].min_page_size = PAGE_SIZE; - usage->regions[0].total_size = man->size << PAGE_SHIFT; + mem_regions->mem_regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; + mem_regions->mem_regions[0].instance = 0; + mem_regions->mem_regions[0].min_page_size = PAGE_SIZE; + mem_regions->mem_regions[0].total_size = man->size << PAGE_SHIFT; if (perfmon_capable()) - usage->regions[0].used = ttm_resource_manager_usage(man); - usage->num_regions = 1; + mem_regions->mem_regions[0].used = ttm_resource_manager_usage(man); + mem_regions->num_mem_regions = 1; for (i = XE_PL_VRAM0; i <= XE_PL_VRAM1; ++i) { man = ttm_manager_type(&xe->ttm, i); if (man) { - usage->regions[usage->num_regions].mem_class = + mem_regions->mem_regions[mem_regions->num_mem_regions].mem_class = DRM_XE_MEM_REGION_CLASS_VRAM; - usage->regions[usage->num_regions].instance = - usage->num_regions; - usage->regions[usage->num_regions].min_page_size = + mem_regions->mem_regions[mem_regions->num_mem_regions].instance = + mem_regions->num_mem_regions; + mem_regions->mem_regions[mem_regions->num_mem_regions].min_page_size = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : PAGE_SIZE; - usage->regions[usage->num_regions].total_size = + mem_regions->mem_regions[mem_regions->num_mem_regions].total_size = man->size; if (perfmon_capable()) { xe_ttm_vram_get_used(man, - &usage->regions[usage->num_regions].used, - &usage->regions[usage->num_regions].cpu_visible_used); + &mem_regions->mem_regions + [mem_regions->num_mem_regions].used, + &mem_regions->mem_regions + [mem_regions->num_mem_regions].cpu_visible_used); } - usage->regions[usage->num_regions].cpu_visible_size = + mem_regions->mem_regions[mem_regions->num_mem_regions].cpu_visible_size = xe_ttm_vram_get_cpu_visible_size(man); - usage->num_regions++; + mem_regions->num_mem_regions++; } } - if (!copy_to_user(query_ptr, usage, size)) + if (!copy_to_user(query_ptr, mem_regions, size)) ret = 0; else ret = -ENOSPC; - kfree(usage); + kfree(mem_regions); return ret; } diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index d7918f6e760f..863963168dc3 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -183,10 +183,10 @@ enum drm_xe_memory_class { }; /** - * struct drm_xe_query_mem_region - Describes some region as known to + * struct drm_xe_mem_region - Describes some region as known to * the driver. */ -struct drm_xe_query_mem_region { +struct drm_xe_mem_region { /** * @mem_class: The memory class describing this region. * @@ -323,12 +323,12 @@ struct drm_xe_query_engine_cycles { * struct drm_xe_query_mem_regions in .data. */ struct drm_xe_query_mem_regions { - /** @num_regions: number of memory regions returned in @regions */ - __u32 num_regions; + /** @num_mem_regions: number of memory regions returned in @mem_regions */ + __u32 num_mem_regions; /** @pad: MBZ */ __u32 pad; - /** @regions: The returned regions for this device */ - struct drm_xe_query_mem_region regions[]; + /** @mem_regions: The returned memory regions for this device */ + struct drm_xe_mem_region mem_regions[]; }; /** -- cgit v1.2.3 From 71c625aa770d4bd2b0901a9da3820fb89636e1a1 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 22 Nov 2023 14:38:25 +0000 Subject: drm/xe/uapi: Align on a common way to return arrays (gt) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The uAPI provides queries which return arrays of elements. As of now the format used in the struct is different depending on which element is queried. However, aligning on the new common pattern: struct drm_xe_query_Xs { __u32 num_Xs; struct drm_xe_X Xs[]; ... } ... would mean bringing back the name "gts" which is avoided per commit fca54ba12470 ("drm/xe/uapi: Rename gts to gt_list") so make an exception for gt and leave gt_list. Also, this change removes "query" in the name of struct drm_xe_query_gt as it is not returned from the query IOCTL. There is no functional change. v2: Leave gt_list (Matt Roper) Signed-off-by: Francois Dugast Reviewed-by: Rodrigo Vivi Reviewed-by: Matt Roper Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_query.c | 2 +- include/uapi/drm/xe_drm.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 34474f8b97f6..a0e3b0c163f9 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -354,7 +354,7 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query { struct xe_gt *gt; size_t size = sizeof(struct drm_xe_query_gt_list) + - xe->info.gt_count * sizeof(struct drm_xe_query_gt); + xe->info.gt_count * sizeof(struct drm_xe_gt); struct drm_xe_query_gt_list __user *query_ptr = u64_to_user_ptr(query->data); struct drm_xe_query_gt_list *gt_list; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 863963168dc3..a8ae845d0c74 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -357,14 +357,14 @@ struct drm_xe_query_config { }; /** - * struct drm_xe_query_gt - describe an individual GT. + * struct drm_xe_gt - describe an individual GT. * * To be used with drm_xe_query_gt_list, which will return a list with all the * existing GT individual descriptions. * Graphics Technology (GT) is a subset of a GPU/tile that is responsible for * implementing graphics and/or media operations. */ -struct drm_xe_query_gt { +struct drm_xe_gt { #define DRM_XE_QUERY_GT_TYPE_MAIN 0 #define DRM_XE_QUERY_GT_TYPE_MEDIA 1 /** @type: GT type: Main or Media */ @@ -404,7 +404,7 @@ struct drm_xe_query_gt_list { /** @pad: MBZ */ __u32 pad; /** @gt_list: The GT list returned for this device */ - struct drm_xe_query_gt gt_list[]; + struct drm_xe_gt gt_list[]; }; /** -- cgit v1.2.3 From 60a6a849fcb338b8a3f3d1ec9ec50c002add925a Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Wed, 22 Nov 2023 14:38:26 +0000 Subject: drm/xe/uapi: Align on a common way to return arrays (engines) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The uAPI provides queries which return arrays of elements. As of now the format used in the struct is different depending on which element is queried. Fix this for engines by applying the pattern below: struct drm_xe_query_Xs { __u32 num_Xs; struct drm_xe_X Xs[]; ... } Instead of directly returning an array of struct drm_xe_query_engine_info, a new struct drm_xe_query_engines is introduced. It contains itself an array of struct drm_xe_engine which holds the information about each engine. v2: Use plural for struct drm_xe_query_engines as multiple engines are returned (José Roberto de Souza) Signed-off-by: Francois Dugast Reviewed-by: Rodrigo Vivi Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_query.c | 31 +++++++++-------- include/uapi/drm/xe_drm.h | 78 ++++++++++++++++++++++++++----------------- 2 files changed, 65 insertions(+), 44 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index a0e3b0c163f9..ad9f23e43920 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -53,7 +53,8 @@ static size_t calc_hw_engine_info_size(struct xe_device *xe) i++; } - return i * sizeof(struct drm_xe_query_engine_info); + return sizeof(struct drm_xe_query_engines) + + i * sizeof(struct drm_xe_engine); } typedef u64 (*__ktime_func_t)(void); @@ -186,9 +187,9 @@ static int query_engines(struct xe_device *xe, struct drm_xe_device_query *query) { size_t size = calc_hw_engine_info_size(xe); - struct drm_xe_query_engine_info __user *query_ptr = + struct drm_xe_query_engines __user *query_ptr = u64_to_user_ptr(query->data); - struct drm_xe_query_engine_info *hw_engine_info; + struct drm_xe_query_engines *engines; struct xe_hw_engine *hwe; enum xe_hw_engine_id id; struct xe_gt *gt; @@ -202,8 +203,8 @@ static int query_engines(struct xe_device *xe, return -EINVAL; } - hw_engine_info = kmalloc(size, GFP_KERNEL); - if (!hw_engine_info) + engines = kmalloc(size, GFP_KERNEL); + if (!engines) return -ENOMEM; for_each_gt(gt, xe, gt_id) @@ -211,22 +212,26 @@ static int query_engines(struct xe_device *xe, if (xe_hw_engine_is_reserved(hwe)) continue; - hw_engine_info[i].instance.engine_class = + engines->engines[i].instance.engine_class = xe_to_user_engine_class[hwe->class]; - hw_engine_info[i].instance.engine_instance = + engines->engines[i].instance.engine_instance = hwe->logical_instance; - hw_engine_info[i].instance.gt_id = gt->info.id; - hw_engine_info[i].instance.pad = 0; - memset(hw_engine_info->reserved, 0, sizeof(hw_engine_info->reserved)); + engines->engines[i].instance.gt_id = gt->info.id; + engines->engines[i].instance.pad = 0; + memset(engines->engines[i].reserved, 0, + sizeof(engines->engines[i].reserved)); i++; } - if (copy_to_user(query_ptr, hw_engine_info, size)) { - kfree(hw_engine_info); + engines->pad = 0; + engines->num_engines = i; + + if (copy_to_user(query_ptr, engines, size)) { + kfree(engines); return -EFAULT; } - kfree(hw_engine_info); + kfree(engines); return 0; } diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index a8ae845d0c74..2e58ddcf92f5 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -127,9 +127,9 @@ struct xe_user_extension { /** * struct drm_xe_engine_class_instance - instance of an engine class * - * It is returned as part of the @drm_xe_query_engine_info, but it also is - * used as the input of engine selection for both @drm_xe_exec_queue_create - * and @drm_xe_query_engine_cycles + * It is returned as part of the @drm_xe_engine, but it also is used as + * the input of engine selection for both @drm_xe_exec_queue_create and + * @drm_xe_query_engine_cycles */ struct drm_xe_engine_class_instance { #define DRM_XE_ENGINE_CLASS_RENDER 0 @@ -154,13 +154,9 @@ struct drm_xe_engine_class_instance { }; /** - * struct drm_xe_query_engine_info - describe hardware engine - * - * If a query is made with a struct @drm_xe_device_query where .query - * is equal to %DRM_XE_DEVICE_QUERY_ENGINES, then the reply uses an array of - * struct @drm_xe_query_engine_info in .data. + * struct drm_xe_engine - describe hardware engine */ -struct drm_xe_query_engine_info { +struct drm_xe_engine { /** @instance: The @drm_xe_engine_class_instance */ struct drm_xe_engine_class_instance instance; @@ -168,6 +164,22 @@ struct drm_xe_query_engine_info { __u64 reserved[3]; }; +/** + * struct drm_xe_query_engines - describe engines + * + * If a query is made with a struct @drm_xe_device_query where .query + * is equal to %DRM_XE_DEVICE_QUERY_ENGINES, then the reply uses an array of + * struct @drm_xe_query_engines in .data. + */ +struct drm_xe_query_engines { + /** @num_engines: number of engines returned in @engines */ + __u32 num_engines; + /** @pad: MBZ */ + __u32 pad; + /** @engines: The returned engines for this device */ + struct drm_xe_engine engines[]; +}; + /** * enum drm_xe_memory_class - Supported memory classes. */ @@ -467,28 +479,32 @@ struct drm_xe_query_topology_mask { * * .. code-block:: C * - * struct drm_xe_engine_class_instance *hwe; - * struct drm_xe_device_query query = { - * .extensions = 0, - * .query = DRM_XE_DEVICE_QUERY_ENGINES, - * .size = 0, - * .data = 0, - * }; - * ioctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query); - * hwe = malloc(query.size); - * query.data = (uintptr_t)hwe; - * ioctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query); - * int num_engines = query.size / sizeof(*hwe); - * for (int i = 0; i < num_engines; i++) { - * printf("Engine %d: %s\n", i, - * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_RENDER ? "RENDER": - * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_COPY ? "COPY": - * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_VIDEO_DECODE ? "VIDEO_DECODE": - * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE ? "VIDEO_ENHANCE": - * hwe[i].engine_class == DRM_XE_ENGINE_CLASS_COMPUTE ? "COMPUTE": - * "UNKNOWN"); - * } - * free(hwe); + * struct drm_xe_query_engines *engines; + * struct drm_xe_device_query query = { + * .extensions = 0, + * .query = DRM_XE_DEVICE_QUERY_ENGINES, + * .size = 0, + * .data = 0, + * }; + * ioctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query); + * engines = malloc(query.size); + * query.data = (uintptr_t)engines; + * ioctl(fd, DRM_IOCTL_XE_DEVICE_QUERY, &query); + * for (int i = 0; i < engines->num_engines; i++) { + * printf("Engine %d: %s\n", i, + * engines->engines[i].instance.engine_class == + * DRM_XE_ENGINE_CLASS_RENDER ? "RENDER": + * engines->engines[i].instance.engine_class == + * DRM_XE_ENGINE_CLASS_COPY ? "COPY": + * engines->engines[i].instance.engine_class == + * DRM_XE_ENGINE_CLASS_VIDEO_DECODE ? "VIDEO_DECODE": + * engines->engines[i].instance.engine_class == + * DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE ? "VIDEO_ENHANCE": + * engines->engines[i].instance.engine_class == + * DRM_XE_ENGINE_CLASS_COMPUTE ? "COMPUTE": + * "UNKNOWN"); + * } + * free(engines); */ struct drm_xe_device_query { /** @extensions: Pointer to the first extension struct, if any */ -- cgit v1.2.3 From 37d078e51b4cba30f90667a2b35e16725d649956 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 22 Nov 2023 14:38:27 +0000 Subject: drm/xe/uapi: Split xe_sync types from flags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's continue on the uapi clean-up with more splits with stuff into their own exclusive fields instead of reusing stuff. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matthew Brost Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_sync.c | 23 +++++++---------------- drivers/gpu/drm/xe/xe_sync_types.h | 1 + include/uapi/drm/xe_drm.h | 16 ++++++++-------- 3 files changed, 16 insertions(+), 24 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_sync.c b/drivers/gpu/drm/xe/xe_sync.c index ea96ba4b41da..936227e79483 100644 --- a/drivers/gpu/drm/xe/xe_sync.c +++ b/drivers/gpu/drm/xe/xe_sync.c @@ -17,8 +17,6 @@ #include "xe_macros.h" #include "xe_sched_job_types.h" -#define SYNC_FLAGS_TYPE_MASK 0x3 - struct user_fence { struct xe_device *xe; struct kref refcount; @@ -109,15 +107,13 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, if (copy_from_user(&sync_in, sync_user, sizeof(*sync_user))) return -EFAULT; - if (XE_IOCTL_DBG(xe, sync_in.flags & - ~(SYNC_FLAGS_TYPE_MASK | DRM_XE_SYNC_FLAG_SIGNAL)) || - XE_IOCTL_DBG(xe, sync_in.pad) || + if (XE_IOCTL_DBG(xe, sync_in.flags & ~DRM_XE_SYNC_FLAG_SIGNAL) || XE_IOCTL_DBG(xe, sync_in.reserved[0] || sync_in.reserved[1])) return -EINVAL; signal = sync_in.flags & DRM_XE_SYNC_FLAG_SIGNAL; - switch (sync_in.flags & SYNC_FLAGS_TYPE_MASK) { - case DRM_XE_SYNC_FLAG_SYNCOBJ: + switch (sync_in.type) { + case DRM_XE_SYNC_TYPE_SYNCOBJ: if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) return -EOPNOTSUPP; @@ -135,7 +131,7 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, } break; - case DRM_XE_SYNC_FLAG_TIMELINE_SYNCOBJ: + case DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ: if (XE_IOCTL_DBG(xe, in_lr_mode && signal)) return -EOPNOTSUPP; @@ -165,12 +161,7 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, } break; - case DRM_XE_SYNC_FLAG_DMA_BUF: - if (XE_IOCTL_DBG(xe, "TODO")) - return -EINVAL; - break; - - case DRM_XE_SYNC_FLAG_USER_FENCE: + case DRM_XE_SYNC_TYPE_USER_FENCE: if (XE_IOCTL_DBG(xe, !signal)) return -EOPNOTSUPP; @@ -192,6 +183,7 @@ int xe_sync_entry_parse(struct xe_device *xe, struct xe_file *xef, return -EINVAL; } + sync->type = sync_in.type; sync->flags = sync_in.flags; sync->timeline_value = sync_in.timeline_value; @@ -252,8 +244,7 @@ void xe_sync_entry_signal(struct xe_sync_entry *sync, struct xe_sched_job *job, user_fence_put(sync->ufence); dma_fence_put(fence); } - } else if ((sync->flags & SYNC_FLAGS_TYPE_MASK) == - DRM_XE_SYNC_FLAG_USER_FENCE) { + } else if (sync->type == DRM_XE_SYNC_TYPE_USER_FENCE) { job->user_fence.used = true; job->user_fence.addr = sync->addr; job->user_fence.value = sync->timeline_value; diff --git a/drivers/gpu/drm/xe/xe_sync_types.h b/drivers/gpu/drm/xe/xe_sync_types.h index 24fccc26cb53..852db5e7884f 100644 --- a/drivers/gpu/drm/xe/xe_sync_types.h +++ b/drivers/gpu/drm/xe/xe_sync_types.h @@ -21,6 +21,7 @@ struct xe_sync_entry { struct user_fence *ufence; u64 addr; u64 timeline_value; + u32 type; u32 flags; }; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 2e58ddcf92f5..978fca7bb235 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -947,16 +947,16 @@ struct drm_xe_sync { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; -#define DRM_XE_SYNC_FLAG_SYNCOBJ 0x0 -#define DRM_XE_SYNC_FLAG_TIMELINE_SYNCOBJ 0x1 -#define DRM_XE_SYNC_FLAG_DMA_BUF 0x2 -#define DRM_XE_SYNC_FLAG_USER_FENCE 0x3 -#define DRM_XE_SYNC_FLAG_SIGNAL 0x10 +#define DRM_XE_SYNC_TYPE_SYNCOBJ 0x0 +#define DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ 0x1 +#define DRM_XE_SYNC_TYPE_USER_FENCE 0x2 + /** @type: Type of the this sync object */ + __u32 type; + +#define DRM_XE_SYNC_FLAG_SIGNAL (1 << 0) + /** @flags: Sync Flags */ __u32 flags; - /** @pad: MBZ */ - __u32 pad; - union { __u32 handle; -- cgit v1.2.3 From cad4a0d6af146e14a82a0f7d43613450dc56ff80 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 22 Nov 2023 14:38:28 +0000 Subject: drm/xe/uapi: Kill tile_mask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is currently unused, so by the rules it cannot go upstream. Also there was the desire to convert that to align with the engine_class_instance selection, but the consensus on that one is to remain with the global gt_id. So we are keeping the gt_id there, not converting to a generic sched_group and also killing this tile_mask and only using the default behavior of 0 that is to create a mapping / page_table entry on every tile, similar to what i915. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: Matthew Brost Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_vm.c | 40 +++++++++------------------------------- drivers/gpu/drm/xe/xe_vm_types.h | 2 -- include/uapi/drm/xe_drm.h | 8 +------- 3 files changed, 10 insertions(+), 40 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index a97a310123fc..ff22eddc2578 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -870,7 +870,6 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, u64 start, u64 end, bool read_only, bool is_null, - u8 tile_mask, u16 pat_index) { struct xe_vma *vma; @@ -903,12 +902,8 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, if (is_null) vma->gpuva.flags |= DRM_GPUVA_SPARSE; - if (tile_mask) { - vma->tile_mask = tile_mask; - } else { - for_each_tile(tile, vm->xe, id) - vma->tile_mask |= 0x1 << id; - } + for_each_tile(tile, vm->xe, id) + vma->tile_mask |= 0x1 << id; if (GRAPHICS_VER(vm->xe) >= 20 || vm->xe->info.platform == XE_PVC) vma->gpuva.flags |= XE_VMA_ATOMIC_PTE_BIT; @@ -2166,7 +2161,7 @@ static void print_op(struct xe_device *xe, struct drm_gpuva_op *op) static struct drm_gpuva_ops * vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, u64 bo_offset_or_userptr, u64 addr, u64 range, - u32 operation, u32 flags, u8 tile_mask, + u32 operation, u32 flags, u32 prefetch_region, u16 pat_index) { struct drm_gem_object *obj = bo ? &bo->ttm.base : NULL; @@ -2229,7 +2224,6 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, drm_gpuva_for_each_op(__op, ops) { struct xe_vma_op *op = gpuva_op_to_vma_op(__op); - op->tile_mask = tile_mask; if (__op->op == DRM_GPUVA_OP_MAP) { op->map.immediate = flags & DRM_XE_VM_BIND_FLAG_IMMEDIATE; @@ -2248,8 +2242,7 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, } static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, - u8 tile_mask, bool read_only, bool is_null, - u16 pat_index) + bool read_only, bool is_null, u16 pat_index) { struct xe_bo *bo = op->gem.obj ? gem_to_xe_bo(op->gem.obj) : NULL; struct xe_vma *vma; @@ -2265,7 +2258,7 @@ static struct xe_vma *new_vma(struct xe_vm *vm, struct drm_gpuva_op_map *op, vma = xe_vma_create(vm, bo, op->gem.offset, op->va.addr, op->va.addr + op->va.range - 1, read_only, is_null, - tile_mask, pat_index); + pat_index); if (bo) xe_bo_unlock(bo); @@ -2409,8 +2402,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, { struct xe_vma *vma; - vma = new_vma(vm, &op->base.map, - op->tile_mask, op->map.read_only, + vma = new_vma(vm, &op->base.map, op->map.read_only, op->map.is_null, op->map.pat_index); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -2435,8 +2427,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, op->base.remap.unmap->va->flags & DRM_GPUVA_SPARSE; - vma = new_vma(vm, op->base.remap.prev, - op->tile_mask, read_only, + vma = new_vma(vm, op->base.remap.prev, read_only, is_null, old->pat_index); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -2469,8 +2460,7 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, op->base.remap.unmap->va->flags & DRM_GPUVA_SPARSE; - vma = new_vma(vm, op->base.remap.next, - op->tile_mask, read_only, + vma = new_vma(vm, op->base.remap.next, read_only, is_null, old->pat_index); if (IS_ERR(vma)) return PTR_ERR(vma); @@ -3024,16 +3014,6 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) err = -EINVAL; goto release_vm_lock; } - - if (bind_ops[i].tile_mask) { - u64 valid_tiles = BIT(xe->info.tile_count) - 1; - - if (XE_IOCTL_DBG(xe, bind_ops[i].tile_mask & - ~valid_tiles)) { - err = -EINVAL; - goto release_vm_lock; - } - } } bos = kzalloc(sizeof(*bos) * args->num_binds, GFP_KERNEL); @@ -3126,14 +3106,12 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) u32 op = bind_ops[i].op; u32 flags = bind_ops[i].flags; u64 obj_offset = bind_ops[i].obj_offset; - u8 tile_mask = bind_ops[i].tile_mask; u32 prefetch_region = bind_ops[i].prefetch_mem_region_instance; u16 pat_index = bind_ops[i].pat_index; ops[i] = vm_bind_ioctl_ops_create(vm, bos[i], obj_offset, addr, range, op, flags, - tile_mask, prefetch_region, - pat_index); + prefetch_region, pat_index); if (IS_ERR(ops[i])) { err = PTR_ERR(ops[i]); ops[i] = NULL; diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 74cdf16a42ad..e70ec6b2fabe 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -400,8 +400,6 @@ struct xe_vma_op { u32 num_syncs; /** @link: async operation link */ struct list_head link; - /** @tile_mask: gt mask for this operation */ - u8 tile_mask; /** @flags: operation flags */ enum xe_vma_op_flags flags; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 978fca7bb235..77d54926e18f 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -750,12 +750,6 @@ struct drm_xe_vm_bind_op { /** @addr: Address to operate on, MBZ for UNMAP_ALL */ __u64 addr; - /** - * @tile_mask: Mask for which tiles to create binds for, 0 == All tiles, - * only applies to creating new VMAs - */ - __u64 tile_mask; - #define DRM_XE_VM_BIND_OP_MAP 0x0 #define DRM_XE_VM_BIND_OP_UNMAP 0x1 #define DRM_XE_VM_BIND_OP_MAP_USERPTR 0x2 @@ -790,7 +784,7 @@ struct drm_xe_vm_bind_op { __u32 prefetch_mem_region_instance; /** @reserved: Reserved */ - __u64 reserved[2]; + __u64 reserved[3]; }; struct drm_xe_vm_bind { -- cgit v1.2.3 From 4016d6bf368c4894c834e0652aecd93f7d2a2fab Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 22 Nov 2023 14:38:29 +0000 Subject: drm/xe/uapi: Crystal Reference Clock updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First of all, let's remove the duplication. But also, let's rename it to remove the word 'frequency' out of it. In general, the first thing people think of frequency is the frequency in which the GTs are operating to execute the GPU instructions. While this frequency here is a crystal reference clock frequency which is the base of everything else, and in this case of this uAPI it is used to calculate a better and precise timestamp. v2: (Suggested by Jose) Remove the engine_cs and keep the GT info one since it might be useful for other SRIOV cases where the engine_cs will be zeroed. So, grabbing from the GT_LIST should be cleaner. v3: Keep comment on put_user() call (José Roberto de Souza) Cc: Matt Roper Umesh Nerlige Ramappa Cc: Jose Souza Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_gt_clock.c | 4 ++-- drivers/gpu/drm/xe/xe_gt_types.h | 4 ++-- drivers/gpu/drm/xe/xe_query.c | 7 +------ include/uapi/drm/xe_drm.h | 11 ++++------- 4 files changed, 9 insertions(+), 17 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_gt_clock.c b/drivers/gpu/drm/xe/xe_gt_clock.c index 25a18eaad9c4..937054e31d72 100644 --- a/drivers/gpu/drm/xe/xe_gt_clock.c +++ b/drivers/gpu/drm/xe/xe_gt_clock.c @@ -75,11 +75,11 @@ int xe_gt_clock_init(struct xe_gt *gt) freq >>= 3 - REG_FIELD_GET(RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK, c0); } - gt->info.clock_freq = freq; + gt->info.reference_clock = freq; return 0; } u64 xe_gt_clock_cycles_to_ns(const struct xe_gt *gt, u64 count) { - return DIV_ROUND_CLOSEST_ULL(count * NSEC_PER_SEC, gt->info.clock_freq); + return DIV_ROUND_CLOSEST_ULL(count * NSEC_PER_SEC, gt->info.reference_clock); } diff --git a/drivers/gpu/drm/xe/xe_gt_types.h b/drivers/gpu/drm/xe/xe_gt_types.h index a96ee7d028aa..a7263738308e 100644 --- a/drivers/gpu/drm/xe/xe_gt_types.h +++ b/drivers/gpu/drm/xe/xe_gt_types.h @@ -107,8 +107,8 @@ struct xe_gt { enum xe_gt_type type; /** @id: Unique ID of this GT within the PCI Device */ u8 id; - /** @clock_freq: clock frequency */ - u32 clock_freq; + /** @reference_clock: clock frequency */ + u32 reference_clock; /** @engine_mask: mask of engines present on GT */ u64 engine_mask; /** diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index ad9f23e43920..3316eab118b1 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -147,8 +147,6 @@ query_engine_cycles(struct xe_device *xe, if (!hwe) return -EINVAL; - resp.engine_frequency = gt->info.clock_freq; - xe_device_mem_access_get(xe); xe_force_wake_get(gt_to_fw(gt), XE_FORCEWAKE_ALL); @@ -165,9 +163,6 @@ query_engine_cycles(struct xe_device *xe, resp.width = 36; /* Only write to the output fields of user query */ - if (put_user(resp.engine_frequency, &query_ptr->engine_frequency)) - return -EFAULT; - if (put_user(resp.cpu_timestamp, &query_ptr->cpu_timestamp)) return -EFAULT; @@ -383,7 +378,7 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query else gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MAIN; gt_list->gt_list[id].gt_id = gt->info.id; - gt_list->gt_list[id].clock_freq = gt->info.clock_freq; + gt_list->gt_list[id].reference_clock = gt->info.reference_clock; if (!IS_DGFX(xe)) gt_list->gt_list[id].near_mem_regions = 0x1; else diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 77d54926e18f..df3e6fcf9b8b 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -281,8 +281,8 @@ struct drm_xe_mem_region { * in .data. struct drm_xe_query_engine_cycles is allocated by the user and * .data points to this allocated structure. * - * The query returns the engine cycles and the frequency that can - * be used to calculate the engine timestamp. In addition the + * The query returns the engine cycles, which along with GT's @reference_clock, + * can be used to calculate the engine timestamp. In addition the * query returns a set of cpu timestamps that indicate when the command * streamer cycle count was captured. */ @@ -310,9 +310,6 @@ struct drm_xe_query_engine_cycles { */ __u64 engine_cycles; - /** @engine_frequency: Frequency of the engine cycles in Hz. */ - __u64 engine_frequency; - /** * @cpu_timestamp: CPU timestamp in ns. The timestamp is captured before * reading the engine_cycles register using the reference clockid set by the @@ -383,8 +380,8 @@ struct drm_xe_gt { __u16 type; /** @gt_id: Unique ID of this GT within the PCI Device */ __u16 gt_id; - /** @clock_freq: A clock frequency for timestamp */ - __u32 clock_freq; + /** @reference_clock: A clock frequency for timestamp */ + __u32 reference_clock; /** * @near_mem_regions: Bit mask of instances from * drm_xe_query_mem_regions that are nearest to the current engines -- cgit v1.2.3 From c3fca1077b9a19e679ec59ff2d2c5f4069e375ae Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 22 Nov 2023 14:38:31 +0000 Subject: drm/xe/uapi: Add Tile ID information to the GT info query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As an information only. So Userspace can use this information and be able to correlate different GTs. Make API symmetric between Engine and GT info. There's no need right now to include a tile_query entry since there's no other information that we need from tile that is not already exposed through different queries. However, this could be added later if we have different Tile information that could matter to userspace. But let's keep the API ready for a direct reference to Tile ID based on the GT entry. Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_query.c | 1 + include/uapi/drm/xe_drm.h | 2 ++ 2 files changed, 3 insertions(+) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 3316eab118b1..4461dd1c9e40 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -377,6 +377,7 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MEDIA; else gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MAIN; + gt_list->gt_list[id].tile_id = gt_to_tile(gt)->id; gt_list->gt_list[id].gt_id = gt->info.id; gt_list->gt_list[id].reference_clock = gt->info.reference_clock; if (!IS_DGFX(xe)) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index df3e6fcf9b8b..584fe08e775c 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -378,6 +378,8 @@ struct drm_xe_gt { #define DRM_XE_QUERY_GT_TYPE_MEDIA 1 /** @type: GT type: Main or Media */ __u16 type; + /** @tile_id: Tile ID where this GT lives (Information only) */ + __u16 tile_id; /** @gt_id: Unique ID of this GT within the PCI Device */ __u16 gt_id; /** @reference_clock: A clock frequency for timestamp */ -- cgit v1.2.3 From 7a56bd0cfbeafab33030c782c40b009e39c4bbc0 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 22 Nov 2023 14:38:32 +0000 Subject: drm/xe/uapi: Fix various struct padding for 64b alignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Let's respect Documentation/process/botching-up-ioctls.rst and add the proper padding for a 64b alignment with all as well as all the required checks and settings for the pads and the reserved entries. v2: Fix remaining holes and double check with pahole (Jose) Ensure with pahole that both 32b and 64b have exact same layout (Thomas) Do not set query's pad and reserved bits to zero since it is redundant and already done by kzalloc (Matt) v3: Fix alignment after rebase (José Roberto de Souza) v4: Fix pad check (Francois Dugast) Cc: Thomas Hellström Cc: Francois Dugast Cc: José Roberto de Souza Cc: Matt Roper Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza --- drivers/gpu/drm/xe/xe_bo.c | 3 ++- drivers/gpu/drm/xe/xe_query.c | 1 + drivers/gpu/drm/xe/xe_vm.c | 8 ++++++++ include/uapi/drm/xe_drm.h | 21 ++++++++++++--------- 4 files changed, 23 insertions(+), 10 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index 0bd1b3581945..9cc78986dbd3 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -1894,7 +1894,8 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, u32 handle; int err; - if (XE_IOCTL_DBG(xe, args->extensions) || XE_IOCTL_DBG(xe, args->pad) || + if (XE_IOCTL_DBG(xe, args->extensions) || + XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) return -EINVAL; diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 4461dd1c9e40..56d61bf596b2 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -372,6 +372,7 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query return -ENOMEM; gt_list->num_gt = xe->info.gt_count; + for_each_gt(gt, xe, id) { if (xe_gt_is_media_type(gt)) gt_list->gt_list[id].type = DRM_XE_QUERY_GT_TYPE_MEDIA; diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index ff22eddc2578..622a869fd18e 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -2825,6 +2825,10 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, int err; int i; + if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + if (XE_IOCTL_DBG(xe, args->extensions) || XE_IOCTL_DBG(xe, !args->num_binds) || XE_IOCTL_DBG(xe, args->num_binds > MAX_BINDS)) @@ -2963,6 +2967,10 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) if (err) return err; + if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || + XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) + return -EINVAL; + if (args->exec_queue_id) { q = xe_exec_queue_lookup(xef, args->exec_queue_id); if (XE_IOCTL_DBG(xe, !q)) { diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 584fe08e775c..512c39ea5d50 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -212,8 +212,6 @@ struct drm_xe_mem_region { * a unique pair. */ __u16 instance; - /** @pad: MBZ */ - __u32 pad; /** * @min_page_size: Min page-size in bytes for this region. * @@ -382,6 +380,8 @@ struct drm_xe_gt { __u16 tile_id; /** @gt_id: Unique ID of this GT within the PCI Device */ __u16 gt_id; + /** @pad: MBZ */ + __u16 pad[3]; /** @reference_clock: A clock frequency for timestamp */ __u32 reference_clock; /** @@ -601,7 +601,7 @@ struct drm_xe_gem_create { #define DRM_XE_GEM_CPU_CACHING_WC 2 __u16 cpu_caching; /** @pad: MBZ */ - __u16 pad; + __u16 pad[3]; /** @reserved: Reserved */ __u64 reserved[2]; @@ -782,6 +782,9 @@ struct drm_xe_vm_bind_op { */ __u32 prefetch_mem_region_instance; + /** @pad: MBZ */ + __u32 pad2; + /** @reserved: Reserved */ __u64 reserved[3]; }; @@ -800,12 +803,12 @@ struct drm_xe_vm_bind { */ __u32 exec_queue_id; - /** @num_binds: number of binds in this IOCTL */ - __u32 num_binds; - /** @pad: MBZ */ __u32 pad; + /** @num_binds: number of binds in this IOCTL */ + __u32 num_binds; + union { /** @bind: used if num_binds == 1 */ struct drm_xe_vm_bind_op bind; @@ -817,12 +820,12 @@ struct drm_xe_vm_bind { __u64 vector_of_binds; }; + /** @pad: MBZ */ + __u32 pad2; + /** @num_syncs: amount of syncs to wait on */ __u32 num_syncs; - /** @pad2: MBZ */ - __u32 pad2; - /** @syncs: pointer to struct drm_xe_sync array */ __u64 syncs; -- cgit v1.2.3 From 926ad2c38007bd490958164be2b30db80be59993 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 22 Nov 2023 14:38:33 +0000 Subject: drm/xe/uapi: Move xe_exec after xe_exec_queue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Although the exec ioctl is a very important one, it makes no sense to explain xe_exec before explaining the exec_queue. So, let's move this down to help bring a better flow on the documentation and code readability. It is important to highlight that this patch is changing all the ioctl numbers in a non-backward compatible way. However, we are doing this final uapi clean-up before we submit our first pull-request to be part of the upstream Kernel. Once we get there, no other change like this will ever happen and all the backward compatibility will be respected. Signed-off-by: Rodrigo Vivi Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza --- include/uapi/drm/xe_drm.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 512c39ea5d50..1be67d6bfd95 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -103,11 +103,11 @@ struct xe_user_extension { #define DRM_XE_VM_CREATE 0x03 #define DRM_XE_VM_DESTROY 0x04 #define DRM_XE_VM_BIND 0x05 -#define DRM_XE_EXEC 0x06 -#define DRM_XE_EXEC_QUEUE_CREATE 0x07 -#define DRM_XE_EXEC_QUEUE_DESTROY 0x08 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x09 -#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x0a +#define DRM_XE_EXEC_QUEUE_CREATE 0x06 +#define DRM_XE_EXEC_QUEUE_DESTROY 0x07 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x08 +#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x09 +#define DRM_XE_EXEC 0x0a #define DRM_XE_WAIT_USER_FENCE 0x0b /* Must be kept compact -- no holes */ @@ -117,11 +117,11 @@ struct xe_user_extension { #define DRM_IOCTL_XE_VM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_CREATE, struct drm_xe_vm_create) #define DRM_IOCTL_XE_VM_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) #define DRM_IOCTL_XE_VM_BIND DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) -#define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) #define DRM_IOCTL_XE_EXEC_QUEUE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_CREATE, struct drm_xe_exec_queue_create) #define DRM_IOCTL_XE_EXEC_QUEUE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_DESTROY, struct drm_xe_exec_queue_destroy) #define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property) #define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property) +#define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) /** -- cgit v1.2.3 From 9329f0667215a5c22d650f870f8a9f5839a5bc5a Mon Sep 17 00:00:00 2001 From: Thomas Hellström Date: Mon, 27 Nov 2023 16:03:30 +0100 Subject: drm/xe/uapi: Use LR abbrev for long-running vms MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Currently we're using "compute mode" for long running VMs using preempt-fences for memory management, and "fault mode" for long running VMs using page faults. Change this to use the terminology "long-running" abbreviated as LR for long-running VMs. These VMs can then either be in preempt-fence mode or fault mode. The user can force fault mode at creation time, but otherwise the driver can choose to use fault- or preempt-fence mode for long-running vms depending on the device capabilities. Initially unless fault-mode is specified, the driver uses preempt-fence mode. v2: - Fix commit message wording and the documentation around CREATE_FLAG_LR_MODE and CREATE_FLAG_FAULT_MODE Cc: Matthew Brost Cc: Rodrigo Vivi Cc: Francois Dugast Signed-off-by: Thomas Hellström Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_vm.c | 8 ++++---- include/uapi/drm/xe_drm.h | 23 ++++++++++++++++++++++- 2 files changed, 26 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 622a869fd18e..f71285e8ef10 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1921,7 +1921,7 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, } #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \ - DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE | \ + DRM_XE_VM_CREATE_FLAG_LR_MODE | \ DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT | \ DRM_XE_VM_CREATE_FLAG_FAULT_MODE) @@ -1957,7 +1957,7 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) return -EINVAL; - if (XE_IOCTL_DBG(xe, args->flags & DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE && + if (XE_IOCTL_DBG(xe, !(args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) && args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE)) return -EINVAL; @@ -1974,12 +1974,12 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, if (args->flags & DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE) flags |= XE_VM_FLAG_SCRATCH_PAGE; - if (args->flags & DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE) + if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) flags |= XE_VM_FLAG_LR_MODE; if (args->flags & DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT) flags |= XE_VM_FLAG_ASYNC_DEFAULT; if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) - flags |= XE_VM_FLAG_LR_MODE | XE_VM_FLAG_FAULT_MODE; + flags |= XE_VM_FLAG_FAULT_MODE; vm = xe_vm_create(xe, flags); if (IS_ERR(vm)) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 1be67d6bfd95..28230a0cd1ba 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -648,8 +648,29 @@ struct drm_xe_vm_create { __u64 extensions; #define DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE (1 << 0) -#define DRM_XE_VM_CREATE_FLAG_COMPUTE_MODE (1 << 1) + /* + * An LR, or Long Running VM accepts exec submissions + * to its exec_queues that don't have an upper time limit on + * the job execution time. But exec submissions to these + * don't allow any of the flags DRM_XE_SYNC_FLAG_SYNCOBJ, + * DRM_XE_SYNC_FLAG_TIMELINE_SYNCOBJ, DRM_XE_SYNC_FLAG_DMA_BUF, + * used as out-syncobjs, that is, together with DRM_XE_SYNC_FLAG_SIGNAL. + * LR VMs can be created in recoverable page-fault mode using + * DRM_XE_VM_CREATE_FLAG_FAULT_MODE, if the device supports it. + * If that flag is omitted, the UMD can not rely on the slightly + * different per-VM overcommit semantics that are enabled by + * DRM_XE_VM_CREATE_FLAG_FAULT_MODE (see below), but KMD may + * still enable recoverable pagefaults if supported by the device. + */ +#define DRM_XE_VM_CREATE_FLAG_LR_MODE (1 << 1) #define DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT (1 << 2) + /* + * DRM_XE_VM_CREATE_FLAG_FAULT_MODE requires also + * DRM_XE_VM_CREATE_FLAG_LR_MODE. It allows memory to be allocated + * on demand when accessed, and also allows per-VM overcommit of memory. + * The xe driver internally uses recoverable pagefaults to implement + * this. + */ #define DRM_XE_VM_CREATE_FLAG_FAULT_MODE (1 << 3) /** @flags: Flags */ __u32 flags; -- cgit v1.2.3 From 9209fbede74f202168f0b525060feb6bf67924ba Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 29 Nov 2023 11:29:00 -0500 Subject: drm/xe: Remove unused extension definition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The vm_create ioctl function doesn't accept any extension. Remove this left over. A backward compatible change. Cc: Francois Dugast Signed-off-by: Rodrigo Vivi Reviewed-by: Matthew Brost Signed-off-by: Francois Dugast Reviewed-by: José Roberto de Souza --- include/uapi/drm/xe_drm.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 28230a0cd1ba..2ab5ee299be0 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -643,7 +643,6 @@ struct drm_xe_ext_set_property { }; struct drm_xe_vm_create { -#define DRM_XE_VM_EXTENSION_SET_PROPERTY 0 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; -- cgit v1.2.3 From 0f1d88f2786458a8986920669bd8fb3fec6e618d Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Wed, 29 Nov 2023 11:41:15 -0500 Subject: drm/xe/uapi: Kill exec_queue_set_property MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All the properties should be immutable and set upon exec_queue creation using the existent extension. So, let's kill this useless and dangerous uapi. Cc: Francois Dugast Cc: José Roberto de Souza Cc: Matthew Brost Signed-off-by: Rodrigo Vivi Reviewed-by: José Roberto de Souza Signed-off-by: Francois Dugast --- drivers/gpu/drm/xe/xe_device.c | 2 -- drivers/gpu/drm/xe/xe_exec_queue.c | 38 ------------------------------ drivers/gpu/drm/xe/xe_exec_queue.h | 2 -- include/uapi/drm/xe_drm.h | 48 +++++++++++--------------------------- 4 files changed, 13 insertions(+), 77 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 65e9aa5e6c31..8423c817111b 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -122,8 +122,6 @@ static const struct drm_ioctl_desc xe_ioctls[] = { DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_DESTROY, xe_exec_queue_destroy_ioctl, DRM_RENDER_ALLOW), - DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_SET_PROPERTY, xe_exec_queue_set_property_ioctl, - DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_EXEC_QUEUE_GET_PROPERTY, xe_exec_queue_get_property_ioctl, DRM_RENDER_ALLOW), DRM_IOCTL_DEF_DRV(XE_WAIT_USER_FENCE, xe_wait_user_fence_ioctl, diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 2bab6fbd82f5..985807d6abbb 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -883,44 +883,6 @@ int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, return 0; } -int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, - struct drm_file *file) -{ - struct xe_device *xe = to_xe_device(dev); - struct xe_file *xef = to_xe_file(file); - struct drm_xe_exec_queue_set_property *args = data; - struct xe_exec_queue *q; - int ret; - u32 idx; - - if (XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - - q = xe_exec_queue_lookup(xef, args->exec_queue_id); - if (XE_IOCTL_DBG(xe, !q)) - return -ENOENT; - - if (XE_IOCTL_DBG(xe, args->property >= - ARRAY_SIZE(exec_queue_set_property_funcs))) { - ret = -EINVAL; - goto out; - } - - idx = array_index_nospec(args->property, - ARRAY_SIZE(exec_queue_set_property_funcs)); - ret = exec_queue_set_property_funcs[idx](xe, q, args->value, false); - if (XE_IOCTL_DBG(xe, ret)) - goto out; - - if (args->extensions) - ret = exec_queue_user_extensions(xe, q, args->extensions, 0, - false); -out: - xe_exec_queue_put(q); - - return ret; -} - static void xe_exec_queue_last_fence_lockdep_assert(struct xe_exec_queue *q, struct xe_vm *vm) { diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 533da1b0c457..d959cc4a1a82 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -55,8 +55,6 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file); int xe_exec_queue_destroy_ioctl(struct drm_device *dev, void *data, struct drm_file *file); -int xe_exec_queue_set_property_ioctl(struct drm_device *dev, void *data, - struct drm_file *file); int xe_exec_queue_get_property_ioctl(struct drm_device *dev, void *data, struct drm_file *file); enum xe_exec_queue_priority xe_exec_queue_device_get_max_priority(struct xe_device *xe); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 2ab5ee299be0..0895e4d2a981 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -105,10 +105,9 @@ struct xe_user_extension { #define DRM_XE_VM_BIND 0x05 #define DRM_XE_EXEC_QUEUE_CREATE 0x06 #define DRM_XE_EXEC_QUEUE_DESTROY 0x07 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY 0x08 -#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x09 -#define DRM_XE_EXEC 0x0a -#define DRM_XE_WAIT_USER_FENCE 0x0b +#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x08 +#define DRM_XE_EXEC 0x09 +#define DRM_XE_WAIT_USER_FENCE 0x0a /* Must be kept compact -- no holes */ #define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query) @@ -867,38 +866,17 @@ struct drm_xe_vm_bind { /* Monitor 64MB contiguous region with 2M sub-granularity */ #define DRM_XE_ACC_GRANULARITY_64M 3 -/** - * struct drm_xe_exec_queue_set_property - exec queue set property - * - * Same namespace for extensions as drm_xe_exec_queue_create - */ -struct drm_xe_exec_queue_set_property { - /** @extensions: Pointer to the first extension struct, if any */ - __u64 extensions; - - /** @exec_queue_id: Exec queue ID */ - __u32 exec_queue_id; - -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT 2 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE 3 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT 4 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER 5 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY 6 -#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY 7 - /** @property: property to set */ - __u32 property; - - /** @value: property value */ - __u64 value; - - /** @reserved: Reserved */ - __u64 reserved[2]; -}; - struct drm_xe_exec_queue_create { -#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 +#define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PREEMPTION_TIMEOUT 2 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PERSISTENCE 3 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_JOB_TIMEOUT 4 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER 5 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY 6 +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY 7 + /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; -- cgit v1.2.3 From 9212da07187f86db8bd124b1ce551a18b8a710d6 Mon Sep 17 00:00:00 2001 From: Bommu Krishnaiah Date: Fri, 15 Dec 2023 15:45:33 +0000 Subject: drm/xe/uapi: add exec_queue_id member to drm_xe_wait_user_fence structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit remove the num_engines/instances members from drm_xe_wait_user_fence structure and add a exec_queue_id member Right now this is only checking if the engine list is sane and nothing else. In the end every operation with this IOCTL is a soft check. So, let's formalize that and only use this IOCTL to wait on the fence. exec_queue_id member will help to user space to get proper error code from kernel while in exec_queue reset Signed-off-by: Bommu Krishnaiah Signed-off-by: Rodrigo Vivi Acked-by: Matthew Brost Reviewed-by: Francois Dugast Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast --- drivers/gpu/drm/xe/xe_wait_user_fence.c | 65 +-------------------------------- include/uapi/drm/xe_drm.h | 17 +++------ 2 files changed, 7 insertions(+), 75 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_wait_user_fence.c b/drivers/gpu/drm/xe/xe_wait_user_fence.c index 4d5c2555ce41..59af65b6ed89 100644 --- a/drivers/gpu/drm/xe/xe_wait_user_fence.c +++ b/drivers/gpu/drm/xe/xe_wait_user_fence.c @@ -50,37 +50,7 @@ static int do_compare(u64 addr, u64 value, u64 mask, u16 op) return passed ? 0 : 1; } -static const enum xe_engine_class user_to_xe_engine_class[] = { - [DRM_XE_ENGINE_CLASS_RENDER] = XE_ENGINE_CLASS_RENDER, - [DRM_XE_ENGINE_CLASS_COPY] = XE_ENGINE_CLASS_COPY, - [DRM_XE_ENGINE_CLASS_VIDEO_DECODE] = XE_ENGINE_CLASS_VIDEO_DECODE, - [DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE] = XE_ENGINE_CLASS_VIDEO_ENHANCE, - [DRM_XE_ENGINE_CLASS_COMPUTE] = XE_ENGINE_CLASS_COMPUTE, -}; - -static int check_hw_engines(struct xe_device *xe, - struct drm_xe_engine_class_instance *eci, - int num_engines) -{ - int i; - - for (i = 0; i < num_engines; ++i) { - enum xe_engine_class user_class = - user_to_xe_engine_class[eci[i].engine_class]; - - if (eci[i].gt_id >= xe->info.tile_count) - return -EINVAL; - - if (!xe_gt_hw_engine(xe_device_get_gt(xe, eci[i].gt_id), - user_class, eci[i].engine_instance, true)) - return -EINVAL; - } - - return 0; -} - -#define VALID_FLAGS (DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP | \ - DRM_XE_UFENCE_WAIT_FLAG_ABSTIME) +#define VALID_FLAGS DRM_XE_UFENCE_WAIT_FLAG_ABSTIME #define MAX_OP DRM_XE_UFENCE_WAIT_OP_LTE static long to_jiffies_timeout(struct xe_device *xe, @@ -132,16 +102,13 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, struct xe_device *xe = to_xe_device(dev); DEFINE_WAIT_FUNC(w_wait, woken_wake_function); struct drm_xe_wait_user_fence *args = data; - struct drm_xe_engine_class_instance eci[XE_HW_ENGINE_MAX_INSTANCE]; - struct drm_xe_engine_class_instance __user *user_eci = - u64_to_user_ptr(args->instances); u64 addr = args->addr; int err; - bool no_engines = args->flags & DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP; long timeout; ktime_t start; if (XE_IOCTL_DBG(xe, args->extensions) || XE_IOCTL_DBG(xe, args->pad) || + XE_IOCTL_DBG(xe, args->pad2) || XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) return -EINVAL; @@ -151,41 +118,13 @@ int xe_wait_user_fence_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, args->op > MAX_OP)) return -EINVAL; - if (XE_IOCTL_DBG(xe, no_engines && - (args->num_engines || args->instances))) - return -EINVAL; - - if (XE_IOCTL_DBG(xe, !no_engines && !args->num_engines)) - return -EINVAL; - if (XE_IOCTL_DBG(xe, addr & 0x7)) return -EINVAL; - if (XE_IOCTL_DBG(xe, args->num_engines > XE_HW_ENGINE_MAX_INSTANCE)) - return -EINVAL; - - if (!no_engines) { - err = copy_from_user(eci, user_eci, - sizeof(struct drm_xe_engine_class_instance) * - args->num_engines); - if (XE_IOCTL_DBG(xe, err)) - return -EFAULT; - - if (XE_IOCTL_DBG(xe, check_hw_engines(xe, eci, - args->num_engines))) - return -EINVAL; - } - timeout = to_jiffies_timeout(xe, args); start = ktime_get(); - /* - * FIXME: Very simple implementation at the moment, single wait queue - * for everything. Could be optimized to have a wait queue for every - * hardware engine. Open coding as 'do_compare' can sleep which doesn't - * work with the wait_event_* macros. - */ add_wait_queue(&xe->ufence_wq, &w_wait); for (;;) { err = do_compare(addr, args->value, args->mask, args->op); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 0895e4d2a981..5a8e3b326347 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1031,8 +1031,7 @@ struct drm_xe_wait_user_fence { /** @op: wait operation (type of comparison) */ __u16 op; -#define DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP (1 << 0) /* e.g. Wait on VM bind */ -#define DRM_XE_UFENCE_WAIT_FLAG_ABSTIME (1 << 1) +#define DRM_XE_UFENCE_WAIT_FLAG_ABSTIME (1 << 0) /** @flags: wait flags */ __u16 flags; @@ -1065,17 +1064,11 @@ struct drm_xe_wait_user_fence { */ __s64 timeout; - /** - * @num_engines: number of engine instances to wait on, must be zero - * when DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP set - */ - __u64 num_engines; + /** @exec_queue_id: exec_queue_id returned from xe_exec_queue_create_ioctl */ + __u32 exec_queue_id; - /** - * @instances: user pointer to array of drm_xe_engine_class_instance to - * wait on, must be NULL when DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP set - */ - __u64 instances; + /** @pad2: MBZ */ + __u32 pad2; /** @reserved: Reserved */ __u64 reserved[2]; -- cgit v1.2.3 From e4f0cc64669bb52e259da49c7c1d5954ae8014c5 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:35 +0000 Subject: drm/xe/uapi: Remove DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The exec_queue_set_property feature was removed in a previous commit 0f1d88f27864 ("drm/xe/uapi: Kill exec_queue_set_property") and is no longer usable, struct drm_xe_exec_queue_set_property does not exist anymore, so let's remove this. Reviewed-by: Lucas De Marchi Acked-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 1 - 1 file changed, 1 deletion(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 5a8e3b326347..128369299e49 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -118,7 +118,6 @@ struct xe_user_extension { #define DRM_IOCTL_XE_VM_BIND DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) #define DRM_IOCTL_XE_EXEC_QUEUE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_CREATE, struct drm_xe_exec_queue_create) #define DRM_IOCTL_XE_EXEC_QUEUE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_DESTROY, struct drm_xe_exec_queue_destroy) -#define DRM_IOCTL_XE_EXEC_QUEUE_SET_PROPERTY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_SET_PROPERTY, struct drm_xe_exec_queue_set_property) #define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property) #define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) -- cgit v1.2.3 From 9d329b4cea1449b4f4948a5f495e2d1db223ad7a Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:36 +0000 Subject: drm/xe/uapi: Remove DRM_XE_UFENCE_WAIT_MASK_* MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Those are just possible values for the comparison mask but they are not specific magic values. Let's keep them as examples in the documentation but remove them from the uAPI. Suggested-by: Matthew Brost Cc: Rodrigo Vivi Reviewed-by: Matthew Brost Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 128369299e49..d122f985435a 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1040,11 +1040,13 @@ struct drm_xe_wait_user_fence { /** @value: compare value */ __u64 value; -#define DRM_XE_UFENCE_WAIT_MASK_U8 0xffu -#define DRM_XE_UFENCE_WAIT_MASK_U16 0xffffu -#define DRM_XE_UFENCE_WAIT_MASK_U32 0xffffffffu -#define DRM_XE_UFENCE_WAIT_MASK_U64 0xffffffffffffffffu - /** @mask: comparison mask */ + /** + * @mask: comparison mask, values can be for example: + * - 0xffu for u8 + * - 0xffffu for u16 + * - 0xffffffffu for u32 + * - 0xffffffffffffffffu for u64 + */ __u64 mask; /** -- cgit v1.2.3 From 90a8b23f9b85a05ac3147498c42b32348bfcc274 Mon Sep 17 00:00:00 2001 From: Ashutosh Dixit Date: Fri, 15 Dec 2023 15:45:37 +0000 Subject: drm/xe/pmu: Remove PMU from Xe till uapi is finalized MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PMU uapi is likely to change in the future. Till the uapi is finalized, remove PMU from Xe. PMU can be re-added after uapi is finalized. v2: Include xe_drm.h in xe/tests/xe_dma_buf.c (Francois) Signed-off-by: Ashutosh Dixit Acked-by: Aravind Iddamsetty Acked-by: Lucas De Marchi Reviewed-by: Umesh Nerlige Ramappa Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/Makefile | 2 - drivers/gpu/drm/xe/regs/xe_gt_regs.h | 5 - drivers/gpu/drm/xe/tests/xe_dma_buf.c | 2 + drivers/gpu/drm/xe/xe_device.c | 2 - drivers/gpu/drm/xe/xe_device_types.h | 4 - drivers/gpu/drm/xe/xe_gt.c | 2 - drivers/gpu/drm/xe/xe_module.c | 5 - drivers/gpu/drm/xe/xe_pmu.c | 645 ---------------------------------- drivers/gpu/drm/xe/xe_pmu.h | 25 -- drivers/gpu/drm/xe/xe_pmu_types.h | 68 ---- include/uapi/drm/xe_drm.h | 40 --- 11 files changed, 2 insertions(+), 798 deletions(-) delete mode 100644 drivers/gpu/drm/xe/xe_pmu.c delete mode 100644 drivers/gpu/drm/xe/xe_pmu.h delete mode 100644 drivers/gpu/drm/xe/xe_pmu_types.h (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index 6790c049d89e..53bd2a8ba1ae 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -276,8 +276,6 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \ i915-display/skl_universal_plane.o \ i915-display/skl_watermark.o -xe-$(CONFIG_PERF_EVENTS) += xe_pmu.o - ifeq ($(CONFIG_ACPI),y) xe-$(CONFIG_DRM_XE_DISPLAY) += \ i915-display/intel_acpi.o \ diff --git a/drivers/gpu/drm/xe/regs/xe_gt_regs.h b/drivers/gpu/drm/xe/regs/xe_gt_regs.h index d7f52a634c11..1dd361046b5d 100644 --- a/drivers/gpu/drm/xe/regs/xe_gt_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_gt_regs.h @@ -316,11 +316,6 @@ #define INVALIDATION_BROADCAST_MODE_DIS REG_BIT(12) #define GLOBAL_INVALIDATION_MODE REG_BIT(2) -#define XE_OAG_RC0_ANY_ENGINE_BUSY_FREE XE_REG(0xdb80) -#define XE_OAG_ANY_MEDIA_FF_BUSY_FREE XE_REG(0xdba0) -#define XE_OAG_BLT_BUSY_FREE XE_REG(0xdbbc) -#define XE_OAG_RENDER_BUSY_FREE XE_REG(0xdbdc) - #define HALF_SLICE_CHICKEN5 XE_REG_MCR(0xe188, XE_REG_OPTION_MASKED) #define DISABLE_SAMPLE_G_PERFORMANCE REG_BIT(0) diff --git a/drivers/gpu/drm/xe/tests/xe_dma_buf.c b/drivers/gpu/drm/xe/tests/xe_dma_buf.c index bb6f6424e06f..9f6d571d7fa9 100644 --- a/drivers/gpu/drm/xe/tests/xe_dma_buf.c +++ b/drivers/gpu/drm/xe/tests/xe_dma_buf.c @@ -3,6 +3,8 @@ * Copyright © 2022 Intel Corporation */ +#include + #include #include diff --git a/drivers/gpu/drm/xe/xe_device.c b/drivers/gpu/drm/xe/xe_device.c index 221e87584352..d9ae77fe7382 100644 --- a/drivers/gpu/drm/xe/xe_device.c +++ b/drivers/gpu/drm/xe/xe_device.c @@ -529,8 +529,6 @@ int xe_device_probe(struct xe_device *xe) xe_debugfs_register(xe); - xe_pmu_register(&xe->pmu); - xe_hwmon_register(xe); err = drmm_add_action_or_reset(&xe->drm, xe_device_sanitize, xe); diff --git a/drivers/gpu/drm/xe/xe_device_types.h b/drivers/gpu/drm/xe/xe_device_types.h index d1a48456e9a3..c45ef17b3473 100644 --- a/drivers/gpu/drm/xe/xe_device_types.h +++ b/drivers/gpu/drm/xe/xe_device_types.h @@ -18,7 +18,6 @@ #include "xe_lmtt_types.h" #include "xe_platform_types.h" #include "xe_pt_types.h" -#include "xe_pmu.h" #include "xe_sriov_types.h" #include "xe_step_types.h" @@ -427,9 +426,6 @@ struct xe_device { */ struct task_struct *pm_callback_task; - /** @pmu: performance monitoring unit */ - struct xe_pmu pmu; - /** @hwmon: hwmon subsystem integration */ struct xe_hwmon *hwmon; diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index dfd9cf01a5d5..f5d18e98f8b6 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -709,8 +709,6 @@ int xe_gt_suspend(struct xe_gt *gt) if (err) goto err_msg; - xe_pmu_suspend(gt); - err = xe_uc_suspend(>->uc); if (err) goto err_force_wake; diff --git a/drivers/gpu/drm/xe/xe_module.c b/drivers/gpu/drm/xe/xe_module.c index 51bf69b7ab22..110b69864656 100644 --- a/drivers/gpu/drm/xe/xe_module.c +++ b/drivers/gpu/drm/xe/xe_module.c @@ -11,7 +11,6 @@ #include "xe_drv.h" #include "xe_hw_fence.h" #include "xe_pci.h" -#include "xe_pmu.h" #include "xe_sched_job.h" struct xe_modparam xe_modparam = { @@ -63,10 +62,6 @@ static const struct init_funcs init_funcs[] = { .init = xe_sched_job_module_init, .exit = xe_sched_job_module_exit, }, - { - .init = xe_pmu_init, - .exit = xe_pmu_exit, - }, { .init = xe_register_pci_driver, .exit = xe_unregister_pci_driver, diff --git a/drivers/gpu/drm/xe/xe_pmu.c b/drivers/gpu/drm/xe/xe_pmu.c deleted file mode 100644 index 9d0b7887cfc4..000000000000 --- a/drivers/gpu/drm/xe/xe_pmu.c +++ /dev/null @@ -1,645 +0,0 @@ -// SPDX-License-Identifier: MIT -/* - * Copyright © 2023 Intel Corporation - */ - -#include -#include -#include - -#include "regs/xe_gt_regs.h" -#include "xe_device.h" -#include "xe_gt_clock.h" -#include "xe_mmio.h" - -static cpumask_t xe_pmu_cpumask; -static unsigned int xe_pmu_target_cpu = -1; - -static unsigned int config_gt_id(const u64 config) -{ - return config >> __DRM_XE_PMU_GT_SHIFT; -} - -static u64 config_counter(const u64 config) -{ - return config & ~(~0ULL << __DRM_XE_PMU_GT_SHIFT); -} - -static void xe_pmu_event_destroy(struct perf_event *event) -{ - struct xe_device *xe = - container_of(event->pmu, typeof(*xe), pmu.base); - - drm_WARN_ON(&xe->drm, event->parent); - - drm_dev_put(&xe->drm); -} - -static u64 __engine_group_busyness_read(struct xe_gt *gt, int sample_type) -{ - u64 val; - - switch (sample_type) { - case __XE_SAMPLE_RENDER_GROUP_BUSY: - val = xe_mmio_read32(gt, XE_OAG_RENDER_BUSY_FREE); - break; - case __XE_SAMPLE_COPY_GROUP_BUSY: - val = xe_mmio_read32(gt, XE_OAG_BLT_BUSY_FREE); - break; - case __XE_SAMPLE_MEDIA_GROUP_BUSY: - val = xe_mmio_read32(gt, XE_OAG_ANY_MEDIA_FF_BUSY_FREE); - break; - case __XE_SAMPLE_ANY_ENGINE_GROUP_BUSY: - val = xe_mmio_read32(gt, XE_OAG_RC0_ANY_ENGINE_BUSY_FREE); - break; - default: - drm_warn(>->tile->xe->drm, "unknown pmu event\n"); - } - - return xe_gt_clock_cycles_to_ns(gt, val * 16); -} - -static u64 engine_group_busyness_read(struct xe_gt *gt, u64 config) -{ - int sample_type = config_counter(config); - const unsigned int gt_id = gt->info.id; - struct xe_device *xe = gt->tile->xe; - struct xe_pmu *pmu = &xe->pmu; - unsigned long flags; - bool device_awake; - u64 val; - - device_awake = xe_device_mem_access_get_if_ongoing(xe); - if (device_awake) { - XE_WARN_ON(xe_force_wake_get(gt_to_fw(gt), XE_FW_GT)); - val = __engine_group_busyness_read(gt, sample_type); - XE_WARN_ON(xe_force_wake_put(gt_to_fw(gt), XE_FW_GT)); - xe_device_mem_access_put(xe); - } - - spin_lock_irqsave(&pmu->lock, flags); - - if (device_awake) - pmu->sample[gt_id][sample_type] = val; - else - val = pmu->sample[gt_id][sample_type]; - - spin_unlock_irqrestore(&pmu->lock, flags); - - return val; -} - -static void engine_group_busyness_store(struct xe_gt *gt) -{ - struct xe_pmu *pmu = >->tile->xe->pmu; - unsigned int gt_id = gt->info.id; - unsigned long flags; - int i; - - spin_lock_irqsave(&pmu->lock, flags); - - for (i = __XE_SAMPLE_RENDER_GROUP_BUSY; i <= __XE_SAMPLE_ANY_ENGINE_GROUP_BUSY; i++) - pmu->sample[gt_id][i] = __engine_group_busyness_read(gt, i); - - spin_unlock_irqrestore(&pmu->lock, flags); -} - -static int -config_status(struct xe_device *xe, u64 config) -{ - unsigned int gt_id = config_gt_id(config); - struct xe_gt *gt = xe_device_get_gt(xe, gt_id); - - if (gt_id >= XE_PMU_MAX_GT) - return -ENOENT; - - switch (config_counter(config)) { - case DRM_XE_PMU_RENDER_GROUP_BUSY(0): - case DRM_XE_PMU_COPY_GROUP_BUSY(0): - case DRM_XE_PMU_ANY_ENGINE_GROUP_BUSY(0): - if (gt->info.type == XE_GT_TYPE_MEDIA) - return -ENOENT; - break; - case DRM_XE_PMU_MEDIA_GROUP_BUSY(0): - if (!(gt->info.engine_mask & (BIT(XE_HW_ENGINE_VCS0) | BIT(XE_HW_ENGINE_VECS0)))) - return -ENOENT; - break; - default: - return -ENOENT; - } - - return 0; -} - -static int xe_pmu_event_init(struct perf_event *event) -{ - struct xe_device *xe = - container_of(event->pmu, typeof(*xe), pmu.base); - struct xe_pmu *pmu = &xe->pmu; - int ret; - - if (pmu->closed) - return -ENODEV; - - if (event->attr.type != event->pmu->type) - return -ENOENT; - - /* unsupported modes and filters */ - if (event->attr.sample_period) /* no sampling */ - return -EINVAL; - - if (has_branch_stack(event)) - return -EOPNOTSUPP; - - if (event->cpu < 0) - return -EINVAL; - - /* only allow running on one cpu at a time */ - if (!cpumask_test_cpu(event->cpu, &xe_pmu_cpumask)) - return -EINVAL; - - ret = config_status(xe, event->attr.config); - if (ret) - return ret; - - if (!event->parent) { - drm_dev_get(&xe->drm); - event->destroy = xe_pmu_event_destroy; - } - - return 0; -} - -static u64 __xe_pmu_event_read(struct perf_event *event) -{ - struct xe_device *xe = - container_of(event->pmu, typeof(*xe), pmu.base); - const unsigned int gt_id = config_gt_id(event->attr.config); - const u64 config = event->attr.config; - struct xe_gt *gt = xe_device_get_gt(xe, gt_id); - u64 val; - - switch (config_counter(config)) { - case DRM_XE_PMU_RENDER_GROUP_BUSY(0): - case DRM_XE_PMU_COPY_GROUP_BUSY(0): - case DRM_XE_PMU_ANY_ENGINE_GROUP_BUSY(0): - case DRM_XE_PMU_MEDIA_GROUP_BUSY(0): - val = engine_group_busyness_read(gt, config); - break; - default: - drm_warn(>->tile->xe->drm, "unknown pmu event\n"); - } - - return val; -} - -static void xe_pmu_event_read(struct perf_event *event) -{ - struct xe_device *xe = - container_of(event->pmu, typeof(*xe), pmu.base); - struct hw_perf_event *hwc = &event->hw; - struct xe_pmu *pmu = &xe->pmu; - u64 prev, new; - - if (pmu->closed) { - event->hw.state = PERF_HES_STOPPED; - return; - } -again: - prev = local64_read(&hwc->prev_count); - new = __xe_pmu_event_read(event); - - if (local64_cmpxchg(&hwc->prev_count, prev, new) != prev) - goto again; - - local64_add(new - prev, &event->count); -} - -static void xe_pmu_enable(struct perf_event *event) -{ - /* - * Store the current counter value so we can report the correct delta - * for all listeners. Even when the event was already enabled and has - * an existing non-zero value. - */ - local64_set(&event->hw.prev_count, __xe_pmu_event_read(event)); -} - -static void xe_pmu_event_start(struct perf_event *event, int flags) -{ - struct xe_device *xe = - container_of(event->pmu, typeof(*xe), pmu.base); - struct xe_pmu *pmu = &xe->pmu; - - if (pmu->closed) - return; - - xe_pmu_enable(event); - event->hw.state = 0; -} - -static void xe_pmu_event_stop(struct perf_event *event, int flags) -{ - if (flags & PERF_EF_UPDATE) - xe_pmu_event_read(event); - - event->hw.state = PERF_HES_STOPPED; -} - -static int xe_pmu_event_add(struct perf_event *event, int flags) -{ - struct xe_device *xe = - container_of(event->pmu, typeof(*xe), pmu.base); - struct xe_pmu *pmu = &xe->pmu; - - if (pmu->closed) - return -ENODEV; - - if (flags & PERF_EF_START) - xe_pmu_event_start(event, flags); - - return 0; -} - -static void xe_pmu_event_del(struct perf_event *event, int flags) -{ - xe_pmu_event_stop(event, PERF_EF_UPDATE); -} - -static int xe_pmu_event_event_idx(struct perf_event *event) -{ - return 0; -} - -struct xe_ext_attribute { - struct device_attribute attr; - unsigned long val; -}; - -static ssize_t xe_pmu_event_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct xe_ext_attribute *eattr; - - eattr = container_of(attr, struct xe_ext_attribute, attr); - return sprintf(buf, "config=0x%lx\n", eattr->val); -} - -static ssize_t cpumask_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - return cpumap_print_to_pagebuf(true, buf, &xe_pmu_cpumask); -} - -static DEVICE_ATTR_RO(cpumask); - -static struct attribute *xe_cpumask_attrs[] = { - &dev_attr_cpumask.attr, - NULL, -}; - -static const struct attribute_group xe_pmu_cpumask_attr_group = { - .attrs = xe_cpumask_attrs, -}; - -#define __event(__counter, __name, __unit) \ -{ \ - .counter = (__counter), \ - .name = (__name), \ - .unit = (__unit), \ - .global = false, \ -} - -#define __global_event(__counter, __name, __unit) \ -{ \ - .counter = (__counter), \ - .name = (__name), \ - .unit = (__unit), \ - .global = true, \ -} - -static struct xe_ext_attribute * -add_xe_attr(struct xe_ext_attribute *attr, const char *name, u64 config) -{ - sysfs_attr_init(&attr->attr.attr); - attr->attr.attr.name = name; - attr->attr.attr.mode = 0444; - attr->attr.show = xe_pmu_event_show; - attr->val = config; - - return ++attr; -} - -static struct perf_pmu_events_attr * -add_pmu_attr(struct perf_pmu_events_attr *attr, const char *name, - const char *str) -{ - sysfs_attr_init(&attr->attr.attr); - attr->attr.attr.name = name; - attr->attr.attr.mode = 0444; - attr->attr.show = perf_event_sysfs_show; - attr->event_str = str; - - return ++attr; -} - -static struct attribute ** -create_event_attributes(struct xe_pmu *pmu) -{ - struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); - static const struct { - unsigned int counter; - const char *name; - const char *unit; - bool global; - } events[] = { - __event(0, "render-group-busy", "ns"), - __event(1, "copy-group-busy", "ns"), - __event(2, "media-group-busy", "ns"), - __event(3, "any-engine-group-busy", "ns"), - }; - - struct perf_pmu_events_attr *pmu_attr = NULL, *pmu_iter; - struct xe_ext_attribute *xe_attr = NULL, *xe_iter; - struct attribute **attr = NULL, **attr_iter; - unsigned int count = 0; - unsigned int i, j; - struct xe_gt *gt; - - /* Count how many counters we will be exposing. */ - for_each_gt(gt, xe, j) { - for (i = 0; i < ARRAY_SIZE(events); i++) { - u64 config = ___DRM_XE_PMU_OTHER(j, events[i].counter); - - if (!config_status(xe, config)) - count++; - } - } - - /* Allocate attribute objects and table. */ - xe_attr = kcalloc(count, sizeof(*xe_attr), GFP_KERNEL); - if (!xe_attr) - goto err_alloc; - - pmu_attr = kcalloc(count, sizeof(*pmu_attr), GFP_KERNEL); - if (!pmu_attr) - goto err_alloc; - - /* Max one pointer of each attribute type plus a termination entry. */ - attr = kcalloc(count * 2 + 1, sizeof(*attr), GFP_KERNEL); - if (!attr) - goto err_alloc; - - xe_iter = xe_attr; - pmu_iter = pmu_attr; - attr_iter = attr; - - for_each_gt(gt, xe, j) { - for (i = 0; i < ARRAY_SIZE(events); i++) { - u64 config = ___DRM_XE_PMU_OTHER(j, events[i].counter); - char *str; - - if (config_status(xe, config)) - continue; - - if (events[i].global) - str = kstrdup(events[i].name, GFP_KERNEL); - else - str = kasprintf(GFP_KERNEL, "%s-gt%u", - events[i].name, j); - if (!str) - goto err; - - *attr_iter++ = &xe_iter->attr.attr; - xe_iter = add_xe_attr(xe_iter, str, config); - - if (events[i].unit) { - if (events[i].global) - str = kasprintf(GFP_KERNEL, "%s.unit", - events[i].name); - else - str = kasprintf(GFP_KERNEL, "%s-gt%u.unit", - events[i].name, j); - if (!str) - goto err; - - *attr_iter++ = &pmu_iter->attr.attr; - pmu_iter = add_pmu_attr(pmu_iter, str, - events[i].unit); - } - } - } - - pmu->xe_attr = xe_attr; - pmu->pmu_attr = pmu_attr; - - return attr; - -err: - for (attr_iter = attr; *attr_iter; attr_iter++) - kfree((*attr_iter)->name); - -err_alloc: - kfree(attr); - kfree(xe_attr); - kfree(pmu_attr); - - return NULL; -} - -static void free_event_attributes(struct xe_pmu *pmu) -{ - struct attribute **attr_iter = pmu->events_attr_group.attrs; - - for (; *attr_iter; attr_iter++) - kfree((*attr_iter)->name); - - kfree(pmu->events_attr_group.attrs); - kfree(pmu->xe_attr); - kfree(pmu->pmu_attr); - - pmu->events_attr_group.attrs = NULL; - pmu->xe_attr = NULL; - pmu->pmu_attr = NULL; -} - -static int xe_pmu_cpu_online(unsigned int cpu, struct hlist_node *node) -{ - struct xe_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); - - /* Select the first online CPU as a designated reader. */ - if (cpumask_empty(&xe_pmu_cpumask)) - cpumask_set_cpu(cpu, &xe_pmu_cpumask); - - return 0; -} - -static int xe_pmu_cpu_offline(unsigned int cpu, struct hlist_node *node) -{ - struct xe_pmu *pmu = hlist_entry_safe(node, typeof(*pmu), cpuhp.node); - unsigned int target = xe_pmu_target_cpu; - - /* - * Unregistering an instance generates a CPU offline event which we must - * ignore to avoid incorrectly modifying the shared xe_pmu_cpumask. - */ - if (pmu->closed) - return 0; - - if (cpumask_test_and_clear_cpu(cpu, &xe_pmu_cpumask)) { - target = cpumask_any_but(topology_sibling_cpumask(cpu), cpu); - - /* Migrate events if there is a valid target */ - if (target < nr_cpu_ids) { - cpumask_set_cpu(target, &xe_pmu_cpumask); - xe_pmu_target_cpu = target; - } - } - - if (target < nr_cpu_ids && target != pmu->cpuhp.cpu) { - perf_pmu_migrate_context(&pmu->base, cpu, target); - pmu->cpuhp.cpu = target; - } - - return 0; -} - -static enum cpuhp_state cpuhp_slot = CPUHP_INVALID; - -int xe_pmu_init(void) -{ - int ret; - - ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, - "perf/x86/intel/xe:online", - xe_pmu_cpu_online, - xe_pmu_cpu_offline); - if (ret < 0) - pr_notice("Failed to setup cpuhp state for xe PMU! (%d)\n", - ret); - else - cpuhp_slot = ret; - - return 0; -} - -void xe_pmu_exit(void) -{ - if (cpuhp_slot != CPUHP_INVALID) - cpuhp_remove_multi_state(cpuhp_slot); -} - -static int xe_pmu_register_cpuhp_state(struct xe_pmu *pmu) -{ - if (cpuhp_slot == CPUHP_INVALID) - return -EINVAL; - - return cpuhp_state_add_instance(cpuhp_slot, &pmu->cpuhp.node); -} - -static void xe_pmu_unregister_cpuhp_state(struct xe_pmu *pmu) -{ - cpuhp_state_remove_instance(cpuhp_slot, &pmu->cpuhp.node); -} - -void xe_pmu_suspend(struct xe_gt *gt) -{ - engine_group_busyness_store(gt); -} - -static void xe_pmu_unregister(struct drm_device *device, void *arg) -{ - struct xe_pmu *pmu = arg; - - if (!pmu->base.event_init) - return; - - /* - * "Disconnect" the PMU callbacks - since all are atomic synchronize_rcu - * ensures all currently executing ones will have exited before we - * proceed with unregistration. - */ - pmu->closed = true; - synchronize_rcu(); - - xe_pmu_unregister_cpuhp_state(pmu); - - perf_pmu_unregister(&pmu->base); - pmu->base.event_init = NULL; - kfree(pmu->base.attr_groups); - kfree(pmu->name); - free_event_attributes(pmu); -} - -void xe_pmu_register(struct xe_pmu *pmu) -{ - struct xe_device *xe = container_of(pmu, typeof(*xe), pmu); - const struct attribute_group *attr_groups[] = { - &pmu->events_attr_group, - &xe_pmu_cpumask_attr_group, - NULL - }; - - int ret = -ENOMEM; - - spin_lock_init(&pmu->lock); - pmu->cpuhp.cpu = -1; - - pmu->name = kasprintf(GFP_KERNEL, - "xe_%s", - dev_name(xe->drm.dev)); - if (pmu->name) - /* tools/perf reserves colons as special. */ - strreplace((char *)pmu->name, ':', '_'); - - if (!pmu->name) - goto err; - - pmu->events_attr_group.name = "events"; - pmu->events_attr_group.attrs = create_event_attributes(pmu); - if (!pmu->events_attr_group.attrs) - goto err_name; - - pmu->base.attr_groups = kmemdup(attr_groups, sizeof(attr_groups), - GFP_KERNEL); - if (!pmu->base.attr_groups) - goto err_attr; - - pmu->base.module = THIS_MODULE; - pmu->base.task_ctx_nr = perf_invalid_context; - pmu->base.event_init = xe_pmu_event_init; - pmu->base.add = xe_pmu_event_add; - pmu->base.del = xe_pmu_event_del; - pmu->base.start = xe_pmu_event_start; - pmu->base.stop = xe_pmu_event_stop; - pmu->base.read = xe_pmu_event_read; - pmu->base.event_idx = xe_pmu_event_event_idx; - - ret = perf_pmu_register(&pmu->base, pmu->name, -1); - if (ret) - goto err_groups; - - ret = xe_pmu_register_cpuhp_state(pmu); - if (ret) - goto err_unreg; - - ret = drmm_add_action_or_reset(&xe->drm, xe_pmu_unregister, pmu); - if (ret) - goto err_cpuhp; - - return; - -err_cpuhp: - xe_pmu_unregister_cpuhp_state(pmu); -err_unreg: - perf_pmu_unregister(&pmu->base); -err_groups: - kfree(pmu->base.attr_groups); -err_attr: - pmu->base.event_init = NULL; - free_event_attributes(pmu); -err_name: - kfree(pmu->name); -err: - drm_notice(&xe->drm, "Failed to register PMU!\n"); -} diff --git a/drivers/gpu/drm/xe/xe_pmu.h b/drivers/gpu/drm/xe/xe_pmu.h deleted file mode 100644 index a99d4ddd023e..000000000000 --- a/drivers/gpu/drm/xe/xe_pmu.h +++ /dev/null @@ -1,25 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#ifndef _XE_PMU_H_ -#define _XE_PMU_H_ - -#include "xe_gt_types.h" -#include "xe_pmu_types.h" - -#if IS_ENABLED(CONFIG_PERF_EVENTS) -int xe_pmu_init(void); -void xe_pmu_exit(void); -void xe_pmu_register(struct xe_pmu *pmu); -void xe_pmu_suspend(struct xe_gt *gt); -#else -static inline int xe_pmu_init(void) { return 0; } -static inline void xe_pmu_exit(void) {} -static inline void xe_pmu_register(struct xe_pmu *pmu) {} -static inline void xe_pmu_suspend(struct xe_gt *gt) {} -#endif - -#endif - diff --git a/drivers/gpu/drm/xe/xe_pmu_types.h b/drivers/gpu/drm/xe/xe_pmu_types.h deleted file mode 100644 index 9cadbd243f57..000000000000 --- a/drivers/gpu/drm/xe/xe_pmu_types.h +++ /dev/null @@ -1,68 +0,0 @@ -/* SPDX-License-Identifier: MIT */ -/* - * Copyright © 2023 Intel Corporation - */ - -#ifndef _XE_PMU_TYPES_H_ -#define _XE_PMU_TYPES_H_ - -#include -#include -#include - -enum { - __XE_SAMPLE_RENDER_GROUP_BUSY, - __XE_SAMPLE_COPY_GROUP_BUSY, - __XE_SAMPLE_MEDIA_GROUP_BUSY, - __XE_SAMPLE_ANY_ENGINE_GROUP_BUSY, - __XE_NUM_PMU_SAMPLERS -}; - -#define XE_PMU_MAX_GT 2 - -struct xe_pmu { - /** - * @cpuhp: Struct used for CPU hotplug handling. - */ - struct { - struct hlist_node node; - unsigned int cpu; - } cpuhp; - /** - * @base: PMU base. - */ - struct pmu base; - /** - * @closed: xe is unregistering. - */ - bool closed; - /** - * @name: Name as registered with perf core. - */ - const char *name; - /** - * @lock: Lock protecting enable mask and ref count handling. - */ - spinlock_t lock; - /** - * @sample: Current and previous (raw) counters. - * - * These counters are updated when the device is awake. - * - */ - u64 sample[XE_PMU_MAX_GT][__XE_NUM_PMU_SAMPLERS]; - /** - * @events_attr_group: Device events attribute group. - */ - struct attribute_group events_attr_group; - /** - * @xe_attr: Memory block holding device attributes. - */ - void *xe_attr; - /** - * @pmu_attr: Memory block holding device attributes. - */ - void *pmu_attr; -}; - -#endif diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index d122f985435a..e1e8fb1846ea 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1074,46 +1074,6 @@ struct drm_xe_wait_user_fence { /** @reserved: Reserved */ __u64 reserved[2]; }; - -/** - * DOC: XE PMU event config IDs - * - * Check 'man perf_event_open' to use the ID's DRM_XE_PMU_XXXX listed in xe_drm.h - * in 'struct perf_event_attr' as part of perf_event_open syscall to read a - * particular event. - * - * For example to open the DRMXE_PMU_RENDER_GROUP_BUSY(0): - * - * .. code-block:: C - * - * struct perf_event_attr attr; - * long long count; - * int cpu = 0; - * int fd; - * - * memset(&attr, 0, sizeof(struct perf_event_attr)); - * attr.type = type; // eg: /sys/bus/event_source/devices/xe_0000_56_00.0/type - * attr.read_format = PERF_FORMAT_TOTAL_TIME_ENABLED; - * attr.use_clockid = 1; - * attr.clockid = CLOCK_MONOTONIC; - * attr.config = DRM_XE_PMU_RENDER_GROUP_BUSY(0); - * - * fd = syscall(__NR_perf_event_open, &attr, -1, cpu, -1, 0); - */ - -/* - * Top bits of every counter are GT id. - */ -#define __DRM_XE_PMU_GT_SHIFT (56) - -#define ___DRM_XE_PMU_OTHER(gt, x) \ - (((__u64)(x)) | ((__u64)(gt) << __DRM_XE_PMU_GT_SHIFT)) - -#define DRM_XE_PMU_RENDER_GROUP_BUSY(gt) ___DRM_XE_PMU_OTHER(gt, 0) -#define DRM_XE_PMU_COPY_GROUP_BUSY(gt) ___DRM_XE_PMU_OTHER(gt, 1) -#define DRM_XE_PMU_MEDIA_GROUP_BUSY(gt) ___DRM_XE_PMU_OTHER(gt, 2) -#define DRM_XE_PMU_ANY_ENGINE_GROUP_BUSY(gt) ___DRM_XE_PMU_OTHER(gt, 3) - #if defined(__cplusplus) } #endif -- cgit v1.2.3 From 7e9337c29fb9251e27d7af092108f05857e733c1 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Fri, 15 Dec 2023 15:45:38 +0000 Subject: drm/xe/uapi: Ensure every uapi struct has drm_xe prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit To ensure consistency and avoid possible later conflicts, let's add drm_xe prefix to xe_user_extension struct. Cc: Francois Dugast Suggested-by: Lucas De Marchi Signed-off-by: Rodrigo Vivi Reviewed-by: Matthew Brost Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki --- drivers/gpu/drm/xe/xe_exec_queue.c | 2 +- include/uapi/drm/xe_drm.h | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index eeb9605dd45f..aa478c66edbb 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -453,7 +453,7 @@ static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue u64 extensions, int ext_number, bool create) { u64 __user *address = u64_to_user_ptr(extensions); - struct xe_user_extension ext; + struct drm_xe_user_extension ext; int err; u32 idx; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index e1e8fb1846ea..87ff6eaa788e 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -27,7 +27,7 @@ extern "C" { #define DRM_XE_RESET_FAILED_UEVENT "DEVICE_STATUS" /** - * struct xe_user_extension - Base class for defining a chain of extensions + * struct drm_xe_user_extension - Base class for defining a chain of extensions * * Many interfaces need to grow over time. In most cases we can simply * extend the struct and have userspace pass in more data. Another option, @@ -45,29 +45,29 @@ extern "C" { * * .. code-block:: C * - * struct xe_user_extension ext3 { + * struct drm_xe_user_extension ext3 { * .next_extension = 0, // end * .name = ..., * }; - * struct xe_user_extension ext2 { + * struct drm_xe_user_extension ext2 { * .next_extension = (uintptr_t)&ext3, * .name = ..., * }; - * struct xe_user_extension ext1 { + * struct drm_xe_user_extension ext1 { * .next_extension = (uintptr_t)&ext2, * .name = ..., * }; * - * Typically the struct xe_user_extension would be embedded in some uAPI + * Typically the struct drm_xe_user_extension would be embedded in some uAPI * struct, and in this case we would feed it the head of the chain(i.e ext1), * which would then apply all of the above extensions. * */ -struct xe_user_extension { +struct drm_xe_user_extension { /** * @next_extension: * - * Pointer to the next struct xe_user_extension, or zero if the end. + * Pointer to the next struct drm_xe_user_extension, or zero if the end. */ __u64 next_extension; @@ -78,7 +78,7 @@ struct xe_user_extension { * * Also note that the name space for this is not global for the whole * driver, but rather its scope/meaning is limited to the specific piece - * of uAPI which has embedded the struct xe_user_extension. + * of uAPI which has embedded the struct drm_xe_user_extension. */ __u32 name; @@ -625,7 +625,7 @@ struct drm_xe_gem_mmap_offset { /** struct drm_xe_ext_set_property - XE set property extension */ struct drm_xe_ext_set_property { /** @base: base user extension */ - struct xe_user_extension base; + struct drm_xe_user_extension base; /** @property: property to set */ __u32 property; -- cgit v1.2.3 From d3d767396a02fa225eab7f919b727cff4e3304bc Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Fri, 15 Dec 2023 15:45:39 +0000 Subject: drm/xe/uapi: Remove sync binds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove concept of async vs sync VM bind queues, rather make all binds async. The following bits have dropped from the uAPI: DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC DRM_XE_ENGINE_CLASS_VM_BIND_SYNC DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT DRM_XE_VM_BIND_FLAG_ASYNC To implement sync binds the UMD is expected to use the out-fence interface. v2: Send correct version v3: Drop drm_xe_syncs Cc: Rodrigo Vivi Cc: Thomas Hellström Cc: Francois Dugast Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Rodrigo Vivi --- drivers/gpu/drm/xe/xe_exec_queue.c | 7 +-- drivers/gpu/drm/xe/xe_exec_queue_types.h | 2 - drivers/gpu/drm/xe/xe_vm.c | 75 ++++---------------------------- drivers/gpu/drm/xe/xe_vm_types.h | 13 +++--- include/uapi/drm/xe_drm.h | 11 ++--- 5 files changed, 20 insertions(+), 88 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index aa478c66edbb..44fe8097b7cd 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -625,10 +625,7 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) return -EINVAL; - if (eci[0].engine_class >= DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC) { - bool sync = eci[0].engine_class == - DRM_XE_ENGINE_CLASS_VM_BIND_SYNC; - + if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { for_each_gt(gt, xe, id) { struct xe_exec_queue *new; @@ -654,8 +651,6 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, args->width, hwe, EXEC_QUEUE_FLAG_PERSISTENT | EXEC_QUEUE_FLAG_VM | - (sync ? 0 : - EXEC_QUEUE_FLAG_VM_ASYNC) | (id ? EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD : 0)); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index bcf08b00d94a..3d7e704ec3d9 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -84,8 +84,6 @@ struct xe_exec_queue { #define EXEC_QUEUE_FLAG_VM BIT(4) /* child of VM queue for multi-tile VM jobs */ #define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD BIT(5) -/* VM jobs for this queue are asynchronous */ -#define EXEC_QUEUE_FLAG_VM_ASYNC BIT(6) /** * @flags: flags for this exec queue, should statically setup aside from ban diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 2f3df9ee67c9..322c1eccecca 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -1343,9 +1343,7 @@ struct xe_vm *xe_vm_create(struct xe_device *xe, u32 flags) struct xe_gt *gt = tile->primary_gt; struct xe_vm *migrate_vm; struct xe_exec_queue *q; - u32 create_flags = EXEC_QUEUE_FLAG_VM | - ((flags & XE_VM_FLAG_ASYNC_DEFAULT) ? - EXEC_QUEUE_FLAG_VM_ASYNC : 0); + u32 create_flags = EXEC_QUEUE_FLAG_VM; if (!vm->pt_root[id]) continue; @@ -1712,12 +1710,6 @@ err_fences: return ERR_PTR(err); } -static bool xe_vm_sync_mode(struct xe_vm *vm, struct xe_exec_queue *q) -{ - return q ? !(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC) : - !(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT); -} - static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, struct xe_exec_queue *q, struct xe_sync_entry *syncs, u32 num_syncs, bool immediate, bool first_op, @@ -1747,8 +1739,6 @@ static int __xe_vm_bind(struct xe_vm *vm, struct xe_vma *vma, if (last_op) xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); - if (last_op && xe_vm_sync_mode(vm, q)) - dma_fence_wait(fence, true); dma_fence_put(fence); return 0; @@ -1791,8 +1781,6 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, xe_vma_destroy(vma, fence); if (last_op) xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); - if (last_op && xe_vm_sync_mode(vm, q)) - dma_fence_wait(fence, true); dma_fence_put(fence); return 0; @@ -1800,7 +1788,6 @@ static int xe_vm_unbind(struct xe_vm *vm, struct xe_vma *vma, #define ALL_DRM_XE_VM_CREATE_FLAGS (DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE | \ DRM_XE_VM_CREATE_FLAG_LR_MODE | \ - DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT | \ DRM_XE_VM_CREATE_FLAG_FAULT_MODE) int xe_vm_create_ioctl(struct drm_device *dev, void *data, @@ -1854,8 +1841,6 @@ int xe_vm_create_ioctl(struct drm_device *dev, void *data, flags |= XE_VM_FLAG_SCRATCH_PAGE; if (args->flags & DRM_XE_VM_CREATE_FLAG_LR_MODE) flags |= XE_VM_FLAG_LR_MODE; - if (args->flags & DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT) - flags |= XE_VM_FLAG_ASYNC_DEFAULT; if (args->flags & DRM_XE_VM_CREATE_FLAG_FAULT_MODE) flags |= XE_VM_FLAG_FAULT_MODE; @@ -2263,8 +2248,7 @@ static int xe_vma_op_commit(struct xe_vm *vm, struct xe_vma_op *op) static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct xe_exec_queue *q, struct drm_gpuva_ops *ops, struct xe_sync_entry *syncs, u32 num_syncs, - struct list_head *ops_list, bool last, - bool async) + struct list_head *ops_list, bool last) { struct xe_vma_op *last_op = NULL; struct drm_gpuva_op *__op; @@ -2696,23 +2680,22 @@ static int vm_bind_ioctl_ops_execute(struct xe_vm *vm, #ifdef TEST_VM_ASYNC_OPS_ERROR #define SUPPORTED_FLAGS \ - (FORCE_ASYNC_OP_ERROR | DRM_XE_VM_BIND_FLAG_ASYNC | \ - DRM_XE_VM_BIND_FLAG_READONLY | DRM_XE_VM_BIND_FLAG_IMMEDIATE | \ - DRM_XE_VM_BIND_FLAG_NULL | 0xffff) + (FORCE_ASYNC_OP_ERROR | DRM_XE_VM_BIND_FLAG_READONLY | \ + DRM_XE_VM_BIND_FLAG_IMMEDIATE | DRM_XE_VM_BIND_FLAG_NULL | 0xffff) #else #define SUPPORTED_FLAGS \ - (DRM_XE_VM_BIND_FLAG_ASYNC | DRM_XE_VM_BIND_FLAG_READONLY | \ + (DRM_XE_VM_BIND_FLAG_READONLY | \ DRM_XE_VM_BIND_FLAG_IMMEDIATE | DRM_XE_VM_BIND_FLAG_NULL | \ 0xffff) #endif #define XE_64K_PAGE_MASK 0xffffull +#define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP) #define MAX_BINDS 512 /* FIXME: Picking random upper limit */ static int vm_bind_ioctl_check_args(struct xe_device *xe, struct drm_xe_vm_bind *args, - struct drm_xe_vm_bind_op **bind_ops, - bool *async) + struct drm_xe_vm_bind_op **bind_ops) { int err; int i; @@ -2775,18 +2758,6 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, goto free_bind_ops; } - if (i == 0) { - *async = !!(flags & DRM_XE_VM_BIND_FLAG_ASYNC); - if (XE_IOCTL_DBG(xe, !*async && args->num_syncs)) { - err = -EINVAL; - goto free_bind_ops; - } - } else if (XE_IOCTL_DBG(xe, *async != - !!(flags & DRM_XE_VM_BIND_FLAG_ASYNC))) { - err = -EINVAL; - goto free_bind_ops; - } - if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) || XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) || XE_IOCTL_DBG(xe, obj && is_null) || @@ -2854,14 +2825,6 @@ static int vm_bind_ioctl_signal_fences(struct xe_vm *vm, xe_exec_queue_last_fence_set(to_wait_exec_queue(vm, q), vm, fence); - - if (xe_vm_sync_mode(vm, q)) { - long timeout = dma_fence_wait(fence, true); - - if (timeout < 0) - err = -EINTR; - } - dma_fence_put(fence); return err; @@ -2881,18 +2844,13 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) struct xe_sync_entry *syncs = NULL; struct drm_xe_vm_bind_op *bind_ops; LIST_HEAD(ops_list); - bool async; int err; int i; - err = vm_bind_ioctl_check_args(xe, args, &bind_ops, &async); + err = vm_bind_ioctl_check_args(xe, args, &bind_ops); if (err) return err; - if (XE_IOCTL_DBG(xe, args->pad || args->pad2) || - XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) - return -EINVAL; - if (args->exec_queue_id) { q = xe_exec_queue_lookup(xef, args->exec_queue_id); if (XE_IOCTL_DBG(xe, !q)) { @@ -2904,12 +2862,6 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) err = -EINVAL; goto put_exec_queue; } - - if (XE_IOCTL_DBG(xe, args->num_binds && async != - !!(q->flags & EXEC_QUEUE_FLAG_VM_ASYNC))) { - err = -EINVAL; - goto put_exec_queue; - } } vm = xe_vm_lookup(xef, args->vm_id); @@ -2918,14 +2870,6 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) goto put_exec_queue; } - if (!args->exec_queue_id) { - if (XE_IOCTL_DBG(xe, args->num_binds && async != - !!(vm->flags & XE_VM_FLAG_ASYNC_DEFAULT))) { - err = -EINVAL; - goto put_vm; - } - } - err = down_write_killable(&vm->lock); if (err) goto put_vm; @@ -3060,8 +3004,7 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) err = vm_bind_ioctl_ops_parse(vm, q, ops[i], syncs, num_syncs, &ops_list, - i == args->num_binds - 1, - async); + i == args->num_binds - 1); if (err) goto unwind_ops; } diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index 2e023596cb15..63e8a50b88e9 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -138,13 +138,12 @@ struct xe_vm { */ #define XE_VM_FLAG_64K BIT(0) #define XE_VM_FLAG_LR_MODE BIT(1) -#define XE_VM_FLAG_ASYNC_DEFAULT BIT(2) -#define XE_VM_FLAG_MIGRATION BIT(3) -#define XE_VM_FLAG_SCRATCH_PAGE BIT(4) -#define XE_VM_FLAG_FAULT_MODE BIT(5) -#define XE_VM_FLAG_BANNED BIT(6) -#define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(8, 7), flags) -#define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(8, 7), (tile)->id) +#define XE_VM_FLAG_MIGRATION BIT(2) +#define XE_VM_FLAG_SCRATCH_PAGE BIT(3) +#define XE_VM_FLAG_FAULT_MODE BIT(4) +#define XE_VM_FLAG_BANNED BIT(5) +#define XE_VM_FLAG_TILE_ID(flags) FIELD_GET(GENMASK(7, 6), flags) +#define XE_VM_FLAG_SET_TILE_ID(tile) FIELD_PREP(GENMASK(7, 6), (tile)->id) unsigned long flags; /** @composite_fence_ctx: context composite fence */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 87ff6eaa788e..2338d87dcb7d 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -139,8 +139,7 @@ struct drm_xe_engine_class_instance { * Kernel only classes (not actual hardware engine class). Used for * creating ordered queues of VM bind operations. */ -#define DRM_XE_ENGINE_CLASS_VM_BIND_ASYNC 5 -#define DRM_XE_ENGINE_CLASS_VM_BIND_SYNC 6 +#define DRM_XE_ENGINE_CLASS_VM_BIND 5 /** @engine_class: engine class id */ __u16 engine_class; /** @engine_instance: engine instance id */ @@ -660,7 +659,6 @@ struct drm_xe_vm_create { * still enable recoverable pagefaults if supported by the device. */ #define DRM_XE_VM_CREATE_FLAG_LR_MODE (1 << 1) -#define DRM_XE_VM_CREATE_FLAG_ASYNC_DEFAULT (1 << 2) /* * DRM_XE_VM_CREATE_FLAG_FAULT_MODE requires also * DRM_XE_VM_CREATE_FLAG_LR_MODE. It allows memory to be allocated @@ -668,7 +666,7 @@ struct drm_xe_vm_create { * The xe driver internally uses recoverable pagefaults to implement * this. */ -#define DRM_XE_VM_CREATE_FLAG_FAULT_MODE (1 << 3) +#define DRM_XE_VM_CREATE_FLAG_FAULT_MODE (1 << 2) /** @flags: Flags */ __u32 flags; @@ -776,12 +774,11 @@ struct drm_xe_vm_bind_op { __u32 op; #define DRM_XE_VM_BIND_FLAG_READONLY (1 << 0) -#define DRM_XE_VM_BIND_FLAG_ASYNC (1 << 1) /* * Valid on a faulting VM only, do the MAP operation immediately rather * than deferring the MAP to the page fault handler. */ -#define DRM_XE_VM_BIND_FLAG_IMMEDIATE (1 << 2) +#define DRM_XE_VM_BIND_FLAG_IMMEDIATE (1 << 1) /* * When the NULL flag is set, the page tables are setup with a special * bit which indicates writes are dropped and all reads return zero. In @@ -789,7 +786,7 @@ struct drm_xe_vm_bind_op { * operations, the BO handle MBZ, and the BO offset MBZ. This flag is * intended to implement VK sparse bindings. */ -#define DRM_XE_VM_BIND_FLAG_NULL (1 << 3) +#define DRM_XE_VM_BIND_FLAG_NULL (1 << 2) /** @flags: Bind flags */ __u32 flags; -- cgit v1.2.3 From b0e47225a16f4e1ed53dd769588700a40d7b9950 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:40 +0000 Subject: drm/xe/uapi: Add a comment to each struct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a comment to each struct to complete documentation, ensure all struct appear in the kernel doc, and bind structs to IOCTLs. Reviewed-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 2338d87dcb7d..43cacb168091 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -464,7 +464,8 @@ struct drm_xe_query_topology_mask { }; /** - * struct drm_xe_device_query - main structure to query device information + * struct drm_xe_device_query - Input of &DRM_IOCTL_XE_DEVICE_QUERY - main + * structure to query device information * * If size is set to 0, the driver fills it with the required size for the * requested type of data to query. If size is equal to the required size, @@ -526,6 +527,10 @@ struct drm_xe_device_query { __u64 reserved[2]; }; +/** + * struct drm_xe_gem_create - Input of &DRM_IOCTL_XE_GEM_CREATE - A structure for + * gem creation + */ struct drm_xe_gem_create { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -604,6 +609,9 @@ struct drm_xe_gem_create { __u64 reserved[2]; }; +/** + * struct drm_xe_gem_mmap_offset - Input of &DRM_IOCTL_XE_GEM_MMAP_OFFSET + */ struct drm_xe_gem_mmap_offset { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -621,7 +629,9 @@ struct drm_xe_gem_mmap_offset { __u64 reserved[2]; }; -/** struct drm_xe_ext_set_property - XE set property extension */ +/** + * struct drm_xe_ext_set_property - XE set property extension + */ struct drm_xe_ext_set_property { /** @base: base user extension */ struct drm_xe_user_extension base; @@ -639,6 +649,9 @@ struct drm_xe_ext_set_property { __u64 reserved[2]; }; +/** + * struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE + */ struct drm_xe_vm_create { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -677,6 +690,9 @@ struct drm_xe_vm_create { __u64 reserved[2]; }; +/** + * struct drm_xe_vm_destroy - Input of &DRM_IOCTL_XE_VM_DESTROY + */ struct drm_xe_vm_destroy { /** @vm_id: VM ID */ __u32 vm_id; @@ -688,6 +704,9 @@ struct drm_xe_vm_destroy { __u64 reserved[2]; }; +/** + * struct drm_xe_vm_bind_op + */ struct drm_xe_vm_bind_op { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -804,6 +823,9 @@ struct drm_xe_vm_bind_op { __u64 reserved[3]; }; +/** + * struct drm_xe_vm_bind - Input of &DRM_IOCTL_XE_VM_BIND + */ struct drm_xe_vm_bind { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -862,6 +884,9 @@ struct drm_xe_vm_bind { /* Monitor 64MB contiguous region with 2M sub-granularity */ #define DRM_XE_ACC_GRANULARITY_64M 3 +/** + * struct drm_xe_exec_queue_create - Input of &DRM_IOCTL_XE_EXEC_QUEUE_CREATE + */ struct drm_xe_exec_queue_create { #define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 @@ -904,6 +929,9 @@ struct drm_xe_exec_queue_create { __u64 reserved[2]; }; +/** + * struct drm_xe_exec_queue_get_property - Input of &DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY + */ struct drm_xe_exec_queue_get_property { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -922,6 +950,9 @@ struct drm_xe_exec_queue_get_property { __u64 reserved[2]; }; +/** + * struct drm_xe_exec_queue_destroy - Input of &DRM_IOCTL_XE_EXEC_QUEUE_DESTROY + */ struct drm_xe_exec_queue_destroy { /** @exec_queue_id: Exec queue ID */ __u32 exec_queue_id; @@ -933,6 +964,9 @@ struct drm_xe_exec_queue_destroy { __u64 reserved[2]; }; +/** + * struct drm_xe_sync + */ struct drm_xe_sync { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -967,6 +1001,9 @@ struct drm_xe_sync { __u64 reserved[2]; }; +/** + * struct drm_xe_exec - Input of &DRM_IOCTL_XE_EXEC + */ struct drm_xe_exec { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -1000,7 +1037,7 @@ struct drm_xe_exec { }; /** - * struct drm_xe_wait_user_fence - wait user fence + * struct drm_xe_wait_user_fence - Input of &DRM_IOCTL_XE_WAIT_USER_FENCE * * Wait on user fence, XE will wake-up on every HW engine interrupt in the * instances list and check if user fence is complete:: -- cgit v1.2.3 From 4efaadd38bc4c6c1016996669002994061990633 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:41 +0000 Subject: drm/xe/uapi: Add missing documentation for struct members MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This removes the documentation build warnings below: include/uapi/drm/xe_drm.h:828: warning: Function parameter or \ member 'pad2' not described in 'drm_xe_vm_bind_op' include/uapi/drm/xe_drm.h:875: warning: Function parameter or \ member 'pad2' not described in 'drm_xe_vm_bind' include/uapi/drm/xe_drm.h:1006: warning: Function parameter or \ member 'handle' not described in 'drm_xe_sync' include/uapi/drm/xe_drm.h:1006: warning: Function parameter or \ member 'timeline_value' not described in 'drm_xe_sync' Reviewed-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 43cacb168091..d7893ccbbf8c 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -816,7 +816,7 @@ struct drm_xe_vm_bind_op { */ __u32 prefetch_mem_region_instance; - /** @pad: MBZ */ + /** @pad2: MBZ */ __u32 pad2; /** @reserved: Reserved */ @@ -857,7 +857,7 @@ struct drm_xe_vm_bind { __u64 vector_of_binds; }; - /** @pad: MBZ */ + /** @pad2: MBZ */ __u32 pad2; /** @num_syncs: amount of syncs to wait on */ @@ -982,6 +982,7 @@ struct drm_xe_sync { __u32 flags; union { + /** @handle: Handle for the object */ __u32 handle; /** @@ -995,6 +996,7 @@ struct drm_xe_sync { __u64 addr; }; + /** @timeline_value: Timeline point of the sync object */ __u64 timeline_value; /** @reserved: Reserved */ -- cgit v1.2.3 From ff6c6bc55258e7d0aabcfc41baa392fcedb450a2 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:42 +0000 Subject: drm/xe/uapi: Document use of size in drm_xe_device_query MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Document the behavior of the driver for IOCTL DRM_IOCTL_XE_DEVICE_QUERY depending on the size value provided in struct drm_xe_device_query. Reviewed-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index d7893ccbbf8c..d759e04e00ee 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -467,9 +467,15 @@ struct drm_xe_query_topology_mask { * struct drm_xe_device_query - Input of &DRM_IOCTL_XE_DEVICE_QUERY - main * structure to query device information * - * If size is set to 0, the driver fills it with the required size for the - * requested type of data to query. If size is equal to the required size, - * the queried information is copied into data. + * The user selects the type of data to query among DRM_XE_DEVICE_QUERY_* + * and sets the value in the query member. This determines the type of + * the structure provided by the driver in data, among struct drm_xe_query_*. + * + * If size is set to 0, the driver fills it with the required size for + * the requested type of data to query. If size is equal to the required + * size, the queried information is copied into data. If size is set to + * a value different from 0 and different from the required size, the + * IOCTL call returns -EINVAL. * * For example the following code snippet allows retrieving and printing * information about the device engines with DRM_XE_DEVICE_QUERY_ENGINES: -- cgit v1.2.3 From af8ea4162b4cb6e83bfabaef3db3bf89d2a07cbc Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:43 +0000 Subject: drm/xe/uapi: Document drm_xe_query_config keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Provide a description of the keys used the struct drm_xe_query_config info array. Closes: https://gitlab.freedesktop.org/drm/xe/kernel/-/issues/637 Reviewed-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index d759e04e00ee..9c43bc258f10 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -350,11 +350,31 @@ struct drm_xe_query_config { /** @pad: MBZ */ __u32 pad; + /* + * Device ID (lower 16 bits) and the device revision (next + * 8 bits) + */ #define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 + /* + * Flags describing the device configuration, see list below + */ #define DRM_XE_QUERY_CONFIG_FLAGS 1 + /* + * Flag is set if the device has usable VRAM + */ #define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) + /* + * Minimal memory alignment required by this device, + * typically SZ_4K or SZ_64K + */ #define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2 + /* + * Maximum bits of a virtual address + */ #define DRM_XE_QUERY_CONFIG_VA_BITS 3 + /* + * Value of the highest available exec queue priority + */ #define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4 /** @info: array of elements containing the config info */ __u64 info[]; -- cgit v1.2.3 From 37958604e69485e9704f8483401b03679e3e4939 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:44 +0000 Subject: drm/xe/uapi: Document DRM_XE_DEVICE_QUERY_HWCONFIG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a documentation on the content and format of when using query type DRM_XE_DEVICE_QUERY_HWCONFIG. The list of keys can be found in IGT under lib/intel_hwconfig_types.h. Reviewed-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 9c43bc258f10..70b42466a811 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -537,6 +537,11 @@ struct drm_xe_device_query { #define DRM_XE_DEVICE_QUERY_MEM_REGIONS 1 #define DRM_XE_DEVICE_QUERY_CONFIG 2 #define DRM_XE_DEVICE_QUERY_GT_LIST 3 + /* + * Query type to retrieve the hardware configuration of the device + * such as information on slices, memory, caches, and so on. It is + * provided as a table of attributes (key / value). + */ #define DRM_XE_DEVICE_QUERY_HWCONFIG 4 #define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5 #define DRM_XE_DEVICE_QUERY_ENGINE_CYCLES 6 -- cgit v1.2.3 From 801989b08aff35ef56743551f4cfeaed360bd201 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:45 +0000 Subject: drm/xe/uapi: Make constant comments visible in kernel doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As there is no direct way to make comments of constants directly visible in the kernel doc, move them to the description of the structure where they can be used. By doing so they appear in the "Description" section of the struct documentation. v2: Remove DRM_XE_UFENCE_WAIT_MASK_* (Francois Dugast) Reviewed-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 271 ++++++++++++++++++++++++++-------------------- 1 file changed, 155 insertions(+), 116 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 70b42466a811..4c11dec57a83 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -128,6 +128,16 @@ struct drm_xe_user_extension { * It is returned as part of the @drm_xe_engine, but it also is used as * the input of engine selection for both @drm_xe_exec_queue_create and * @drm_xe_query_engine_cycles + * + * The @engine_class can be: + * - %DRM_XE_ENGINE_CLASS_RENDER + * - %DRM_XE_ENGINE_CLASS_COPY + * - %DRM_XE_ENGINE_CLASS_VIDEO_DECODE + * - %DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE + * - %DRM_XE_ENGINE_CLASS_COMPUTE + * - %DRM_XE_ENGINE_CLASS_VM_BIND - Kernel only classes (not actual + * hardware engine class). Used for creating ordered queues of VM + * bind operations. */ struct drm_xe_engine_class_instance { #define DRM_XE_ENGINE_CLASS_RENDER 0 @@ -135,10 +145,6 @@ struct drm_xe_engine_class_instance { #define DRM_XE_ENGINE_CLASS_VIDEO_DECODE 2 #define DRM_XE_ENGINE_CLASS_VIDEO_ENHANCE 3 #define DRM_XE_ENGINE_CLASS_COMPUTE 4 - /* - * Kernel only classes (not actual hardware engine class). Used for - * creating ordered queues of VM bind operations. - */ #define DRM_XE_ENGINE_CLASS_VM_BIND 5 /** @engine_class: engine class id */ __u16 engine_class; @@ -342,6 +348,19 @@ struct drm_xe_query_mem_regions { * is equal to DRM_XE_DEVICE_QUERY_CONFIG, then the reply uses * struct drm_xe_query_config in .data. * + * The index in @info can be: + * - %DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID - Device ID (lower 16 bits) + * and the device revision (next 8 bits) + * - %DRM_XE_QUERY_CONFIG_FLAGS - Flags describing the device + * configuration, see list below + * + * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device + * has usable VRAM + * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment + * required by this device, typically SZ_4K or SZ_64K + * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address + * - %DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY - Value of the highest + * available exec queue priority */ struct drm_xe_query_config { /** @num_params: number of parameters returned in info */ @@ -350,31 +369,11 @@ struct drm_xe_query_config { /** @pad: MBZ */ __u32 pad; - /* - * Device ID (lower 16 bits) and the device revision (next - * 8 bits) - */ #define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 - /* - * Flags describing the device configuration, see list below - */ #define DRM_XE_QUERY_CONFIG_FLAGS 1 - /* - * Flag is set if the device has usable VRAM - */ #define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) - /* - * Minimal memory alignment required by this device, - * typically SZ_4K or SZ_64K - */ #define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2 - /* - * Maximum bits of a virtual address - */ #define DRM_XE_QUERY_CONFIG_VA_BITS 3 - /* - * Value of the highest available exec queue priority - */ #define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4 /** @info: array of elements containing the config info */ __u64 info[]; @@ -387,6 +386,10 @@ struct drm_xe_query_config { * existing GT individual descriptions. * Graphics Technology (GT) is a subset of a GPU/tile that is responsible for * implementing graphics and/or media operations. + * + * The index in @type can be: + * - %DRM_XE_QUERY_GT_TYPE_MAIN + * - %DRM_XE_QUERY_GT_TYPE_MEDIA */ struct drm_xe_gt { #define DRM_XE_QUERY_GT_TYPE_MAIN 0 @@ -444,34 +447,30 @@ struct drm_xe_query_gt_list { * If a query is made with a struct drm_xe_device_query where .query * is equal to DRM_XE_DEVICE_QUERY_GT_TOPOLOGY, then the reply uses * struct drm_xe_query_topology_mask in .data. + * + * The @type can be: + * - %DRM_XE_TOPO_DSS_GEOMETRY - To query the mask of Dual Sub Slices + * (DSS) available for geometry operations. For example a query response + * containing the following in mask: + * ``DSS_GEOMETRY ff ff ff ff 00 00 00 00`` + * means 32 DSS are available for geometry. + * - %DRM_XE_TOPO_DSS_COMPUTE - To query the mask of Dual Sub Slices + * (DSS) available for compute operations. For example a query response + * containing the following in mask: + * ``DSS_COMPUTE ff ff ff ff 00 00 00 00`` + * means 32 DSS are available for compute. + * - %DRM_XE_TOPO_EU_PER_DSS - To query the mask of Execution Units (EU) + * available per Dual Sub Slices (DSS). For example a query response + * containing the following in mask: + * ``EU_PER_DSS ff ff 00 00 00 00 00 00`` + * means each DSS has 16 EU. */ struct drm_xe_query_topology_mask { /** @gt_id: GT ID the mask is associated with */ __u16 gt_id; - /* - * To query the mask of Dual Sub Slices (DSS) available for geometry - * operations. For example a query response containing the following - * in mask: - * DSS_GEOMETRY ff ff ff ff 00 00 00 00 - * means 32 DSS are available for geometry. - */ #define DRM_XE_TOPO_DSS_GEOMETRY (1 << 0) - /* - * To query the mask of Dual Sub Slices (DSS) available for compute - * operations. For example a query response containing the following - * in mask: - * DSS_COMPUTE ff ff ff ff 00 00 00 00 - * means 32 DSS are available for compute. - */ #define DRM_XE_TOPO_DSS_COMPUTE (1 << 1) - /* - * To query the mask of Execution Units (EU) available per Dual Sub - * Slices (DSS). For example a query response containing the following - * in mask: - * EU_PER_DSS ff ff 00 00 00 00 00 00 - * means each DSS has 16 EU. - */ #define DRM_XE_TOPO_EU_PER_DSS (1 << 2) /** @type: type of mask */ __u16 type; @@ -491,6 +490,18 @@ struct drm_xe_query_topology_mask { * and sets the value in the query member. This determines the type of * the structure provided by the driver in data, among struct drm_xe_query_*. * + * The @query can be: + * - %DRM_XE_DEVICE_QUERY_ENGINES + * - %DRM_XE_DEVICE_QUERY_MEM_REGIONS + * - %DRM_XE_DEVICE_QUERY_CONFIG + * - %DRM_XE_DEVICE_QUERY_GT_LIST + * - %DRM_XE_DEVICE_QUERY_HWCONFIG - Query type to retrieve the hardware + * configuration of the device such as information on slices, memory, + * caches, and so on. It is provided as a table of key / value + * attributes. + * - %DRM_XE_DEVICE_QUERY_GT_TOPOLOGY + * - %DRM_XE_DEVICE_QUERY_ENGINE_CYCLES + * * If size is set to 0, the driver fills it with the required size for * the requested type of data to query. If size is equal to the required * size, the queried information is copied into data. If size is set to @@ -537,11 +548,6 @@ struct drm_xe_device_query { #define DRM_XE_DEVICE_QUERY_MEM_REGIONS 1 #define DRM_XE_DEVICE_QUERY_CONFIG 2 #define DRM_XE_DEVICE_QUERY_GT_LIST 3 - /* - * Query type to retrieve the hardware configuration of the device - * such as information on slices, memory, caches, and so on. It is - * provided as a table of attributes (key / value). - */ #define DRM_XE_DEVICE_QUERY_HWCONFIG 4 #define DRM_XE_DEVICE_QUERY_GT_TOPOLOGY 5 #define DRM_XE_DEVICE_QUERY_ENGINE_CYCLES 6 @@ -561,6 +567,33 @@ struct drm_xe_device_query { /** * struct drm_xe_gem_create - Input of &DRM_IOCTL_XE_GEM_CREATE - A structure for * gem creation + * + * The @flags can be: + * - %DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING + * - %DRM_XE_GEM_CREATE_FLAG_SCANOUT + * - %DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM - When using VRAM as a + * possible placement, ensure that the corresponding VRAM allocation + * will always use the CPU accessible part of VRAM. This is important + * for small-bar systems (on full-bar systems this gets turned into a + * noop). + * Note1: System memory can be used as an extra placement if the kernel + * should spill the allocation to system memory, if space can't be made + * available in the CPU accessible part of VRAM (giving the same + * behaviour as the i915 interface, see + * I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS). + * Note2: For clear-color CCS surfaces the kernel needs to read the + * clear-color value stored in the buffer, and on discrete platforms we + * need to use VRAM for display surfaces, therefore the kernel requires + * setting this flag for such objects, otherwise an error is thrown on + * small-bar systems. + * + * @cpu_caching supports the following values: + * - %DRM_XE_GEM_CPU_CACHING_WB - Allocate the pages with write-back + * caching. On iGPU this can't be used for scanout surfaces. Currently + * not allowed for objects placed in VRAM. + * - %DRM_XE_GEM_CPU_CACHING_WC - Allocate the pages as write-combined. This + * is uncached. Scanout surfaces should likely use this. All objects + * that can be placed in VRAM must use this. */ struct drm_xe_gem_create { /** @extensions: Pointer to the first extension struct, if any */ @@ -577,21 +610,6 @@ struct drm_xe_gem_create { #define DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING (1 << 0) #define DRM_XE_GEM_CREATE_FLAG_SCANOUT (1 << 1) -/* - * When using VRAM as a possible placement, ensure that the corresponding VRAM - * allocation will always use the CPU accessible part of VRAM. This is important - * for small-bar systems (on full-bar systems this gets turned into a noop). - * - * Note: System memory can be used as an extra placement if the kernel should - * spill the allocation to system memory, if space can't be made available in - * the CPU accessible part of VRAM (giving the same behaviour as the i915 - * interface, see I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS). - * - * Note: For clear-color CCS surfaces the kernel needs to read the clear-color - * value stored in the buffer, and on discrete platforms we need to use VRAM for - * display surfaces, therefore the kernel requires setting this flag for such - * objects, otherwise an error is thrown on small-bar systems. - */ #define DRM_XE_GEM_CREATE_FLAG_NEEDS_VISIBLE_VRAM (1 << 2) /** * @flags: Flags, currently a mask of memory instances of where BO can @@ -619,16 +637,6 @@ struct drm_xe_gem_create { /** * @cpu_caching: The CPU caching mode to select for this object. If * mmaping the object the mode selected here will also be used. - * - * Supported values: - * - * DRM_XE_GEM_CPU_CACHING_WB: Allocate the pages with write-back - * caching. On iGPU this can't be used for scanout surfaces. Currently - * not allowed for objects placed in VRAM. - * - * DRM_XE_GEM_CPU_CACHING_WC: Allocate the pages as write-combined. This - * is uncached. Scanout surfaces should likely use this. All objects - * that can be placed in VRAM must use this. */ #define DRM_XE_GEM_CPU_CACHING_WB 1 #define DRM_XE_GEM_CPU_CACHING_WC 2 @@ -682,34 +690,33 @@ struct drm_xe_ext_set_property { /** * struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE + * + * The @flags can be: + * - %DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE + * - %DRM_XE_VM_CREATE_FLAG_LR_MODE - An LR, or Long Running VM accepts + * exec submissions to its exec_queues that don't have an upper time + * limit on the job execution time. But exec submissions to these + * don't allow any of the flags DRM_XE_SYNC_FLAG_SYNCOBJ, + * DRM_XE_SYNC_FLAG_TIMELINE_SYNCOBJ, DRM_XE_SYNC_FLAG_DMA_BUF, + * used as out-syncobjs, that is, together with DRM_XE_SYNC_FLAG_SIGNAL. + * LR VMs can be created in recoverable page-fault mode using + * DRM_XE_VM_CREATE_FLAG_FAULT_MODE, if the device supports it. + * If that flag is omitted, the UMD can not rely on the slightly + * different per-VM overcommit semantics that are enabled by + * DRM_XE_VM_CREATE_FLAG_FAULT_MODE (see below), but KMD may + * still enable recoverable pagefaults if supported by the device. + * - %DRM_XE_VM_CREATE_FLAG_FAULT_MODE - Requires also + * DRM_XE_VM_CREATE_FLAG_LR_MODE. It allows memory to be allocated on + * demand when accessed, and also allows per-VM overcommit of memory. + * The xe driver internally uses recoverable pagefaults to implement + * this. */ struct drm_xe_vm_create { /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; #define DRM_XE_VM_CREATE_FLAG_SCRATCH_PAGE (1 << 0) - /* - * An LR, or Long Running VM accepts exec submissions - * to its exec_queues that don't have an upper time limit on - * the job execution time. But exec submissions to these - * don't allow any of the flags DRM_XE_SYNC_FLAG_SYNCOBJ, - * DRM_XE_SYNC_FLAG_TIMELINE_SYNCOBJ, DRM_XE_SYNC_FLAG_DMA_BUF, - * used as out-syncobjs, that is, together with DRM_XE_SYNC_FLAG_SIGNAL. - * LR VMs can be created in recoverable page-fault mode using - * DRM_XE_VM_CREATE_FLAG_FAULT_MODE, if the device supports it. - * If that flag is omitted, the UMD can not rely on the slightly - * different per-VM overcommit semantics that are enabled by - * DRM_XE_VM_CREATE_FLAG_FAULT_MODE (see below), but KMD may - * still enable recoverable pagefaults if supported by the device. - */ #define DRM_XE_VM_CREATE_FLAG_LR_MODE (1 << 1) - /* - * DRM_XE_VM_CREATE_FLAG_FAULT_MODE requires also - * DRM_XE_VM_CREATE_FLAG_LR_MODE. It allows memory to be allocated - * on demand when accessed, and also allows per-VM overcommit of memory. - * The xe driver internally uses recoverable pagefaults to implement - * this. - */ #define DRM_XE_VM_CREATE_FLAG_FAULT_MODE (1 << 2) /** @flags: Flags */ __u32 flags; @@ -736,7 +743,27 @@ struct drm_xe_vm_destroy { }; /** - * struct drm_xe_vm_bind_op + * struct drm_xe_vm_bind_op - run bind operations + * + * The @op can be: + * - %DRM_XE_VM_BIND_OP_MAP + * - %DRM_XE_VM_BIND_OP_UNMAP + * - %DRM_XE_VM_BIND_OP_MAP_USERPTR + * - %DRM_XE_VM_BIND_OP_UNMAP_ALL + * - %DRM_XE_VM_BIND_OP_PREFETCH + * + * and the @flags can be: + * - %DRM_XE_VM_BIND_FLAG_READONLY + * - %DRM_XE_VM_BIND_FLAG_ASYNC + * - %DRM_XE_VM_BIND_FLAG_IMMEDIATE - Valid on a faulting VM only, do the + * MAP operation immediately rather than deferring the MAP to the page + * fault handler. + * - %DRM_XE_VM_BIND_FLAG_NULL - When the NULL flag is set, the page + * tables are setup with a special bit which indicates writes are + * dropped and all reads return zero. In the future, the NULL flags + * will only be valid for DRM_XE_VM_BIND_OP_MAP operations, the BO + * handle MBZ, and the BO offset MBZ. This flag is intended to + * implement VK sparse bindings. */ struct drm_xe_vm_bind_op { /** @extensions: Pointer to the first extension struct, if any */ @@ -824,18 +851,7 @@ struct drm_xe_vm_bind_op { __u32 op; #define DRM_XE_VM_BIND_FLAG_READONLY (1 << 0) - /* - * Valid on a faulting VM only, do the MAP operation immediately rather - * than deferring the MAP to the page fault handler. - */ #define DRM_XE_VM_BIND_FLAG_IMMEDIATE (1 << 1) - /* - * When the NULL flag is set, the page tables are setup with a special - * bit which indicates writes are dropped and all reads return zero. In - * the future, the NULL flags will only be valid for DRM_XE_VM_BIND_OP_MAP - * operations, the BO handle MBZ, and the BO offset MBZ. This flag is - * intended to implement VK sparse bindings. - */ #define DRM_XE_VM_BIND_FLAG_NULL (1 << 2) /** @flags: Bind flags */ __u32 flags; @@ -962,6 +978,9 @@ struct drm_xe_exec_queue_create { /** * struct drm_xe_exec_queue_get_property - Input of &DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY + * + * The @property can be: + * - %DRM_XE_EXEC_QUEUE_GET_PROPERTY_BAN */ struct drm_xe_exec_queue_get_property { /** @extensions: Pointer to the first extension struct, if any */ @@ -996,7 +1015,15 @@ struct drm_xe_exec_queue_destroy { }; /** - * struct drm_xe_sync + * struct drm_xe_sync - sync object + * + * The @type can be: + * - %DRM_XE_SYNC_TYPE_SYNCOBJ + * - %DRM_XE_SYNC_TYPE_TIMELINE_SYNCOBJ + * - %DRM_XE_SYNC_TYPE_USER_FENCE + * + * and the @flags can be: + * - %DRM_XE_SYNC_FLAG_SIGNAL */ struct drm_xe_sync { /** @extensions: Pointer to the first extension struct, if any */ @@ -1078,6 +1105,24 @@ struct drm_xe_exec { * (*addr & MASK) OP (VALUE & MASK) * * Returns to user on user fence completion or timeout. + * + * The @op can be: + * - %DRM_XE_UFENCE_WAIT_OP_EQ + * - %DRM_XE_UFENCE_WAIT_OP_NEQ + * - %DRM_XE_UFENCE_WAIT_OP_GT + * - %DRM_XE_UFENCE_WAIT_OP_GTE + * - %DRM_XE_UFENCE_WAIT_OP_LT + * - %DRM_XE_UFENCE_WAIT_OP_LTE + * + * and the @flags can be: + * - %DRM_XE_UFENCE_WAIT_FLAG_ABSTIME + * - %DRM_XE_UFENCE_WAIT_FLAG_SOFT_OP + * + * The @mask values can be for example: + * - 0xffu for u8 + * - 0xffffu for u16 + * - 0xffffffffu for u32 + * - 0xffffffffffffffffu for u64 */ struct drm_xe_wait_user_fence { /** @extensions: Pointer to the first extension struct, if any */ @@ -1107,13 +1152,7 @@ struct drm_xe_wait_user_fence { /** @value: compare value */ __u64 value; - /** - * @mask: comparison mask, values can be for example: - * - 0xffu for u8 - * - 0xffffu for u16 - * - 0xffffffffu for u32 - * - 0xffffffffffffffffu for u64 - */ + /** @mask: comparison mask */ __u64 mask; /** -- cgit v1.2.3 From 76ca3a22c00bed8a43afd14de4b42691f224801b Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Fri, 15 Dec 2023 15:45:46 +0000 Subject: drm/xe/uapi: Order sections MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch doesn't modify any text or uapi entries themselves. It only move things up and down aiming a better organization of the uAPI. While fixing the documentation I noticed that query_engine_cs_cycles was in the middle of the memory_region info. Then I noticed more mismatches on the order when compared to the order of the IOCTL and QUERY entries declaration. So this patch aims to bring some order to the uAPI so it gets easier to read and the documentation generated in the end is able to tell a consistent story. Overall order: 1. IOCTL definition 2. Extension definition and helper structs 3. IOCTL's Query structs in the order of the Query's entries. 4. The rest of IOCTL structs in the order of IOCTL declaration. 5. uEvents Signed-off-by: Rodrigo Vivi Reviewed-by: Lucas De Marchi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast --- include/uapi/drm/xe_drm.h | 252 ++++++++++++++++++++++++---------------------- 1 file changed, 130 insertions(+), 122 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 4c11dec57a83..b62dd51fa895 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -12,19 +12,48 @@ extern "C" { #endif -/* Please note that modifications to all structs defined here are +/* + * Please note that modifications to all structs defined here are * subject to backwards-compatibility constraints. + * Sections in this file are organized as follows: + * 1. IOCTL definition + * 2. Extension definition and helper structs + * 3. IOCTL's Query structs in the order of the Query's entries. + * 4. The rest of IOCTL structs in the order of IOCTL declaration. + * 5. uEvents */ -/** - * DOC: uevent generated by xe on it's pci node. +/* + * xe specific ioctls. * - * DRM_XE_RESET_FAILED_UEVENT - Event is generated when attempt to reset gt - * fails. The value supplied with the event is always "NEEDS_RESET". - * Additional information supplied is tile id and gt id of the gt unit for - * which reset has failed. + * The device specific ioctl range is [DRM_COMMAND_BASE, DRM_COMMAND_END) ie + * [0x40, 0xa0) (a0 is excluded). The numbers below are defined as offset + * against DRM_COMMAND_BASE and should be between [0x0, 0x60). */ -#define DRM_XE_RESET_FAILED_UEVENT "DEVICE_STATUS" +#define DRM_XE_DEVICE_QUERY 0x00 +#define DRM_XE_GEM_CREATE 0x01 +#define DRM_XE_GEM_MMAP_OFFSET 0x02 +#define DRM_XE_VM_CREATE 0x03 +#define DRM_XE_VM_DESTROY 0x04 +#define DRM_XE_VM_BIND 0x05 +#define DRM_XE_EXEC_QUEUE_CREATE 0x06 +#define DRM_XE_EXEC_QUEUE_DESTROY 0x07 +#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x08 +#define DRM_XE_EXEC 0x09 +#define DRM_XE_WAIT_USER_FENCE 0x0a +/* Must be kept compact -- no holes */ + +#define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query) +#define DRM_IOCTL_XE_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_CREATE, struct drm_xe_gem_create) +#define DRM_IOCTL_XE_GEM_MMAP_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_MMAP_OFFSET, struct drm_xe_gem_mmap_offset) +#define DRM_IOCTL_XE_VM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_CREATE, struct drm_xe_vm_create) +#define DRM_IOCTL_XE_VM_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) +#define DRM_IOCTL_XE_VM_BIND DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) +#define DRM_IOCTL_XE_EXEC_QUEUE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_CREATE, struct drm_xe_exec_queue_create) +#define DRM_IOCTL_XE_EXEC_QUEUE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_DESTROY, struct drm_xe_exec_queue_destroy) +#define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property) +#define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) +#define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) /** * struct drm_xe_user_extension - Base class for defining a chain of extensions @@ -90,37 +119,25 @@ struct drm_xe_user_extension { __u32 pad; }; -/* - * xe specific ioctls. - * - * The device specific ioctl range is [DRM_COMMAND_BASE, DRM_COMMAND_END) ie - * [0x40, 0xa0) (a0 is excluded). The numbers below are defined as offset - * against DRM_COMMAND_BASE and should be between [0x0, 0x60). +/** + * struct drm_xe_ext_set_property - XE set property extension */ -#define DRM_XE_DEVICE_QUERY 0x00 -#define DRM_XE_GEM_CREATE 0x01 -#define DRM_XE_GEM_MMAP_OFFSET 0x02 -#define DRM_XE_VM_CREATE 0x03 -#define DRM_XE_VM_DESTROY 0x04 -#define DRM_XE_VM_BIND 0x05 -#define DRM_XE_EXEC_QUEUE_CREATE 0x06 -#define DRM_XE_EXEC_QUEUE_DESTROY 0x07 -#define DRM_XE_EXEC_QUEUE_GET_PROPERTY 0x08 -#define DRM_XE_EXEC 0x09 -#define DRM_XE_WAIT_USER_FENCE 0x0a -/* Must be kept compact -- no holes */ +struct drm_xe_ext_set_property { + /** @base: base user extension */ + struct drm_xe_user_extension base; -#define DRM_IOCTL_XE_DEVICE_QUERY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_DEVICE_QUERY, struct drm_xe_device_query) -#define DRM_IOCTL_XE_GEM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_CREATE, struct drm_xe_gem_create) -#define DRM_IOCTL_XE_GEM_MMAP_OFFSET DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_GEM_MMAP_OFFSET, struct drm_xe_gem_mmap_offset) -#define DRM_IOCTL_XE_VM_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_VM_CREATE, struct drm_xe_vm_create) -#define DRM_IOCTL_XE_VM_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_DESTROY, struct drm_xe_vm_destroy) -#define DRM_IOCTL_XE_VM_BIND DRM_IOW(DRM_COMMAND_BASE + DRM_XE_VM_BIND, struct drm_xe_vm_bind) -#define DRM_IOCTL_XE_EXEC_QUEUE_CREATE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_CREATE, struct drm_xe_exec_queue_create) -#define DRM_IOCTL_XE_EXEC_QUEUE_DESTROY DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_DESTROY, struct drm_xe_exec_queue_destroy) -#define DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_EXEC_QUEUE_GET_PROPERTY, struct drm_xe_exec_queue_get_property) -#define DRM_IOCTL_XE_EXEC DRM_IOW(DRM_COMMAND_BASE + DRM_XE_EXEC, struct drm_xe_exec) -#define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) + /** @property: property to set */ + __u32 property; + + /** @pad: MBZ */ + __u32 pad; + + /** @value: property value */ + __u64 value; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; /** * struct drm_xe_engine_class_instance - instance of an engine class @@ -274,57 +291,6 @@ struct drm_xe_mem_region { __u64 reserved[6]; }; -/** - * struct drm_xe_query_engine_cycles - correlate CPU and GPU timestamps - * - * If a query is made with a struct drm_xe_device_query where .query is equal to - * DRM_XE_DEVICE_QUERY_ENGINE_CYCLES, then the reply uses struct drm_xe_query_engine_cycles - * in .data. struct drm_xe_query_engine_cycles is allocated by the user and - * .data points to this allocated structure. - * - * The query returns the engine cycles, which along with GT's @reference_clock, - * can be used to calculate the engine timestamp. In addition the - * query returns a set of cpu timestamps that indicate when the command - * streamer cycle count was captured. - */ -struct drm_xe_query_engine_cycles { - /** - * @eci: This is input by the user and is the engine for which command - * streamer cycles is queried. - */ - struct drm_xe_engine_class_instance eci; - - /** - * @clockid: This is input by the user and is the reference clock id for - * CPU timestamp. For definition, see clock_gettime(2) and - * perf_event_open(2). Supported clock ids are CLOCK_MONOTONIC, - * CLOCK_MONOTONIC_RAW, CLOCK_REALTIME, CLOCK_BOOTTIME, CLOCK_TAI. - */ - __s32 clockid; - - /** @width: Width of the engine cycle counter in bits. */ - __u32 width; - - /** - * @engine_cycles: Engine cycles as read from its register - * at 0x358 offset. - */ - __u64 engine_cycles; - - /** - * @cpu_timestamp: CPU timestamp in ns. The timestamp is captured before - * reading the engine_cycles register using the reference clockid set by the - * user. - */ - __u64 cpu_timestamp; - - /** - * @cpu_delta: Time delta in ns captured around reading the lower dword - * of the engine_cycles register. - */ - __u64 cpu_delta; -}; - /** * struct drm_xe_query_mem_regions - describe memory regions * @@ -482,6 +448,57 @@ struct drm_xe_query_topology_mask { __u8 mask[]; }; +/** + * struct drm_xe_query_engine_cycles - correlate CPU and GPU timestamps + * + * If a query is made with a struct drm_xe_device_query where .query is equal to + * DRM_XE_DEVICE_QUERY_ENGINE_CYCLES, then the reply uses struct drm_xe_query_engine_cycles + * in .data. struct drm_xe_query_engine_cycles is allocated by the user and + * .data points to this allocated structure. + * + * The query returns the engine cycles, which along with GT's @reference_clock, + * can be used to calculate the engine timestamp. In addition the + * query returns a set of cpu timestamps that indicate when the command + * streamer cycle count was captured. + */ +struct drm_xe_query_engine_cycles { + /** + * @eci: This is input by the user and is the engine for which command + * streamer cycles is queried. + */ + struct drm_xe_engine_class_instance eci; + + /** + * @clockid: This is input by the user and is the reference clock id for + * CPU timestamp. For definition, see clock_gettime(2) and + * perf_event_open(2). Supported clock ids are CLOCK_MONOTONIC, + * CLOCK_MONOTONIC_RAW, CLOCK_REALTIME, CLOCK_BOOTTIME, CLOCK_TAI. + */ + __s32 clockid; + + /** @width: Width of the engine cycle counter in bits. */ + __u32 width; + + /** + * @engine_cycles: Engine cycles as read from its register + * at 0x358 offset. + */ + __u64 engine_cycles; + + /** + * @cpu_timestamp: CPU timestamp in ns. The timestamp is captured before + * reading the engine_cycles register using the reference clockid set by the + * user. + */ + __u64 cpu_timestamp; + + /** + * @cpu_delta: Time delta in ns captured around reading the lower dword + * of the engine_cycles register. + */ + __u64 cpu_delta; +}; + /** * struct drm_xe_device_query - Input of &DRM_IOCTL_XE_DEVICE_QUERY - main * structure to query device information @@ -668,26 +685,6 @@ struct drm_xe_gem_mmap_offset { __u64 reserved[2]; }; -/** - * struct drm_xe_ext_set_property - XE set property extension - */ -struct drm_xe_ext_set_property { - /** @base: base user extension */ - struct drm_xe_user_extension base; - - /** @property: property to set */ - __u32 property; - - /** @pad: MBZ */ - __u32 pad; - - /** @value: property value */ - __u64 value; - - /** @reserved: Reserved */ - __u64 reserved[2]; -}; - /** * struct drm_xe_vm_create - Input of &DRM_IOCTL_XE_VM_CREATE * @@ -976,6 +973,20 @@ struct drm_xe_exec_queue_create { __u64 reserved[2]; }; +/** + * struct drm_xe_exec_queue_destroy - Input of &DRM_IOCTL_XE_EXEC_QUEUE_DESTROY + */ +struct drm_xe_exec_queue_destroy { + /** @exec_queue_id: Exec queue ID */ + __u32 exec_queue_id; + + /** @pad: MBZ */ + __u32 pad; + + /** @reserved: Reserved */ + __u64 reserved[2]; +}; + /** * struct drm_xe_exec_queue_get_property - Input of &DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY * @@ -1000,20 +1011,6 @@ struct drm_xe_exec_queue_get_property { __u64 reserved[2]; }; -/** - * struct drm_xe_exec_queue_destroy - Input of &DRM_IOCTL_XE_EXEC_QUEUE_DESTROY - */ -struct drm_xe_exec_queue_destroy { - /** @exec_queue_id: Exec queue ID */ - __u32 exec_queue_id; - - /** @pad: MBZ */ - __u32 pad; - - /** @reserved: Reserved */ - __u64 reserved[2]; -}; - /** * struct drm_xe_sync - sync object * @@ -1180,6 +1177,17 @@ struct drm_xe_wait_user_fence { /** @reserved: Reserved */ __u64 reserved[2]; }; + +/** + * DOC: uevent generated by xe on it's pci node. + * + * DRM_XE_RESET_FAILED_UEVENT - Event is generated when attempt to reset gt + * fails. The value supplied with the event is always "NEEDS_RESET". + * Additional information supplied is tile id and gt id of the gt unit for + * which reset has failed. + */ +#define DRM_XE_RESET_FAILED_UEVENT "DEVICE_STATUS" + #if defined(__cplusplus) } #endif -- cgit v1.2.3 From 4b437893a826b2f1d15f73e72506349656ea14b2 Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Fri, 15 Dec 2023 15:45:47 +0000 Subject: drm/xe/uapi: More uAPI documentation additions and cosmetic updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit No functional change in this patch. Let's ensure all of our structs are documented and with a certain standard. Also, let's have an overview and list of IOCTLs as the very beginning of the generated HTML doc. v2: Nits (Lucas De Marchi) Signed-off-by: Rodrigo Vivi Reviewed-by: Lucas De Marchi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast --- include/uapi/drm/xe_drm.h | 47 ++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 7 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index b62dd51fa895..5a01d033b780 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -23,6 +23,27 @@ extern "C" { * 5. uEvents */ +/** + * DOC: Xe uAPI Overview + * + * This section aims to describe the Xe's IOCTL entries, its structs, and other + * Xe related uAPI such as uevents and PMU (Platform Monitoring Unit) related + * entries and usage. + * + * List of supported IOCTLs: + * - &DRM_IOCTL_XE_DEVICE_QUERY + * - &DRM_IOCTL_XE_GEM_CREATE + * - &DRM_IOCTL_XE_GEM_MMAP_OFFSET + * - &DRM_IOCTL_XE_VM_CREATE + * - &DRM_IOCTL_XE_VM_DESTROY + * - &DRM_IOCTL_XE_VM_BIND + * - &DRM_IOCTL_XE_EXEC_QUEUE_CREATE + * - &DRM_IOCTL_XE_EXEC_QUEUE_DESTROY + * - &DRM_IOCTL_XE_EXEC_QUEUE_GET_PROPERTY + * - &DRM_IOCTL_XE_EXEC + * - &DRM_IOCTL_XE_WAIT_USER_FENCE + */ + /* * xe specific ioctls. * @@ -56,7 +77,10 @@ extern "C" { #define DRM_IOCTL_XE_WAIT_USER_FENCE DRM_IOWR(DRM_COMMAND_BASE + DRM_XE_WAIT_USER_FENCE, struct drm_xe_wait_user_fence) /** - * struct drm_xe_user_extension - Base class for defining a chain of extensions + * DOC: Xe IOCTL Extensions + * + * Before detailing the IOCTLs and its structs, it is important to highlight + * that every IOCTL in Xe is extensible. * * Many interfaces need to grow over time. In most cases we can simply * extend the struct and have userspace pass in more data. Another option, @@ -90,7 +114,10 @@ extern "C" { * Typically the struct drm_xe_user_extension would be embedded in some uAPI * struct, and in this case we would feed it the head of the chain(i.e ext1), * which would then apply all of the above extensions. - * +*/ + +/** + * struct drm_xe_user_extension - Base class for defining a chain of extensions */ struct drm_xe_user_extension { /** @@ -120,7 +147,10 @@ struct drm_xe_user_extension { }; /** - * struct drm_xe_ext_set_property - XE set property extension + * struct drm_xe_ext_set_property - Generic set property extension + * + * A generic struct that allows any of the Xe's IOCTL to be extended + * with a set_property operation. */ struct drm_xe_ext_set_property { /** @base: base user extension */ @@ -287,7 +317,7 @@ struct drm_xe_mem_region { * here will always be zero). */ __u64 cpu_visible_used; - /** @reserved: MBZ */ + /** @reserved: Reserved */ __u64 reserved[6]; }; @@ -1041,8 +1071,8 @@ struct drm_xe_sync { __u32 handle; /** - * @addr: Address of user fence. When sync passed in via exec - * IOCTL this a GPU address in the VM. When sync passed in via + * @addr: Address of user fence. When sync is passed in via exec + * IOCTL this is a GPU address in the VM. When sync passed in via * VM bind IOCTL this is a user pointer. In either case, it is * the users responsibility that this address is present and * mapped when the user fence is signalled. Must be qword @@ -1051,7 +1081,10 @@ struct drm_xe_sync { __u64 addr; }; - /** @timeline_value: Timeline point of the sync object */ + /** + * @timeline_value: Input for the timeline sync object. Needs to be + * different than 0 when used with %DRM_XE_SYNC_FLAG_TIMELINE_SYNCOBJ. + */ __u64 timeline_value; /** @reserved: Reserved */ -- cgit v1.2.3 From 535881a8c50b79085327e7dbe26a4c55f3e1591b Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Fri, 15 Dec 2023 15:45:48 +0000 Subject: drm/xe/uapi: Document the memory_region bitmask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The uAPI should stay generic in regarding to the bitmask. It is the userspace responsibility to check for the type/class of the memory, without any assumption. Also add comments inside the code to explain how it is actually constructed so we don't accidentally change the assignment of the instance and the masks. No functional change in this patch. It only explains and document the memory_region masks. A further follow-up work with the organization of all memory regions around struct xe_mem_regions is desired, but not part of this patch. Signed-off-by: Rodrigo Vivi Reviewed-by: Lucas De Marchi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast --- drivers/gpu/drm/xe/xe_query.c | 19 +++++++++++++++++++ include/uapi/drm/xe_drm.h | 23 ++++++++++++++++++----- 2 files changed, 37 insertions(+), 5 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 56d61bf596b2..9b35673b286c 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -266,6 +266,11 @@ static int query_mem_regions(struct xe_device *xe, man = ttm_manager_type(&xe->ttm, XE_PL_TT); mem_regions->mem_regions[0].mem_class = DRM_XE_MEM_REGION_CLASS_SYSMEM; + /* + * The instance needs to be a unique number that represents the index + * in the placement mask used at xe_gem_create_ioctl() for the + * xe_bo_create() placement. + */ mem_regions->mem_regions[0].instance = 0; mem_regions->mem_regions[0].min_page_size = PAGE_SIZE; mem_regions->mem_regions[0].total_size = man->size << PAGE_SHIFT; @@ -381,6 +386,20 @@ static int query_gt_list(struct xe_device *xe, struct drm_xe_device_query *query gt_list->gt_list[id].tile_id = gt_to_tile(gt)->id; gt_list->gt_list[id].gt_id = gt->info.id; gt_list->gt_list[id].reference_clock = gt->info.reference_clock; + /* + * The mem_regions indexes in the mask below need to + * directly identify the struct + * drm_xe_query_mem_regions' instance constructed at + * query_mem_regions() + * + * For our current platforms: + * Bit 0 -> System Memory + * Bit 1 -> VRAM0 on Tile0 + * Bit 2 -> VRAM1 on Tile1 + * However the uAPI is generic and it's userspace's + * responsibility to check the mem_class, without any + * assumption. + */ if (!IS_DGFX(xe)) gt_list->gt_list[id].near_mem_regions = 0x1; else diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 5a01d033b780..6c719ba8fc8e 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -256,10 +256,9 @@ struct drm_xe_mem_region { */ __u16 mem_class; /** - * @instance: The instance for this region. - * - * The @mem_class and @instance taken together will always give - * a unique pair. + * @instance: The unique ID for this region, which serves as the + * index in the placement bitmask used as argument for + * &DRM_IOCTL_XE_GEM_CREATE */ __u16 instance; /** @@ -404,6 +403,10 @@ struct drm_xe_gt { * @near_mem_regions: Bit mask of instances from * drm_xe_query_mem_regions that are nearest to the current engines * of this GT. + * Each index in this mask refers directly to the struct + * drm_xe_query_mem_regions' instance, no assumptions should + * be made about order. The type of each region is described + * by struct drm_xe_query_mem_regions' mem_class. */ __u64 near_mem_regions; /** @@ -412,6 +415,10 @@ struct drm_xe_gt { * In general, they have extra indirections when compared to the * @near_mem_regions. For a discrete device this could mean system * memory and memory living in a different tile. + * Each index in this mask refers directly to the struct + * drm_xe_query_mem_regions' instance, no assumptions should + * be made about order. The type of each region is described + * by struct drm_xe_query_mem_regions' mem_class. */ __u64 far_mem_regions; /** @reserved: Reserved */ @@ -652,7 +659,13 @@ struct drm_xe_gem_create { */ __u64 size; - /** @placement: A mask of memory instances of where BO can be placed. */ + /** + * @placement: A mask of memory instances of where BO can be placed. + * Each index in this mask refers directly to the struct + * drm_xe_query_mem_regions' instance, no assumptions should + * be made about order. The type of each region is described + * by struct drm_xe_query_mem_regions' mem_class. + */ __u32 placement; #define DRM_XE_GEM_CREATE_FLAG_DEFER_BACKING (1 << 0) -- cgit v1.2.3 From 33c6fda687a37ef871ca04adf2e05ffc646e3b13 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:49 +0000 Subject: drm/xe/uapi: Add block diagram of a device MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to make proper use the uAPI, a prerequisite is to understand some key concepts about the discrete GPU devices which are supported by the Xe driver. For example, some structs defined in the uAPI are an abstraction of a hardware component with a specific role. This diagram helps to build a mental representation of a device how it is seen by the Xe driver. As written in the documentation, it does not intend to be a literal representation of an existing device. A lot more information could be added but the intention for the overview is to keep it simple, and go into detail as needed in other sections. v2: Add GT1 inside Tile0 (José Roberto de Souza) Reviewed-by: José Roberto de Souza Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 6c719ba8fc8e..4b5d41543280 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -23,6 +23,45 @@ extern "C" { * 5. uEvents */ +/** + * DOC: Xe Device Block Diagram + * + * The diagram below represents a high-level simplification of a discrete + * GPU supported by the Xe driver. It shows some device components which + * are necessary to understand this API, as well as how their relations + * to each other. This diagram does not represent real hardware:: + * + * ┌──────────────────────────────────────────────────────────────────┐ + * │ ┌──────────────────────────────────────────────────┐ ┌─────────┐ │ + * │ │ ┌───────────────────────┐ ┌─────┐ │ │ ┌─────┐ │ │ + * │ │ │ VRAM0 ├───┤ ... │ │ │ │VRAM1│ │ │ + * │ │ └───────────┬───────────┘ └─GT1─┘ │ │ └──┬──┘ │ │ + * │ │ ┌──────────────────┴───────────────────────────┐ │ │ ┌──┴──┐ │ │ + * │ │ │ ┌─────────────────────┐ ┌─────────────────┐ │ │ │ │ │ │ │ + * │ │ │ │ ┌──┐ ┌──┐ ┌──┐ ┌──┐ │ │ ┌─────┐ ┌─────┐ │ │ │ │ │ │ │ │ + * │ │ │ │ │EU│ │EU│ │EU│ │EU│ │ │ │RCS0 │ │BCS0 │ │ │ │ │ │ │ │ │ + * │ │ │ │ └──┘ └──┘ └──┘ └──┘ │ │ └─────┘ └─────┘ │ │ │ │ │ │ │ │ + * │ │ │ │ ┌──┐ ┌──┐ ┌──┐ ┌──┐ │ │ ┌─────┐ ┌─────┐ │ │ │ │ │ │ │ │ + * │ │ │ │ │EU│ │EU│ │EU│ │EU│ │ │ │VCS0 │ │VCS1 │ │ │ │ │ │ │ │ │ + * │ │ │ │ └──┘ └──┘ └──┘ └──┘ │ │ └─────┘ └─────┘ │ │ │ │ │ │ │ │ + * │ │ │ │ ┌──┐ ┌──┐ ┌──┐ ┌──┐ │ │ ┌─────┐ ┌─────┐ │ │ │ │ │ │ │ │ + * │ │ │ │ │EU│ │EU│ │EU│ │EU│ │ │ │VECS0│ │VECS1│ │ │ │ │ │ ... │ │ │ + * │ │ │ │ └──┘ └──┘ └──┘ └──┘ │ │ └─────┘ └─────┘ │ │ │ │ │ │ │ │ + * │ │ │ │ ┌──┐ ┌──┐ ┌──┐ ┌──┐ │ │ ┌─────┐ ┌─────┐ │ │ │ │ │ │ │ │ + * │ │ │ │ │EU│ │EU│ │EU│ │EU│ │ │ │CCS0 │ │CCS1 │ │ │ │ │ │ │ │ │ + * │ │ │ │ └──┘ └──┘ └──┘ └──┘ │ │ └─────┘ └─────┘ │ │ │ │ │ │ │ │ + * │ │ │ └─────────DSS─────────┘ │ ┌─────┐ ┌─────┐ │ │ │ │ │ │ │ │ + * │ │ │ │ │CCS2 │ │CCS3 │ │ │ │ │ │ │ │ │ + * │ │ │ ┌─────┐ ┌─────┐ ┌─────┐ │ └─────┘ └─────┘ │ │ │ │ │ │ │ │ + * │ │ │ │ ... │ │ ... │ │ ... │ │ │ │ │ │ │ │ │ │ + * │ │ │ └─DSS─┘ └─DSS─┘ └─DSS─┘ └─────Engines─────┘ │ │ │ │ │ │ │ + * │ │ └───────────────────────────GT0────────────────┘ │ │ └─GT2─┘ │ │ + * │ └────────────────────────────Tile0─────────────────┘ └─ Tile1──┘ │ + * └─────────────────────────────Device0───────┬──────────────────────┘ + * │ + * ───────────────────────┴────────── PCI bus + */ + /** * DOC: Xe uAPI Overview * -- cgit v1.2.3 From db35331176f93125cc4bfa0d05283688607200f5 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:50 +0000 Subject: drm/xe/uapi: Add examples of user space code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete the documentation of some structs by adding functional examples of user space code. Those examples are intentionally kept very simple. Put together, they provide a foundation for a minimal application that executes a job using the Xe driver. v2: Remove use of DRM_XE_VM_BIND_FLAG_ASYNC (Francois Dugast) Reviewed-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 84 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 4b5d41543280..5240653eeefd 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -951,6 +951,30 @@ struct drm_xe_vm_bind_op { /** * struct drm_xe_vm_bind - Input of &DRM_IOCTL_XE_VM_BIND + * + * Below is an example of a minimal use of @drm_xe_vm_bind to + * asynchronously bind the buffer `data` at address `BIND_ADDRESS` to + * illustrate `userptr`. It can be synchronized by using the example + * provided for @drm_xe_sync. + * + * .. code-block:: C + * + * data = aligned_alloc(ALIGNMENT, BO_SIZE); + * struct drm_xe_vm_bind bind = { + * .vm_id = vm, + * .num_binds = 1, + * .bind.obj = 0, + * .bind.obj_offset = to_user_pointer(data), + * .bind.range = BO_SIZE, + * .bind.addr = BIND_ADDRESS, + * .bind.op = DRM_XE_VM_BIND_OP_MAP_USERPTR, + * .bind.flags = 0, + * .num_syncs = 1, + * .syncs = &sync, + * .exec_queue_id = 0, + * }; + * ioctl(fd, DRM_IOCTL_XE_VM_BIND, &bind); + * */ struct drm_xe_vm_bind { /** @extensions: Pointer to the first extension struct, if any */ @@ -1012,6 +1036,25 @@ struct drm_xe_vm_bind { /** * struct drm_xe_exec_queue_create - Input of &DRM_IOCTL_XE_EXEC_QUEUE_CREATE + * + * The example below shows how to use @drm_xe_exec_queue_create to create + * a simple exec_queue (no parallel submission) of class + * &DRM_XE_ENGINE_CLASS_RENDER. + * + * .. code-block:: C + * + * struct drm_xe_engine_class_instance instance = { + * .engine_class = DRM_XE_ENGINE_CLASS_RENDER, + * }; + * struct drm_xe_exec_queue_create exec_queue_create = { + * .extensions = 0, + * .vm_id = vm, + * .num_bb_per_exec = 1, + * .num_eng_per_bb = 1, + * .instances = to_user_pointer(&instance), + * }; + * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); + * */ struct drm_xe_exec_queue_create { #define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 @@ -1103,6 +1146,30 @@ struct drm_xe_exec_queue_get_property { * * and the @flags can be: * - %DRM_XE_SYNC_FLAG_SIGNAL + * + * A minimal use of @drm_xe_sync looks like this: + * + * .. code-block:: C + * + * struct drm_xe_sync sync = { + * .flags = DRM_XE_SYNC_FLAG_SIGNAL, + * .type = DRM_XE_SYNC_TYPE_SYNCOBJ, + * }; + * struct drm_syncobj_create syncobj_create = { 0 }; + * ioctl(fd, DRM_IOCTL_SYNCOBJ_CREATE, &syncobj_create); + * sync.handle = syncobj_create.handle; + * ... + * use of &sync in drm_xe_exec or drm_xe_vm_bind + * ... + * struct drm_syncobj_wait wait = { + * .handles = &sync.handle, + * .timeout_nsec = INT64_MAX, + * .count_handles = 1, + * .flags = 0, + * .first_signaled = 0, + * .pad = 0, + * }; + * ioctl(fd, DRM_IOCTL_SYNCOBJ_WAIT, &wait); */ struct drm_xe_sync { /** @extensions: Pointer to the first extension struct, if any */ @@ -1145,6 +1212,23 @@ struct drm_xe_sync { /** * struct drm_xe_exec - Input of &DRM_IOCTL_XE_EXEC + * + * This is an example to use @drm_xe_exec for execution of the object + * at BIND_ADDRESS (see example in @drm_xe_vm_bind) by an exec_queue + * (see example in @drm_xe_exec_queue_create). It can be synchronized + * by using the example provided for @drm_xe_sync. + * + * .. code-block:: C + * + * struct drm_xe_exec exec = { + * .exec_queue_id = exec_queue, + * .syncs = &sync, + * .num_syncs = 1, + * .address = BIND_ADDRESS, + * .num_batch_buffer = 1, + * }; + * ioctl(fd, DRM_IOCTL_XE_EXEC, &exec); + * */ struct drm_xe_exec { /** @extensions: Pointer to the first extension struct, if any */ -- cgit v1.2.3 From 0bf90a8c223759564964d4a1ecd44608876ab02d Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:51 +0000 Subject: drm/xe/uapi: Move CPU_CACHING defines before doc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move those defines to align on the rule used elsewhere in the file which was introduced by commit 4f082f2c3a37 ("drm/xe: Move defines before relevant fields"). Reviewed-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 5240653eeefd..8a69abea0725 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -733,12 +733,12 @@ struct drm_xe_gem_create { */ __u32 handle; +#define DRM_XE_GEM_CPU_CACHING_WB 1 +#define DRM_XE_GEM_CPU_CACHING_WC 2 /** * @cpu_caching: The CPU caching mode to select for this object. If * mmaping the object the mode selected here will also be used. */ -#define DRM_XE_GEM_CPU_CACHING_WB 1 -#define DRM_XE_GEM_CPU_CACHING_WC 2 __u16 cpu_caching; /** @pad: MBZ */ __u16 pad[3]; -- cgit v1.2.3 From 9f7ceec2cd25e7aea31cd0630b6fcf439770e322 Mon Sep 17 00:00:00 2001 From: Francois Dugast Date: Fri, 15 Dec 2023 15:45:52 +0000 Subject: drm/xe/uapi: Move DRM_XE_ACC_GRANULARITY_* where they are used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bring those defines close to the context where they can be used. Also apply indentation as it is done for other subsets of defines. Reviewed-by: Rodrigo Vivi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) (limited to 'include/uapi') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 8a69abea0725..919aa72c4481 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1020,20 +1020,6 @@ struct drm_xe_vm_bind { __u64 reserved[2]; }; -/* For use with DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY */ - -/* Monitor 128KB contiguous region with 4K sub-granularity */ -#define DRM_XE_ACC_GRANULARITY_128K 0 - -/* Monitor 2MB contiguous region with 64KB sub-granularity */ -#define DRM_XE_ACC_GRANULARITY_2M 1 - -/* Monitor 16MB contiguous region with 512KB sub-granularity */ -#define DRM_XE_ACC_GRANULARITY_16M 2 - -/* Monitor 64MB contiguous region with 2M sub-granularity */ -#define DRM_XE_ACC_GRANULARITY_64M 3 - /** * struct drm_xe_exec_queue_create - Input of &DRM_IOCTL_XE_EXEC_QUEUE_CREATE * @@ -1066,6 +1052,14 @@ struct drm_xe_exec_queue_create { #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_TRIGGER 5 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_NOTIFY 6 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_ACC_GRANULARITY 7 +/* Monitor 128KB contiguous region with 4K sub-granularity */ +#define DRM_XE_ACC_GRANULARITY_128K 0 +/* Monitor 2MB contiguous region with 64KB sub-granularity */ +#define DRM_XE_ACC_GRANULARITY_2M 1 +/* Monitor 16MB contiguous region with 512KB sub-granularity */ +#define DRM_XE_ACC_GRANULARITY_16M 2 +/* Monitor 64MB contiguous region with 2M sub-granularity */ +#define DRM_XE_ACC_GRANULARITY_64M 3 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; -- cgit v1.2.3 From 77a0d4d1cea2140ef56929ab1cfa5e525772c90e Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Fri, 15 Dec 2023 15:45:53 +0000 Subject: drm/xe/uapi: Remove reset uevent for now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This kernel uevent is getting removed for now. It will come back later with a better future proof name. v2: Rebase (Francois Dugast) Cc: Himal Prasad Ghimiray Cc: Lucas De Marchi Cc: Francois Dugast Cc: Aravind Iddamsetty Signed-off-by: Rodrigo Vivi Reviewed-by: Himal Prasad Ghimiray Acked-by: Lucas De Marchi Acked-by: José Roberto de Souza Acked-by: Mateusz Naklicki Signed-off-by: Francois Dugast --- drivers/gpu/drm/xe/xe_gt.c | 18 ------------------ include/uapi/drm/xe_drm.h | 11 ----------- 2 files changed, 29 deletions(-) (limited to 'include/uapi') diff --git a/drivers/gpu/drm/xe/xe_gt.c b/drivers/gpu/drm/xe/xe_gt.c index f5d18e98f8b6..3af2adec1295 100644 --- a/drivers/gpu/drm/xe/xe_gt.c +++ b/drivers/gpu/drm/xe/xe_gt.c @@ -589,20 +589,6 @@ static int do_gt_restart(struct xe_gt *gt) return 0; } -static void xe_uevent_gt_reset_failure(struct pci_dev *pdev, u8 tile_id, u8 gt_id) -{ - char *reset_event[4]; - - reset_event[0] = DRM_XE_RESET_FAILED_UEVENT "=NEEDS_RESET"; - reset_event[1] = kasprintf(GFP_KERNEL, "TILE_ID=%d", tile_id); - reset_event[2] = kasprintf(GFP_KERNEL, "GT_ID=%d", gt_id); - reset_event[3] = NULL; - kobject_uevent_env(&pdev->dev.kobj, KOBJ_CHANGE, reset_event); - - kfree(reset_event[1]); - kfree(reset_event[2]); -} - static int gt_reset(struct xe_gt *gt) { int err; @@ -659,10 +645,6 @@ err_msg: err_fail: xe_gt_err(gt, "reset failed (%pe)\n", ERR_PTR(err)); - /* Notify userspace about gt reset failure */ - xe_uevent_gt_reset_failure(to_pci_dev(gt_to_xe(gt)->drm.dev), - gt_to_tile(gt)->id, gt->info.id); - gt_to_xe(gt)->needs_flr_on_fini = true; return err; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 919aa72c4481..9fa3ae324731 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -20,7 +20,6 @@ extern "C" { * 2. Extension definition and helper structs * 3. IOCTL's Query structs in the order of the Query's entries. * 4. The rest of IOCTL structs in the order of IOCTL declaration. - * 5. uEvents */ /** @@ -1341,16 +1340,6 @@ struct drm_xe_wait_user_fence { __u64 reserved[2]; }; -/** - * DOC: uevent generated by xe on it's pci node. - * - * DRM_XE_RESET_FAILED_UEVENT - Event is generated when attempt to reset gt - * fails. The value supplied with the event is always "NEEDS_RESET". - * Additional information supplied is tile id and gt id of the gt unit for - * which reset has failed. - */ -#define DRM_XE_RESET_FAILED_UEVENT "DEVICE_STATUS" - #if defined(__cplusplus) } #endif -- cgit v1.2.3