From 75d37750a753e7ae079e470ea9699caeae756e3d Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Mon, 13 Jan 2025 17:12:01 +0530 Subject: drm/xe/mmap: Add mmap support for PCI memory barrier In order to avoid having userspace to use MI_MEM_FENCE, we are adding a mechanism for userspace to generate a PCI memory barrier with low overhead (avoiding IOCTL call as well as writing to VRAM will adds some overhead). This is implemented by memory-mapping a page as uncached that is backed by MMIO on the dGPU and thus allowing userspace to do memory write to the page without invoking an IOCTL. We are selecting the MMIO so that it is not accessible from the PCI bus so that the MMIO writes themselves are ignored, but the PCI memory barrier will still take action as the MMIO filtering will happen after the memory barrier effect. When we detect special defined offset in mmap(), We are mapping 4K page which contains the last of page of doorbell MMIO range to userspace for same purpose. For user to query special offset we are adding special flag in mmap_offset ioctl which needs to be passed as follows, struct drm_xe_gem_mmap_offset mmo = { .handle = 0, /* this must be 0 */ .flags = DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER, }; igt_ioctl(fd, DRM_IOCTL_XE_GEM_MMAP_OFFSET, &mmo); map = mmap(NULL, size, PROT_WRITE, MAP_SHARED, fd, mmo); IGT : https://gitlab.freedesktop.org/drm/igt-gpu-tools/-/commit/b2dbc6f22815128c0dd5c737504f42e1f1a6ad62 UMD : https://github.com/intel/compute-runtime/pull/772 V7: - Dgpu filter added V6(MAuld) - Move physical mmap to fault handler - Modify kernel-doc and attach UMD PR when ready V5(MAuld) - Return invalid early in case of non 4K PAGE_SIZE - Format kernel-doc and add note for 4K PAGE_SIZE HW limit V4(MAuld) - Add kernel-doc for uapi change - Restrict page size to 4K V3(MAuld) - Remove offset defination from UAPI to be able to change later - Edit commit message for special flag addition V2(MAuld) - Add fault handler with dummy page to handle unplug device - Add Build check for special offset to be below normal start page - Test d3hot, mapping seems to be valid in d3hot as well - Add more info to commit message Cc: Matthew Auld Acked-by: Michal Mrozek Reviewed-by: Matthew Auld Signed-off-by: Tejas Upadhyay Signed-off-by: Matthew Auld Link: https://patchwork.freedesktop.org/patch/msgid/20250113114201.3178806-1-tejas.upadhyay@intel.com --- include/uapi/drm/xe_drm.h | 29 ++++++++++++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) (limited to 'include/uapi/drm') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index f62689ca861a..cac607a30f6d 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -811,6 +811,32 @@ struct drm_xe_gem_create { /** * struct drm_xe_gem_mmap_offset - Input of &DRM_IOCTL_XE_GEM_MMAP_OFFSET + * + * The @flags can be: + * - %DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER - For user to query special offset + * for use in mmap ioctl. Writing to the returned mmap address will generate a + * PCI memory barrier with low overhead (avoiding IOCTL call as well as writing + * to VRAM which would also add overhead), acting like an MI_MEM_FENCE + * instruction. + * + * Note: The mmap size can be at most 4K, due to HW limitations. As a result + * this interface is only supported on CPU architectures that support 4K page + * size. The mmap_offset ioctl will detect this and gracefully return an + * error, where userspace is expected to have a different fallback method for + * triggering a barrier. + * + * Roughly the usage would be as follows: + * + * .. code-block:: C + * + * struct drm_xe_gem_mmap_offset mmo = { + * .handle = 0, // must be set to 0 + * .flags = DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER, + * }; + * + * err = ioctl(fd, DRM_IOCTL_XE_GEM_MMAP_OFFSET, &mmo); + * map = mmap(NULL, size, PROT_WRITE, MAP_SHARED, fd, mmo.offset); + * map[i] = 0xdeadbeaf; // issue barrier */ struct drm_xe_gem_mmap_offset { /** @extensions: Pointer to the first extension struct, if any */ @@ -819,7 +845,8 @@ struct drm_xe_gem_mmap_offset { /** @handle: Handle for the object being mapped. */ __u32 handle; - /** @flags: Must be zero */ +#define DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER (1 << 0) + /** @flags: Flags */ __u32 flags; /** @offset: The fake offset to use for subsequent mmap call */ -- cgit v1.3 From a46ea12eca59fd3741ddfec3042d43f87fadf58f Mon Sep 17 00:00:00 2001 From: Rodrigo Vivi Date: Fri, 17 Jan 2025 14:38:27 -0500 Subject: drm/xe/uapi: Fix documentation indentation Fix these issues: Documentation/gpu/driver-uapi:29: include/uapi/drm/xe_drm.h:817: WARNING: +Bullet list ends without a blank line; unexpected unindent. Documentation/gpu/driver-uapi:29: include/uapi/drm/xe_drm.h:835: WARNING: +Definition list ends without a blank line; unexpected unindent. Fixes: 75d37750a753 ("drm/xe/mmap: Add mmap support for PCI memory barrier") Reported-by: Stephen Rothwell Closes: https://lore.kernel.org/intel-xe/20250117164023.3fdc00b9@canb.auug.org.au/ Cc: Tejas Upadhyay Tested-by: Bagas Sanjaya Reviewed-by: Tejas Upadhyay Link: https://patchwork.freedesktop.org/patch/msgid/20250117193827.91779-1-rodrigo.vivi@intel.com Signed-off-by: Rodrigo Vivi --- include/uapi/drm/xe_drm.h | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'include/uapi/drm') diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index cac607a30f6d..e2160330ad01 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -814,29 +814,29 @@ struct drm_xe_gem_create { * * The @flags can be: * - %DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER - For user to query special offset - * for use in mmap ioctl. Writing to the returned mmap address will generate a - * PCI memory barrier with low overhead (avoiding IOCTL call as well as writing - * to VRAM which would also add overhead), acting like an MI_MEM_FENCE - * instruction. + * for use in mmap ioctl. Writing to the returned mmap address will generate a + * PCI memory barrier with low overhead (avoiding IOCTL call as well as writing + * to VRAM which would also add overhead), acting like an MI_MEM_FENCE + * instruction. * - * Note: The mmap size can be at most 4K, due to HW limitations. As a result - * this interface is only supported on CPU architectures that support 4K page - * size. The mmap_offset ioctl will detect this and gracefully return an - * error, where userspace is expected to have a different fallback method for - * triggering a barrier. + * Note: The mmap size can be at most 4K, due to HW limitations. As a result + * this interface is only supported on CPU architectures that support 4K page + * size. The mmap_offset ioctl will detect this and gracefully return an + * error, where userspace is expected to have a different fallback method for + * triggering a barrier. * - * Roughly the usage would be as follows: + * Roughly the usage would be as follows: * - * .. code-block:: C + * .. code-block:: C * - * struct drm_xe_gem_mmap_offset mmo = { - * .handle = 0, // must be set to 0 - * .flags = DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER, - * }; + * struct drm_xe_gem_mmap_offset mmo = { + * .handle = 0, // must be set to 0 + * .flags = DRM_XE_MMAP_OFFSET_FLAG_PCI_BARRIER, + * }; * - * err = ioctl(fd, DRM_IOCTL_XE_GEM_MMAP_OFFSET, &mmo); - * map = mmap(NULL, size, PROT_WRITE, MAP_SHARED, fd, mmo.offset); - * map[i] = 0xdeadbeaf; // issue barrier + * err = ioctl(fd, DRM_IOCTL_XE_GEM_MMAP_OFFSET, &mmo); + * map = mmap(NULL, size, PROT_WRITE, MAP_SHARED, fd, mmo.offset); + * map[i] = 0xdeadbeaf; // issue barrier */ struct drm_xe_gem_mmap_offset { /** @extensions: Pointer to the first extension struct, if any */ -- cgit v1.3 From 72d479601d67026c4fafaad21762a777cf41f906 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Wed, 29 Jan 2025 09:41:32 -0800 Subject: drm/xe/pxp/uapi: Add userspace and LRC support for PXP-using queues Userspace is required to mark a queue as using PXP to guarantee that the PXP instructions will work. In addition to managing the PXP sessions, when a PXP queue is created the driver will set the relevant bits in its context control register. On submission of a valid PXP queue, the driver will validate all encrypted objects mapped to the VM to ensured they were encrypted with the current key. v2: Remove pxp_types include outside of PXP code (Jani), better comments and code cleanup (John) v3: split the internal PXP management to a separate patch for ease of review. re-order ioctl checks to always return -EINVAL if parameters are invalid, rebase on msix changes. Signed-off-by: Daniele Ceraolo Spurio Cc: John Harrison Reviewed-by: John Harrison Link: https://patchwork.freedesktop.org/patch/msgid/20250129174140.948829-9-daniele.ceraolospurio@intel.com --- drivers/gpu/drm/xe/regs/xe_engine_regs.h | 1 + drivers/gpu/drm/xe/xe_exec_queue.c | 56 ++++++++++++++++++++++++++++++-- drivers/gpu/drm/xe/xe_exec_queue.h | 5 +++ drivers/gpu/drm/xe/xe_exec_queue_types.h | 2 ++ drivers/gpu/drm/xe/xe_execlist.c | 2 +- drivers/gpu/drm/xe/xe_lrc.c | 18 ++++++++-- drivers/gpu/drm/xe/xe_lrc.h | 4 ++- drivers/gpu/drm/xe/xe_pxp.c | 35 +++++++++++++++++--- drivers/gpu/drm/xe/xe_pxp.h | 4 +-- include/uapi/drm/xe_drm.h | 40 ++++++++++++++++++++++- 10 files changed, 153 insertions(+), 14 deletions(-) (limited to 'include/uapi/drm') diff --git a/drivers/gpu/drm/xe/regs/xe_engine_regs.h b/drivers/gpu/drm/xe/regs/xe_engine_regs.h index d86219dedde2..c8fd3d5ca502 100644 --- a/drivers/gpu/drm/xe/regs/xe_engine_regs.h +++ b/drivers/gpu/drm/xe/regs/xe_engine_regs.h @@ -132,6 +132,7 @@ #define RING_EXECLIST_STATUS_HI(base) XE_REG((base) + 0x234 + 4) #define RING_CONTEXT_CONTROL(base) XE_REG((base) + 0x244, XE_REG_OPTION_MASKED) +#define CTX_CTRL_PXP_ENABLE REG_BIT(10) #define CTX_CTRL_OAC_CONTEXT_ENABLE REG_BIT(8) #define CTX_CTRL_RUN_ALONE REG_BIT(7) #define CTX_CTRL_INDIRECT_RING_STATE_ENABLE REG_BIT(4) diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 2ec4e2eb6f2a..6051db78d706 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -25,6 +25,7 @@ #include "xe_ring_ops_types.h" #include "xe_trace.h" #include "xe_vm.h" +#include "xe_pxp.h" enum xe_exec_queue_sched_prop { XE_EXEC_QUEUE_JOB_TIMEOUT = 0, @@ -38,6 +39,8 @@ static int exec_queue_user_extensions(struct xe_device *xe, struct xe_exec_queue static void __xe_exec_queue_free(struct xe_exec_queue *q) { + if (xe_exec_queue_uses_pxp(q)) + xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); if (q->vm) xe_vm_put(q->vm); @@ -113,6 +116,21 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q) { struct xe_vm *vm = q->vm; int i, err; + u32 flags = 0; + + /* + * PXP workloads executing on RCS or CCS must run in isolation (i.e. no + * other workload can use the EUs at the same time). On MTL this is done + * by setting the RUNALONE bit in the LRC, while starting on Xe2 there + * is a dedicated bit for it. + */ + if (xe_exec_queue_uses_pxp(q) && + (q->class == XE_ENGINE_CLASS_RENDER || q->class == XE_ENGINE_CLASS_COMPUTE)) { + if (GRAPHICS_VER(gt_to_xe(q->gt)) >= 20) + flags |= XE_LRC_CREATE_PXP; + else + flags |= XE_LRC_CREATE_RUNALONE; + } if (vm) { err = xe_vm_lock(vm, true); @@ -121,7 +139,7 @@ static int __xe_exec_queue_init(struct xe_exec_queue *q) } for (i = 0; i < q->width; ++i) { - q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K, q->msix_vec); + q->lrc[i] = xe_lrc_create(q->hwe, q->vm, SZ_16K, q->msix_vec, flags); if (IS_ERR(q->lrc[i])) { err = PTR_ERR(q->lrc[i]); goto err_unlock; @@ -166,6 +184,19 @@ struct xe_exec_queue *xe_exec_queue_create(struct xe_device *xe, struct xe_vm *v if (err) goto err_post_alloc; + /* + * We can only add the queue to the PXP list after the init is complete, + * because the PXP termination can call exec_queue_kill and that will + * go bad if the queue is only half-initialized. This means that we + * can't do it when we handle the PXP extension in __xe_exec_queue_alloc + * and we need to do it here instead. + */ + if (xe_exec_queue_uses_pxp(q)) { + err = xe_pxp_exec_queue_add(xe->pxp, q); + if (err) + goto err_post_alloc; + } + return q; err_post_alloc: @@ -254,6 +285,9 @@ void xe_exec_queue_destroy(struct kref *ref) struct xe_exec_queue *q = container_of(ref, struct xe_exec_queue, refcount); struct xe_exec_queue *eq, *next; + if (xe_exec_queue_uses_pxp(q)) + xe_pxp_exec_queue_remove(gt_to_xe(q->gt)->pxp, q); + xe_exec_queue_last_fence_put_unlocked(q); if (!(q->flags & EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD)) { list_for_each_entry_safe(eq, next, &q->multi_gt_list, @@ -409,6 +443,22 @@ static int exec_queue_set_timeslice(struct xe_device *xe, struct xe_exec_queue * return 0; } +static int +exec_queue_set_pxp_type(struct xe_device *xe, struct xe_exec_queue *q, u64 value) +{ + if (value == DRM_XE_PXP_TYPE_NONE) + return 0; + + /* we only support HWDRM sessions right now */ + if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) + return -EINVAL; + + if (!xe_pxp_is_enabled(xe->pxp)) + return -ENODEV; + + return xe_pxp_exec_queue_set_type(xe->pxp, q, DRM_XE_PXP_TYPE_HWDRM); +} + typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, struct xe_exec_queue *q, u64 value); @@ -416,6 +466,7 @@ typedef int (*xe_exec_queue_set_property_fn)(struct xe_device *xe, static const xe_exec_queue_set_property_fn exec_queue_set_property_funcs[] = { [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY] = exec_queue_set_priority, [DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE] = exec_queue_set_timeslice, + [DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE] = exec_queue_set_pxp_type, }; static int exec_queue_user_ext_set_property(struct xe_device *xe, @@ -435,7 +486,8 @@ static int exec_queue_user_ext_set_property(struct xe_device *xe, ARRAY_SIZE(exec_queue_set_property_funcs)) || XE_IOCTL_DBG(xe, ext.pad) || XE_IOCTL_DBG(xe, ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY && - ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE)) + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE && + ext.property != DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE)) return -EINVAL; idx = array_index_nospec(ext.property, ARRAY_SIZE(exec_queue_set_property_funcs)); diff --git a/drivers/gpu/drm/xe/xe_exec_queue.h b/drivers/gpu/drm/xe/xe_exec_queue.h index 90c7f73eab88..17bc50a7f05a 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.h +++ b/drivers/gpu/drm/xe/xe_exec_queue.h @@ -57,6 +57,11 @@ static inline bool xe_exec_queue_is_parallel(struct xe_exec_queue *q) return q->width > 1; } +static inline bool xe_exec_queue_uses_pxp(struct xe_exec_queue *q) +{ + return q->pxp.type; +} + bool xe_exec_queue_is_lr(struct xe_exec_queue *q); bool xe_exec_queue_ring_full(struct xe_exec_queue *q); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 6d85a069947f..6eb7ff091534 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -132,6 +132,8 @@ struct xe_exec_queue { /** @pxp: PXP info tracking */ struct { + /** @pxp.type: PXP session type used by this queue */ + u8 type; /** @pxp.link: link into the list of PXP exec queues */ struct list_head link; } pxp; diff --git a/drivers/gpu/drm/xe/xe_execlist.c b/drivers/gpu/drm/xe/xe_execlist.c index 5ef96deaa881..779a52daf3d7 100644 --- a/drivers/gpu/drm/xe/xe_execlist.c +++ b/drivers/gpu/drm/xe/xe_execlist.c @@ -269,7 +269,7 @@ struct xe_execlist_port *xe_execlist_port_create(struct xe_device *xe, port->hwe = hwe; - port->lrc = xe_lrc_create(hwe, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX); + port->lrc = xe_lrc_create(hwe, NULL, SZ_16K, XE_IRQ_DEFAULT_MSIX, 0); if (IS_ERR(port->lrc)) { err = PTR_ERR(port->lrc); goto err; diff --git a/drivers/gpu/drm/xe/xe_lrc.c b/drivers/gpu/drm/xe/xe_lrc.c index bbb9ffbf6367..df3ceddede07 100644 --- a/drivers/gpu/drm/xe/xe_lrc.c +++ b/drivers/gpu/drm/xe/xe_lrc.c @@ -883,7 +883,8 @@ static void xe_lrc_finish(struct xe_lrc *lrc) #define PVC_CTX_ACC_CTR_THOLD (0x2a + 1) static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, - struct xe_vm *vm, u32 ring_size, u16 msix_vec) + struct xe_vm *vm, u32 ring_size, u16 msix_vec, + u32 init_flags) { struct xe_gt *gt = hwe->gt; struct xe_tile *tile = gt_to_tile(gt); @@ -979,6 +980,16 @@ static int xe_lrc_init(struct xe_lrc *lrc, struct xe_hw_engine *hwe, RING_CTL_SIZE(lrc->ring.size) | RING_VALID); } + if (init_flags & XE_LRC_CREATE_RUNALONE) + xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, + xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | + _MASKED_BIT_ENABLE(CTX_CTRL_RUN_ALONE)); + + if (init_flags & XE_LRC_CREATE_PXP) + xe_lrc_write_ctx_reg(lrc, CTX_CONTEXT_CONTROL, + xe_lrc_read_ctx_reg(lrc, CTX_CONTEXT_CONTROL) | + _MASKED_BIT_ENABLE(CTX_CTRL_PXP_ENABLE)); + xe_lrc_write_ctx_reg(lrc, CTX_TIMESTAMP, 0); if (xe->info.has_asid && vm) @@ -1021,6 +1032,7 @@ err_lrc_finish: * @vm: The VM (address space) * @ring_size: LRC ring size * @msix_vec: MSI-X interrupt vector (for platforms that support it) + * @flags: LRC initialization flags * * Allocate and initialize the Logical Ring Context (LRC). * @@ -1028,7 +1040,7 @@ err_lrc_finish: * upon failure. */ struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, - u32 ring_size, u16 msix_vec) + u32 ring_size, u16 msix_vec, u32 flags) { struct xe_lrc *lrc; int err; @@ -1037,7 +1049,7 @@ struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, if (!lrc) return ERR_PTR(-ENOMEM); - err = xe_lrc_init(lrc, hwe, vm, ring_size, msix_vec); + err = xe_lrc_init(lrc, hwe, vm, ring_size, msix_vec, flags); if (err) { kfree(lrc); return ERR_PTR(err); diff --git a/drivers/gpu/drm/xe/xe_lrc.h b/drivers/gpu/drm/xe/xe_lrc.h index b27e80cd842a..0b40f349ab95 100644 --- a/drivers/gpu/drm/xe/xe_lrc.h +++ b/drivers/gpu/drm/xe/xe_lrc.h @@ -42,8 +42,10 @@ struct xe_lrc_snapshot { #define LRC_PPHWSP_FLUSH_INVAL_SCRATCH_ADDR (0x34 * 4) #define LRC_PPHWSP_PXP_INVAL_SCRATCH_ADDR (0x40 * 4) +#define XE_LRC_CREATE_RUNALONE 0x1 +#define XE_LRC_CREATE_PXP 0x2 struct xe_lrc *xe_lrc_create(struct xe_hw_engine *hwe, struct xe_vm *vm, - u32 ring_size, u16 msix_vec); + u32 ring_size, u16 msix_vec, u32 flags); void xe_lrc_destroy(struct kref *ref); /** diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index 69d49e34e34d..b32121273e0d 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -6,6 +6,7 @@ #include "xe_pxp.h" #include +#include #include "xe_device_types.h" #include "xe_exec_queue.h" @@ -47,7 +48,7 @@ bool xe_pxp_is_supported(const struct xe_device *xe) return xe->info.has_pxp && IS_ENABLED(CONFIG_INTEL_MEI_GSC_PROXY); } -static bool pxp_is_enabled(const struct xe_pxp *pxp) +bool xe_pxp_is_enabled(const struct xe_pxp *pxp) { return pxp; } @@ -249,7 +250,7 @@ void xe_pxp_irq_handler(struct xe_device *xe, u16 iir) { struct xe_pxp *pxp = xe->pxp; - if (!pxp_is_enabled(pxp)) { + if (!xe_pxp_is_enabled(pxp)) { drm_err(&xe->drm, "PXP irq 0x%x received with PXP disabled!\n", iir); return; } @@ -424,6 +425,27 @@ out_force_wake: return ret; } +/** + * xe_pxp_exec_queue_set_type - Mark a queue as using PXP + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @q: the queue to mark as using PXP + * @type: the type of PXP session this queue will use + * + * Returns 0 if the selected PXP type is supported, -ENODEV otherwise. + */ +int xe_pxp_exec_queue_set_type(struct xe_pxp *pxp, struct xe_exec_queue *q, u8 type) +{ + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + /* we only support HWDRM sessions right now */ + xe_assert(pxp->xe, type == DRM_XE_PXP_TYPE_HWDRM); + + q->pxp.type = type; + + return 0; +} + static void __exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q) { spin_lock_irq(&pxp->queues.lock); @@ -449,9 +471,12 @@ int xe_pxp_exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q) { int ret = 0; - if (!pxp_is_enabled(pxp)) + if (!xe_pxp_is_enabled(pxp)) return -ENODEV; + /* we only support HWDRM sessions right now */ + xe_assert(pxp->xe, q->pxp.type == DRM_XE_PXP_TYPE_HWDRM); + /* * Runtime suspend kills PXP, so we take a reference to prevent it from * happening while we have active queues that use PXP @@ -589,7 +614,7 @@ void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q) { bool need_pm_put = false; - if (!pxp_is_enabled(pxp)) + if (!xe_pxp_is_enabled(pxp)) return; spin_lock_irq(&pxp->queues.lock); @@ -599,6 +624,8 @@ void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q) need_pm_put = true; } + q->pxp.type = DRM_XE_PXP_TYPE_NONE; + spin_unlock_irq(&pxp->queues.lock); if (need_pm_put) diff --git a/drivers/gpu/drm/xe/xe_pxp.h b/drivers/gpu/drm/xe/xe_pxp.h index f482567c27b5..2e0ab186072a 100644 --- a/drivers/gpu/drm/xe/xe_pxp.h +++ b/drivers/gpu/drm/xe/xe_pxp.h @@ -12,13 +12,13 @@ struct xe_device; struct xe_exec_queue; struct xe_pxp; -#define DRM_XE_PXP_HWDRM_DEFAULT_SESSION 0xF /* TODO: move to uapi */ - bool xe_pxp_is_supported(const struct xe_device *xe); +bool xe_pxp_is_enabled(const struct xe_pxp *pxp); int xe_pxp_init(struct xe_device *xe); void xe_pxp_irq_handler(struct xe_device *xe, u16 iir); +int xe_pxp_exec_queue_set_type(struct xe_pxp *pxp, struct xe_exec_queue *q, u8 type); int xe_pxp_exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q); void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index e2160330ad01..9d53834c4c0a 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1114,6 +1114,24 @@ struct drm_xe_vm_bind { /** * struct drm_xe_exec_queue_create - Input of &DRM_IOCTL_XE_EXEC_QUEUE_CREATE * + * This ioctl supports setting the following properties via the + * %DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY extension, which uses the + * generic @drm_xe_ext_set_property struct: + * + * - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY - set the queue priority. + * CAP_SYS_NICE is required to set a value above normal. + * - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE - set the queue timeslice + * duration in microseconds. + * - %DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE - set the type of PXP session + * this queue will be used with. Valid values are listed in enum + * drm_xe_pxp_session_type. %DRM_XE_PXP_TYPE_NONE is the default behavior, so + * there is no need to explicitly set that. When a queue of type + * %DRM_XE_PXP_TYPE_HWDRM is created, the PXP default HWDRM session + * (%XE_PXP_HWDRM_DEFAULT_SESSION) will be started, if isn't already running. + * Given that going into a power-saving state kills PXP HWDRM sessions, + * runtime PM will be blocked while queues of this type are alive. + * All PXP queues will be killed if a PXP invalidation event occurs. + * * The example below shows how to use @drm_xe_exec_queue_create to create * a simple exec_queue (no parallel submission) of class * &DRM_XE_ENGINE_CLASS_RENDER. @@ -1137,7 +1155,7 @@ struct drm_xe_exec_queue_create { #define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PRIORITY 0 #define DRM_XE_EXEC_QUEUE_SET_PROPERTY_TIMESLICE 1 - +#define DRM_XE_EXEC_QUEUE_SET_PROPERTY_PXP_TYPE 2 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -1756,6 +1774,26 @@ struct drm_xe_oa_stream_info { __u64 reserved[3]; }; +/** + * enum drm_xe_pxp_session_type - Supported PXP session types. + * + * We currently only support HWDRM sessions, which are used for protected + * content that ends up being displayed, but the HW supports multiple types, so + * we might extend support in the future. + */ +enum drm_xe_pxp_session_type { + /** @DRM_XE_PXP_TYPE_NONE: PXP not used */ + DRM_XE_PXP_TYPE_NONE = 0, + /** + * @DRM_XE_PXP_TYPE_HWDRM: HWDRM sessions are used for content that ends + * up on the display. + */ + DRM_XE_PXP_TYPE_HWDRM = 1, +}; + +/* ID of the protected content session managed by Xe when PXP is active */ +#define DRM_XE_PXP_HWDRM_DEFAULT_SESSION 0xf + #if defined(__cplusplus) } #endif -- cgit v1.3 From bd98ac2e05855ea781c9b7ad30b5e1a234aefe95 Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Wed, 29 Jan 2025 09:41:33 -0800 Subject: drm/xe/pxp/uapi: Add a query for PXP status MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PXP prerequisites (SW proxy and HuC auth via GSC) are completed asynchronously from driver load, which means that userspace can start submitting before we're ready to start a PXP session. Therefore, we need a query that userspace can use to check not only if PXP is supported but also to wait until the prerequisites are done. v2: Improve doc, do not report TYPE_NONE as supported (José) v3: Better comments, remove unneeded copy_from_user (John) Signed-off-by: Daniele Ceraolo Spurio Cc: José Roberto de Souza Cc: John Harrison Reviewed-by: John Harrison Link: https://patchwork.freedesktop.org/patch/msgid/20250129174140.948829-10-daniele.ceraolospurio@intel.com --- drivers/gpu/drm/xe/xe_pxp.c | 32 ++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_pxp.h | 1 + drivers/gpu/drm/xe/xe_query.c | 29 +++++++++++++++++++++++++++++ include/uapi/drm/xe_drm.h | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+) (limited to 'include/uapi/drm') diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index b32121273e0d..24aef5c0f04a 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -80,6 +80,38 @@ static bool pxp_prerequisites_done(const struct xe_pxp *pxp) return ready; } +/** + * xe_pxp_get_readiness_status - check whether PXP is ready for userspace use + * @pxp: the xe_pxp pointer (can be NULL if PXP is disabled) + * + * Returns: 0 if PXP is not ready yet, 1 if it is ready, a negative errno value + * if PXP is not supported/enabled or if something went wrong in the + * initialization of the prerequisites. Note that the return values of this + * function follow the uapi (see drm_xe_query_pxp_status), so they can be used + * directly in the query ioctl. + */ +int xe_pxp_get_readiness_status(struct xe_pxp *pxp) +{ + int ret = 0; + + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + /* if the GSC or HuC FW are in an error state, PXP will never work */ + if (xe_uc_fw_status_to_error(pxp->gt->uc.huc.fw.status) || + xe_uc_fw_status_to_error(pxp->gt->uc.gsc.fw.status)) + return -EIO; + + xe_pm_runtime_get(pxp->xe); + + /* PXP requires both HuC loaded and GSC proxy initialized */ + if (pxp_prerequisites_done(pxp)) + ret = 1; + + xe_pm_runtime_put(pxp->xe); + return ret; +} + static bool pxp_session_is_in_play(struct xe_pxp *pxp, u32 id) { struct xe_gt *gt = pxp->gt; diff --git a/drivers/gpu/drm/xe/xe_pxp.h b/drivers/gpu/drm/xe/xe_pxp.h index 2e0ab186072a..868813cc84b9 100644 --- a/drivers/gpu/drm/xe/xe_pxp.h +++ b/drivers/gpu/drm/xe/xe_pxp.h @@ -14,6 +14,7 @@ struct xe_pxp; bool xe_pxp_is_supported(const struct xe_device *xe); bool xe_pxp_is_enabled(const struct xe_pxp *pxp); +int xe_pxp_get_readiness_status(struct xe_pxp *pxp); int xe_pxp_init(struct xe_device *xe); void xe_pxp_irq_handler(struct xe_device *xe, u16 iir); diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index c059639613f7..042f87a688e7 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -24,6 +24,7 @@ #include "xe_macros.h" #include "xe_mmio.h" #include "xe_oa.h" +#include "xe_pxp.h" #include "xe_ttm_vram_mgr.h" #include "xe_wa.h" @@ -698,6 +699,33 @@ static int query_oa_units(struct xe_device *xe, return ret ? -EFAULT : 0; } +static int query_pxp_status(struct xe_device *xe, struct drm_xe_device_query *query) +{ + struct drm_xe_query_pxp_status __user *query_ptr = u64_to_user_ptr(query->data); + size_t size = sizeof(struct drm_xe_query_pxp_status); + struct drm_xe_query_pxp_status resp = { 0 }; + int ret; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + ret = xe_pxp_get_readiness_status(xe->pxp); + if (ret < 0) + return ret; + + resp.status = ret; + resp.supported_session_types = BIT(DRM_XE_PXP_TYPE_HWDRM); + + if (copy_to_user(query_ptr, &resp, size)) + return -EFAULT; + + return 0; +} + static int (* const xe_query_funcs[])(struct xe_device *xe, struct drm_xe_device_query *query) = { query_engines, @@ -709,6 +737,7 @@ static int (* const xe_query_funcs[])(struct xe_device *xe, query_engine_cycles, query_uc_fw_version, query_oa_units, + query_pxp_status, }; int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 9d53834c4c0a..112fd27f3c75 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -629,6 +629,39 @@ struct drm_xe_query_uc_fw_version { __u64 reserved; }; +/** + * struct drm_xe_query_pxp_status - query if PXP is ready + * + * If PXP is enabled and no fatal error has occurred, the status will be set to + * one of the following values: + * 0: PXP init still in progress + * 1: PXP init complete + * + * If PXP is not enabled or something has gone wrong, the query will be failed + * with one of the following error codes: + * -ENODEV: PXP not supported or disabled; + * -EIO: fatal error occurred during init, so PXP will never be enabled; + * -EINVAL: incorrect value provided as part of the query; + * -EFAULT: error copying the memory between kernel and userspace. + * + * The status can only be 0 in the first few seconds after driver load. If + * everything works as expected, the status will transition to init complete in + * less than 1 second, while in case of errors the driver might take longer to + * start returning an error code, but it should still take less than 10 seconds. + * + * The supported session type bitmask is based on the values in + * enum drm_xe_pxp_session_type. TYPE_NONE is always supported and therefore + * is not reported in the bitmask. + * + */ +struct drm_xe_query_pxp_status { + /** @status: current PXP status */ + __u32 status; + + /** @supported_session_types: bitmask of supported PXP session types */ + __u32 supported_session_types; +}; + /** * struct drm_xe_device_query - Input of &DRM_IOCTL_XE_DEVICE_QUERY - main * structure to query device information @@ -648,6 +681,7 @@ struct drm_xe_query_uc_fw_version { * attributes. * - %DRM_XE_DEVICE_QUERY_GT_TOPOLOGY * - %DRM_XE_DEVICE_QUERY_ENGINE_CYCLES + * - %DRM_XE_DEVICE_QUERY_PXP_STATUS * * If size is set to 0, the driver fills it with the required size for * the requested type of data to query. If size is equal to the required @@ -700,6 +734,7 @@ struct drm_xe_device_query { #define DRM_XE_DEVICE_QUERY_ENGINE_CYCLES 6 #define DRM_XE_DEVICE_QUERY_UC_FW_VERSION 7 #define DRM_XE_DEVICE_QUERY_OA_UNITS 8 +#define DRM_XE_DEVICE_QUERY_PXP_STATUS 9 /** @query: The type of data to query */ __u32 query; -- cgit v1.3 From 41a97c4a12947c2786a1680d6839bb72d1c57cec Mon Sep 17 00:00:00 2001 From: Daniele Ceraolo Spurio Date: Wed, 29 Jan 2025 09:41:34 -0800 Subject: drm/xe/pxp/uapi: Add API to mark a BO as using PXP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver needs to know if a BO is encrypted with PXP to enable the display decryption at flip time. Furthermore, we want to keep track of the status of the encryption and reject any operation that involves a BO that is encrypted using an old key. There are two points in time where such checks can kick in: 1 - at VM bind time, all operations except for unmapping will be rejected if the key used to encrypt the BO is no longer valid. This check is opt-in via a new VM_BIND flag, to avoid a scenario where a malicious app purposely shares an invalid BO with a non-PXP aware app (such as a compositor). If the VM_BIND was failed, the compositor would be unable to display anything at all. Allowing the bind to go through means that output still works, it just displays garbage data within the bounds of the illegal BO. 2 - at job submission time, if the queue is marked as using PXP, all objects bound to the VM will be checked and the submission will be rejected if any of them was encrypted with a key that is no longer valid. Note that there is no risk of leaking the encrypted data if a user does not opt-in to those checks; the only consequence is that the user will not realize that the encryption key is changed and that the data is no longer valid. v2: Better commnnts and descriptions (John), rebase v3: Properly return the result of key_assign up the stack, do not use xe_bo in display headers (Jani) v4: improve key_instance variable documentation (John) Signed-off-by: Daniele Ceraolo Spurio Cc: Matthew Brost Cc: Thomas Hellström Cc: John Harrison Cc: Jani Nikula Reviewed-by: John Harrison Link: https://patchwork.freedesktop.org/patch/msgid/20250129174140.948829-11-daniele.ceraolospurio@intel.com --- .../gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h | 12 ++- drivers/gpu/drm/xe/display/intel_bo.c | 2 +- drivers/gpu/drm/xe/xe_bo.c | 98 +++++++++++++++++++++- drivers/gpu/drm/xe/xe_bo.h | 5 ++ drivers/gpu/drm/xe/xe_bo_types.h | 6 ++ drivers/gpu/drm/xe/xe_exec.c | 6 ++ drivers/gpu/drm/xe/xe_pxp.c | 90 ++++++++++++++++++++ drivers/gpu/drm/xe/xe_pxp.h | 6 ++ drivers/gpu/drm/xe/xe_pxp_types.h | 11 +++ drivers/gpu/drm/xe/xe_vm.c | 46 +++++++++- drivers/gpu/drm/xe/xe_vm.h | 2 + include/uapi/drm/xe_drm.h | 19 +++++ 12 files changed, 296 insertions(+), 7 deletions(-) (limited to 'include/uapi/drm') diff --git a/drivers/gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h b/drivers/gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h index 419e8e926f00..d2eb8e1f6c4b 100644 --- a/drivers/gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h +++ b/drivers/gpu/drm/xe/compat-i915-headers/pxp/intel_pxp.h @@ -9,6 +9,8 @@ #include #include +#include "xe_pxp.h" + struct drm_gem_object; struct xe_pxp; @@ -16,7 +18,15 @@ static inline int intel_pxp_key_check(struct xe_pxp *pxp, struct drm_gem_object *obj, bool assign) { - return -ENODEV; + /* + * The assign variable is used in i915 to assign the key to the BO at + * first submission time. In Xe the key is instead assigned at BO + * creation time, so the assign variable must always be false. + */ + if (assign) + return -EINVAL; + + return xe_pxp_obj_key_check(pxp, obj); } #endif diff --git a/drivers/gpu/drm/xe/display/intel_bo.c b/drivers/gpu/drm/xe/display/intel_bo.c index b463f5bd4eed..27437c22bd70 100644 --- a/drivers/gpu/drm/xe/display/intel_bo.c +++ b/drivers/gpu/drm/xe/display/intel_bo.c @@ -25,7 +25,7 @@ bool intel_bo_is_shmem(struct drm_gem_object *obj) bool intel_bo_is_protected(struct drm_gem_object *obj) { - return false; + return xe_bo_is_protected(gem_to_xe_bo(obj)); } void intel_bo_flush_if_display(struct drm_gem_object *obj) diff --git a/drivers/gpu/drm/xe/xe_bo.c b/drivers/gpu/drm/xe/xe_bo.c index c32201123d44..6812164e1470 100644 --- a/drivers/gpu/drm/xe/xe_bo.c +++ b/drivers/gpu/drm/xe/xe_bo.c @@ -6,6 +6,7 @@ #include "xe_bo.h" #include +#include #include #include @@ -26,6 +27,7 @@ #include "xe_migrate.h" #include "xe_pm.h" #include "xe_preempt_fence.h" +#include "xe_pxp.h" #include "xe_res_cursor.h" #include "xe_trace_bo.h" #include "xe_ttm_stolen_mgr.h" @@ -2155,6 +2157,93 @@ void xe_bo_vunmap(struct xe_bo *bo) __xe_bo_vunmap(bo); } +static int gem_create_set_pxp_type(struct xe_device *xe, struct xe_bo *bo, u64 value) +{ + if (value == DRM_XE_PXP_TYPE_NONE) + return 0; + + /* we only support DRM_XE_PXP_TYPE_HWDRM for now */ + if (XE_IOCTL_DBG(xe, value != DRM_XE_PXP_TYPE_HWDRM)) + return -EINVAL; + + return xe_pxp_key_assign(xe->pxp, bo); +} + +typedef int (*xe_gem_create_set_property_fn)(struct xe_device *xe, + struct xe_bo *bo, + u64 value); + +static const xe_gem_create_set_property_fn gem_create_set_property_funcs[] = { + [DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY] = gem_create_set_pxp_type, +}; + +static int gem_create_user_ext_set_property(struct xe_device *xe, + struct xe_bo *bo, + u64 extension) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_ext_set_property ext; + int err; + u32 idx; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.property >= + ARRAY_SIZE(gem_create_set_property_funcs)) || + XE_IOCTL_DBG(xe, ext.pad) || + XE_IOCTL_DBG(xe, ext.property != DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY)) + return -EINVAL; + + idx = array_index_nospec(ext.property, ARRAY_SIZE(gem_create_set_property_funcs)); + if (!gem_create_set_property_funcs[idx]) + return -EINVAL; + + return gem_create_set_property_funcs[idx](xe, bo, ext.value); +} + +typedef int (*xe_gem_create_user_extension_fn)(struct xe_device *xe, + struct xe_bo *bo, + u64 extension); + +static const xe_gem_create_user_extension_fn gem_create_user_extension_funcs[] = { + [DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY] = gem_create_user_ext_set_property, +}; + +#define MAX_USER_EXTENSIONS 16 +static int gem_create_user_extensions(struct xe_device *xe, struct xe_bo *bo, + u64 extensions, int ext_number) +{ + u64 __user *address = u64_to_user_ptr(extensions); + struct drm_xe_user_extension ext; + int err; + u32 idx; + + if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) + return -E2BIG; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.pad) || + XE_IOCTL_DBG(xe, ext.name >= ARRAY_SIZE(gem_create_user_extension_funcs))) + return -EINVAL; + + idx = array_index_nospec(ext.name, + ARRAY_SIZE(gem_create_user_extension_funcs)); + err = gem_create_user_extension_funcs[idx](xe, bo, extensions); + if (XE_IOCTL_DBG(xe, err)) + return err; + + if (ext.next_extension) + return gem_create_user_extensions(xe, bo, ext.next_extension, + ++ext_number); + + return 0; +} + int xe_gem_create_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { @@ -2167,8 +2256,7 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, u32 handle; int err; - if (XE_IOCTL_DBG(xe, args->extensions) || - XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || + if (XE_IOCTL_DBG(xe, args->pad[0] || args->pad[1] || args->pad[2]) || XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) return -EINVAL; @@ -2250,6 +2338,12 @@ int xe_gem_create_ioctl(struct drm_device *dev, void *data, goto out_vm; } + if (args->extensions) { + err = gem_create_user_extensions(xe, bo, args->extensions, 0); + if (err) + goto out_bulk; + } + err = drm_gem_handle_create(file, &bo->ttm.base, &handle); if (err) goto out_bulk; diff --git a/drivers/gpu/drm/xe/xe_bo.h b/drivers/gpu/drm/xe/xe_bo.h index 04995c5ced32..f09b9315721b 100644 --- a/drivers/gpu/drm/xe/xe_bo.h +++ b/drivers/gpu/drm/xe/xe_bo.h @@ -186,6 +186,11 @@ static inline bool xe_bo_is_pinned(struct xe_bo *bo) return bo->ttm.pin_count; } +static inline bool xe_bo_is_protected(const struct xe_bo *bo) +{ + return bo->pxp_key_instance; +} + static inline void xe_bo_unpin_map_no_vm(struct xe_bo *bo) { if (likely(bo)) { diff --git a/drivers/gpu/drm/xe/xe_bo_types.h b/drivers/gpu/drm/xe/xe_bo_types.h index 46dc9e4e3e46..60c522866500 100644 --- a/drivers/gpu/drm/xe/xe_bo_types.h +++ b/drivers/gpu/drm/xe/xe_bo_types.h @@ -57,6 +57,12 @@ struct xe_bo { */ struct list_head client_link; #endif + /** + * @pxp_key_instance: PXP key instance this BO was created against. A + * 0 in this variable indicates that the BO does not use PXP encryption. + */ + u32 pxp_key_instance; + /** @freed: List node for delayed put. */ struct llist_node freed; /** @update_index: Update index if PT BO */ diff --git a/drivers/gpu/drm/xe/xe_exec.c b/drivers/gpu/drm/xe/xe_exec.c index df8ce550deb4..b75adfc99fb7 100644 --- a/drivers/gpu/drm/xe/xe_exec.c +++ b/drivers/gpu/drm/xe/xe_exec.c @@ -262,6 +262,12 @@ retry: goto err_exec; } + if (xe_exec_queue_uses_pxp(q)) { + err = xe_vm_validate_protected(q->vm); + if (err) + goto err_exec; + } + job = xe_sched_job_create(q, xe_exec_queue_is_parallel(q) ? addresses : &args->address); if (IS_ERR(job)) { diff --git a/drivers/gpu/drm/xe/xe_pxp.c b/drivers/gpu/drm/xe/xe_pxp.c index 24aef5c0f04a..8060b4050be8 100644 --- a/drivers/gpu/drm/xe/xe_pxp.c +++ b/drivers/gpu/drm/xe/xe_pxp.c @@ -8,6 +8,8 @@ #include #include +#include "xe_bo.h" +#include "xe_bo_types.h" #include "xe_device_types.h" #include "xe_exec_queue.h" #include "xe_force_wake.h" @@ -185,6 +187,9 @@ static void pxp_terminate(struct xe_pxp *pxp) pxp_invalidate_queues(pxp); + if (pxp->status == XE_PXP_ACTIVE) + pxp->key_instance++; + /* * If we have a termination already in progress, we need to wait for * it to complete before queueing another one. Once the first @@ -385,6 +390,8 @@ int xe_pxp_init(struct xe_device *xe) pxp->xe = xe; pxp->gt = gt; + pxp->key_instance = 1; + /* * we'll use the completions to check if there is an action pending, * so we start them as completed and we reinit it when an action is @@ -689,3 +696,86 @@ static void pxp_invalidate_queues(struct xe_pxp *pxp) spin_unlock_irq(&pxp->queues.lock); } + +/** + * xe_pxp_key_assign - mark a BO as using the current PXP key iteration + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @bo: the BO to mark + * + * Returns: -ENODEV if PXP is disabled, 0 otherwise. + */ +int xe_pxp_key_assign(struct xe_pxp *pxp, struct xe_bo *bo) +{ + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + xe_assert(pxp->xe, !bo->pxp_key_instance); + + /* + * Note that the PXP key handling is inherently racey, because the key + * can theoretically change at any time (although it's unlikely to do + * so without triggers), even right after we copy it. Taking a lock + * wouldn't help because the value might still change as soon as we + * release the lock. + * Userspace needs to handle the fact that their BOs can go invalid at + * any point. + */ + bo->pxp_key_instance = pxp->key_instance; + + return 0; +} + +/** + * xe_pxp_bo_key_check - check if the key used by a xe_bo is valid + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @bo: the BO we want to check + * + * Checks whether a BO was encrypted with the current key or an obsolete one. + * + * Returns: 0 if the key is valid, -ENODEV if PXP is disabled, -EINVAL if the + * BO is not using PXP, -ENOEXEC if the key is not valid. + */ +int xe_pxp_bo_key_check(struct xe_pxp *pxp, struct xe_bo *bo) +{ + if (!xe_pxp_is_enabled(pxp)) + return -ENODEV; + + if (!xe_bo_is_protected(bo)) + return -EINVAL; + + xe_assert(pxp->xe, bo->pxp_key_instance); + + /* + * Note that the PXP key handling is inherently racey, because the key + * can theoretically change at any time (although it's unlikely to do + * so without triggers), even right after we check it. Taking a lock + * wouldn't help because the value might still change as soon as we + * release the lock. + * We mitigate the risk by checking the key at multiple points (on each + * submission involving the BO and right before flipping it on the + * display), but there is still a very small chance that we could + * operate on an invalid BO for a single submission or a single frame + * flip. This is a compromise made to protect the encrypted data (which + * is what the key termination is for). + */ + if (bo->pxp_key_instance != pxp->key_instance) + return -ENOEXEC; + + return 0; +} + +/** + * xe_pxp_obj_key_check - check if the key used by a drm_gem_obj is valid + * @pxp: the xe->pxp pointer (it will be NULL if PXP is disabled) + * @obj: the drm_gem_obj we want to check + * + * Checks whether a drm_gem_obj was encrypted with the current key or an + * obsolete one. + * + * Returns: 0 if the key is valid, -ENODEV if PXP is disabled, -EINVAL if the + * obj is not using PXP, -ENOEXEC if the key is not valid. + */ +int xe_pxp_obj_key_check(struct xe_pxp *pxp, struct drm_gem_object *obj) +{ + return xe_pxp_bo_key_check(pxp, gem_to_xe_bo(obj)); +} diff --git a/drivers/gpu/drm/xe/xe_pxp.h b/drivers/gpu/drm/xe/xe_pxp.h index 868813cc84b9..3dd70eac9da6 100644 --- a/drivers/gpu/drm/xe/xe_pxp.h +++ b/drivers/gpu/drm/xe/xe_pxp.h @@ -8,6 +8,8 @@ #include +struct drm_gem_object; +struct xe_bo; struct xe_device; struct xe_exec_queue; struct xe_pxp; @@ -23,4 +25,8 @@ int xe_pxp_exec_queue_set_type(struct xe_pxp *pxp, struct xe_exec_queue *q, u8 t int xe_pxp_exec_queue_add(struct xe_pxp *pxp, struct xe_exec_queue *q); void xe_pxp_exec_queue_remove(struct xe_pxp *pxp, struct xe_exec_queue *q); +int xe_pxp_key_assign(struct xe_pxp *pxp, struct xe_bo *bo); +int xe_pxp_bo_key_check(struct xe_pxp *pxp, struct xe_bo *bo); +int xe_pxp_obj_key_check(struct xe_pxp *pxp, struct drm_gem_object *obj); + #endif /* __XE_PXP_H__ */ diff --git a/drivers/gpu/drm/xe/xe_pxp_types.h b/drivers/gpu/drm/xe/xe_pxp_types.h index bd741720f67d..8e4569f0173d 100644 --- a/drivers/gpu/drm/xe/xe_pxp_types.h +++ b/drivers/gpu/drm/xe/xe_pxp_types.h @@ -112,6 +112,17 @@ struct xe_pxp { /** @queues.list: list of exec_queues that use PXP */ struct list_head list; } queues; + + /** + * @key_instance: keep track of the current iteration of the PXP key. + * Note that, due to the time needed for PXP termination and re-start + * to complete, the minimum time between 2 subsequent increases of this + * variable is 50ms, and even that only if there is a continuous attack; + * normal behavior is for this to increase much much slower than that. + * This means that we don't expect this to ever wrap and don't implement + * that case in the code. + */ + u32 key_instance; }; #endif /* __XE_PXP_TYPES_H__ */ diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index b9270d059e18..d664f2e418b2 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -33,6 +33,7 @@ #include "xe_pm.h" #include "xe_preempt_fence.h" #include "xe_pt.h" +#include "xe_pxp.h" #include "xe_res_cursor.h" #include "xe_sync.h" #include "xe_trace_bo.h" @@ -2726,7 +2727,8 @@ ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_execute, ERRNO); (DRM_XE_VM_BIND_FLAG_READONLY | \ DRM_XE_VM_BIND_FLAG_IMMEDIATE | \ DRM_XE_VM_BIND_FLAG_NULL | \ - DRM_XE_VM_BIND_FLAG_DUMPABLE) + DRM_XE_VM_BIND_FLAG_DUMPABLE | \ + DRM_XE_VM_BIND_FLAG_CHECK_PXP) #ifdef TEST_VM_OPS_ERROR #define SUPPORTED_FLAGS (SUPPORTED_FLAGS_STUB | FORCE_OP_ERROR) @@ -2889,7 +2891,7 @@ static void xe_vma_ops_init(struct xe_vma_ops *vops, struct xe_vm *vm, static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo, u64 addr, u64 range, u64 obj_offset, - u16 pat_index) + u16 pat_index, u32 op, u32 bind_flags) { u16 coh_mode; @@ -2933,6 +2935,12 @@ static int xe_vm_bind_ioctl_validate_bo(struct xe_device *xe, struct xe_bo *bo, return -EINVAL; } + /* If a BO is protected it can only be mapped if the key is still valid */ + if ((bind_flags & DRM_XE_VM_BIND_FLAG_CHECK_PXP) && xe_bo_is_protected(bo) && + op != DRM_XE_VM_BIND_OP_UNMAP && op != DRM_XE_VM_BIND_OP_UNMAP_ALL) + if (XE_IOCTL_DBG(xe, xe_pxp_bo_key_check(xe->pxp, bo) != 0)) + return -ENOEXEC; + return 0; } @@ -3022,6 +3030,8 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) u32 obj = bind_ops[i].obj; u64 obj_offset = bind_ops[i].obj_offset; u16 pat_index = bind_ops[i].pat_index; + u32 op = bind_ops[i].op; + u32 bind_flags = bind_ops[i].flags; if (!obj) continue; @@ -3034,7 +3044,8 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) bos[i] = gem_to_xe_bo(gem_obj); err = xe_vm_bind_ioctl_validate_bo(xe, bos[i], addr, range, - obj_offset, pat_index); + obj_offset, pat_index, op, + bind_flags); if (err) goto put_obj; } @@ -3334,6 +3345,35 @@ wait: return ret; } +int xe_vm_validate_protected(struct xe_vm *vm) +{ + struct drm_gpuva *gpuva; + int err = 0; + + if (!vm) + return -ENODEV; + + mutex_lock(&vm->snap_mutex); + + drm_gpuvm_for_each_va(gpuva, &vm->gpuvm) { + struct xe_vma *vma = gpuva_to_vma(gpuva); + struct xe_bo *bo = vma->gpuva.gem.obj ? + gem_to_xe_bo(vma->gpuva.gem.obj) : NULL; + + if (!bo) + continue; + + if (xe_bo_is_protected(bo)) { + err = xe_pxp_bo_key_check(vm->xe->pxp, bo); + if (err) + break; + } + } + + mutex_unlock(&vm->snap_mutex); + return err; +} + struct xe_vm_snapshot { unsigned long num_snaps; struct { diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index 0a2fa6c0815b..f66075f8a6fe 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -215,6 +215,8 @@ struct dma_fence *xe_vma_rebind(struct xe_vm *vm, struct xe_vma *vma, int xe_vm_invalidate_vma(struct xe_vma *vma); +int xe_vm_validate_protected(struct xe_vm *vm); + static inline void xe_vm_queue_rebind_worker(struct xe_vm *vm) { xe_assert(vm->xe, xe_vm_in_preempt_fence_mode(vm)); diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 112fd27f3c75..892f54d3aa09 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -778,8 +778,23 @@ struct drm_xe_device_query { * - %DRM_XE_GEM_CPU_CACHING_WC - Allocate the pages as write-combined. This * is uncached. Scanout surfaces should likely use this. All objects * that can be placed in VRAM must use this. + * + * This ioctl supports setting the following properties via the + * %DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY extension, which uses the + * generic @drm_xe_ext_set_property struct: + * + * - %DRM_XE_GEM_CREATE_SET_PROPERTY_PXP_TYPE - set the type of PXP session + * this object will be used with. Valid values are listed in enum + * drm_xe_pxp_session_type. %DRM_XE_PXP_TYPE_NONE is the default behavior, so + * there is no need to explicitly set that. Objects used with session of type + * %DRM_XE_PXP_TYPE_HWDRM will be marked as invalid if a PXP invalidation + * event occurs after their creation. Attempting to flip an invalid object + * will cause a black frame to be displayed instead. Submissions with invalid + * objects mapped in the VM will be rejected. */ struct drm_xe_gem_create { +#define DRM_XE_GEM_CREATE_EXTENSION_SET_PROPERTY 0 +#define DRM_XE_GEM_CREATE_SET_PROPERTY_PXP_TYPE 0 /** @extensions: Pointer to the first extension struct, if any */ __u64 extensions; @@ -968,6 +983,9 @@ struct drm_xe_vm_destroy { * will only be valid for DRM_XE_VM_BIND_OP_MAP operations, the BO * handle MBZ, and the BO offset MBZ. This flag is intended to * implement VK sparse bindings. + * - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, + * reject the binding if the encryption key is no longer valid. This + * flag has no effect on BOs that are not marked as using PXP. */ struct drm_xe_vm_bind_op { /** @extensions: Pointer to the first extension struct, if any */ @@ -1058,6 +1076,7 @@ struct drm_xe_vm_bind_op { #define DRM_XE_VM_BIND_FLAG_IMMEDIATE (1 << 1) #define DRM_XE_VM_BIND_FLAG_NULL (1 << 2) #define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3) +#define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) /** @flags: Bind flags */ __u32 flags; -- cgit v1.3 From aafe181f7dfbb726004c2ecb1d28297b84f3f34b Mon Sep 17 00:00:00 2001 From: Asad Kamal Date: Mon, 27 May 2024 12:15:15 +0800 Subject: drm/amdgpu: Add flags to distinguish vf/pf/pt mode Add extra flag definition for ids_flag field to distinguish between vf/pf/pt modes v2: Updated kms driver minor version & removed pf check as default is 0 v3: Fix up version (Alex) v4: rebase (Alex) Proposed userspace: https://github.com/ROCm/amdsmi/commit/e663bed7d6b3df79f5959e73981749b1f22ec698 Signed-off-by: Asad Kamal Reviewed-by: Lijo Lazar Reviewed-by: Hawking Zhang Signed-off-by: Alex Deucher --- drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c | 3 ++- drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c | 9 +++++++++ include/uapi/drm/amdgpu_drm.h | 10 ++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) (limited to 'include/uapi/drm') diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c index f52f674477eb..1819166cb4cf 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c @@ -121,9 +121,10 @@ * - 3.59.0 - Cleared VRAM * - 3.60.0 - Add AMDGPU_TILING_GFX12_DCC_WRITE_COMPRESS_DISABLE (Vulkan requirement) * - 3.61.0 - Contains fix for RV/PCO compute queues + * - 3.62.0 - Add AMDGPU_IDS_FLAGS_MODE_PF, AMDGPU_IDS_FLAGS_MODE_VF & AMDGPU_IDS_FLAGS_MODE_PT */ #define KMS_DRIVER_MAJOR 3 -#define KMS_DRIVER_MINOR 61 +#define KMS_DRIVER_MINOR 62 #define KMS_DRIVER_PATCHLEVEL 0 /* diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c index 98528ee94c15..05c73bf7541c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c @@ -888,6 +888,15 @@ out: if (adev->gfx.config.ta_cntl2_truncate_coord_mode) dev_info->ids_flags |= AMDGPU_IDS_FLAGS_CONFORMANT_TRUNC_COORD; + if (amdgpu_passthrough(adev)) + dev_info->ids_flags |= (AMDGPU_IDS_FLAGS_MODE_PT << + AMDGPU_IDS_FLAGS_MODE_SHIFT) & + AMDGPU_IDS_FLAGS_MODE_MASK; + else if (amdgpu_sriov_vf(adev)) + dev_info->ids_flags |= (AMDGPU_IDS_FLAGS_MODE_VF << + AMDGPU_IDS_FLAGS_MODE_SHIFT) & + AMDGPU_IDS_FLAGS_MODE_MASK; + vm_size = adev->vm_manager.max_pfn * AMDGPU_GPU_PAGE_SIZE; vm_size -= AMDGPU_VA_RESERVED_TOP; diff --git a/include/uapi/drm/amdgpu_drm.h b/include/uapi/drm/amdgpu_drm.h index aaa4f3bc688b..25d5c6e90a99 100644 --- a/include/uapi/drm/amdgpu_drm.h +++ b/include/uapi/drm/amdgpu_drm.h @@ -763,6 +763,16 @@ struct drm_amdgpu_cs_chunk_cp_gfx_shadow { #define AMDGPU_IDS_FLAGS_TMZ 0x4 #define AMDGPU_IDS_FLAGS_CONFORMANT_TRUNC_COORD 0x8 +/* + * Query h/w info: Flag identifying VF/PF/PT mode + * + */ +#define AMDGPU_IDS_FLAGS_MODE_MASK 0x300 +#define AMDGPU_IDS_FLAGS_MODE_SHIFT 0x8 +#define AMDGPU_IDS_FLAGS_MODE_PF 0x0 +#define AMDGPU_IDS_FLAGS_MODE_VF 0x1 +#define AMDGPU_IDS_FLAGS_MODE_PT 0x2 + /* indicate if acceleration can be working */ #define AMDGPU_INFO_ACCEL_WORKING 0x00 /* get the crtc_id from the mode object id? */ -- cgit v1.3 From 1537ec85ebd7d7aa3ce1a003007cd3588cd58bda Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Tue, 25 Feb 2025 17:47:06 -0800 Subject: drm/xe/uapi: Introduce API for EU stall sampling A new hardware feature first introduced in PVC gives capability to periodically sample EU stall state and record counts for different stall reasons, on a per IP basis, aggregate across all EUs in a subslice and record the samples in a buffer in each subslice. Eventually, the aggregated data is written out to a buffer in the memory. This feature is also supported in XE2 and later architecture GPUs. Use an existing IOCTL - DRM_IOCTL_XE_OBSERVATION as the interface into the driver from the user space to do initial setup and obtain a file descriptor for the EU stall data stream. Input parameter to the IOCTL is a struct drm_xe_observation_param in which observation_type should be set to DRM_XE_OBSERVATION_TYPE_EU_STALL, observation_op should be DRM_XE_OBSERVATION_OP_STREAM_OPEN and param should point to a chain of drm_xe_ext_set_property structures in which each structure has a pair of property and value. The EU stall sampling input properties are defined in drm_xe_eu_stall_property_id enum. With the file descriptor obtained from DRM_IOCTL_XE_OBSERVATION, user space can enable and disable EU stall sampling with the IOCTLs: DRM_XE_OBSERVATION_IOCTL_ENABLE and DRM_XE_OBSERVATION_IOCTL_DISABLE. User space can also call poll() to check for availability of data in the buffer. The data can be read with read(). Finally, the file descriptor can be closed with close(). v11: Changed a couple of variables in struct eu_stall_open_properties from unsigned int to int. v10: Use extension number while parsing chain of extensions. Remove function description for static functions. Move code around as per review feedback. v9: Changed some u32 to unsigned int. Moved some code around as per review feedback from v8. v8: Used div_u64 instead of / to fix 32-bit build issue. Changed copyright year in xe_eu_stall.c/h to 2025. v7: Renamed input property DRM_XE_EU_STALL_PROP_EVENT_REPORT_COUNT to DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS to be consistent with OA. Renamed the corresponding internal variables. Fixed some commit messages based on review feedback. v6: Change the input sampling rate to GPU cycles instead of GPU cycles multiplier. Reviewed-by: Ashutosh Dixit Signed-off-by: Harish Chegondi Signed-off-by: Ashutosh Dixit Link: https://patchwork.freedesktop.org/patch/msgid/bb707a27975c33e4a912b9839b023acb7a1f9c90.1740533885.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/Makefile | 1 + drivers/gpu/drm/xe/xe_eu_stall.c | 218 ++++++++++++++++++++++++++++++++++++ drivers/gpu/drm/xe/xe_eu_stall.h | 14 +++ drivers/gpu/drm/xe/xe_observation.c | 14 +++ include/uapi/drm/xe_drm.h | 38 +++++++ 5 files changed, 285 insertions(+) create mode 100644 drivers/gpu/drm/xe/xe_eu_stall.c create mode 100644 drivers/gpu/drm/xe/xe_eu_stall.h (limited to 'include/uapi/drm') diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index ffc836fa8e60..4fb3a4a336fd 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -33,6 +33,7 @@ xe-y += xe_bb.o \ xe_device_sysfs.o \ xe_dma_buf.o \ xe_drm_client.o \ + xe_eu_stall.o \ xe_exec.o \ xe_exec_queue.o \ xe_execlist.o \ diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c new file mode 100644 index 000000000000..62a92aa161e8 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -0,0 +1,218 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include +#include +#include +#include + +#include + +#include "xe_device.h" +#include "xe_eu_stall.h" +#include "xe_gt_printk.h" +#include "xe_gt_topology.h" +#include "xe_macros.h" +#include "xe_observation.h" + +/** + * struct eu_stall_open_properties - EU stall sampling properties received + * from user space at open. + * @sampling_rate_mult: EU stall sampling rate multiplier. + * HW will sample every (sampling_rate_mult x 251) cycles. + * @wait_num_reports: Minimum number of EU stall data reports to unblock poll(). + * @gt: GT on which EU stall data will be captured. + */ +struct eu_stall_open_properties { + int sampling_rate_mult; + int wait_num_reports; + struct xe_gt *gt; +}; + +static int set_prop_eu_stall_sampling_rate(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props) +{ + value = div_u64(value, 251); + if (value == 0 || value > 7) { + drm_dbg(&xe->drm, "Invalid EU stall sampling rate %llu\n", value); + return -EINVAL; + } + props->sampling_rate_mult = value; + return 0; +} + +static int set_prop_eu_stall_wait_num_reports(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props) +{ + props->wait_num_reports = value; + + return 0; +} + +static int set_prop_eu_stall_gt_id(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props) +{ + if (value >= xe->info.gt_count) { + drm_dbg(&xe->drm, "Invalid GT ID %llu for EU stall sampling\n", value); + return -EINVAL; + } + props->gt = xe_device_get_gt(xe, value); + return 0; +} + +typedef int (*set_eu_stall_property_fn)(struct xe_device *xe, u64 value, + struct eu_stall_open_properties *props); + +static const set_eu_stall_property_fn xe_set_eu_stall_property_funcs[] = { + [DRM_XE_EU_STALL_PROP_SAMPLE_RATE] = set_prop_eu_stall_sampling_rate, + [DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS] = set_prop_eu_stall_wait_num_reports, + [DRM_XE_EU_STALL_PROP_GT_ID] = set_prop_eu_stall_gt_id, +}; + +static int xe_eu_stall_user_ext_set_property(struct xe_device *xe, u64 extension, + struct eu_stall_open_properties *props) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_ext_set_property ext; + int err; + u32 idx; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.property >= ARRAY_SIZE(xe_set_eu_stall_property_funcs)) || + XE_IOCTL_DBG(xe, ext.pad)) + return -EINVAL; + + idx = array_index_nospec(ext.property, ARRAY_SIZE(xe_set_eu_stall_property_funcs)); + return xe_set_eu_stall_property_funcs[idx](xe, ext.value, props); +} + +typedef int (*xe_eu_stall_user_extension_fn)(struct xe_device *xe, u64 extension, + struct eu_stall_open_properties *props); +static const xe_eu_stall_user_extension_fn xe_eu_stall_user_extension_funcs[] = { + [DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY] = xe_eu_stall_user_ext_set_property, +}; + +#define MAX_USER_EXTENSIONS 5 +static int xe_eu_stall_user_extensions(struct xe_device *xe, u64 extension, + int ext_number, struct eu_stall_open_properties *props) +{ + u64 __user *address = u64_to_user_ptr(extension); + struct drm_xe_user_extension ext; + int err; + u32 idx; + + if (XE_IOCTL_DBG(xe, ext_number >= MAX_USER_EXTENSIONS)) + return -E2BIG; + + err = __copy_from_user(&ext, address, sizeof(ext)); + if (XE_IOCTL_DBG(xe, err)) + return -EFAULT; + + if (XE_IOCTL_DBG(xe, ext.pad) || + XE_IOCTL_DBG(xe, ext.name >= ARRAY_SIZE(xe_eu_stall_user_extension_funcs))) + return -EINVAL; + + idx = array_index_nospec(ext.name, ARRAY_SIZE(xe_eu_stall_user_extension_funcs)); + err = xe_eu_stall_user_extension_funcs[idx](xe, extension, props); + if (XE_IOCTL_DBG(xe, err)) + return err; + + if (ext.next_extension) + return xe_eu_stall_user_extensions(xe, ext.next_extension, ++ext_number, props); + + return 0; +} + +/* + * Userspace must enable the EU stall stream with DRM_XE_OBSERVATION_IOCTL_ENABLE + * before calling read(). + */ +static ssize_t xe_eu_stall_stream_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + ssize_t ret = 0; + + return ret; +} + +static __poll_t xe_eu_stall_stream_poll(struct file *file, poll_table *wait) +{ + __poll_t ret = 0; + + return ret; +} + +static long xe_eu_stall_stream_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + return 0; +} + +static int xe_eu_stall_stream_close(struct inode *inode, struct file *file) +{ + return 0; +} + +static const struct file_operations fops_eu_stall = { + .owner = THIS_MODULE, + .llseek = noop_llseek, + .release = xe_eu_stall_stream_close, + .poll = xe_eu_stall_stream_poll, + .read = xe_eu_stall_stream_read, + .unlocked_ioctl = xe_eu_stall_stream_ioctl, + .compat_ioctl = xe_eu_stall_stream_ioctl, +}; + +static inline bool has_eu_stall_sampling_support(struct xe_device *xe) +{ + return false; +} + +/** + * xe_eu_stall_stream_open - Open a xe EU stall data stream fd + * + * @dev: DRM device pointer + * @data: pointer to first struct @drm_xe_ext_set_property in + * the chain of input properties from the user space. + * @file: DRM file pointer + * + * This function opens a EU stall data stream with input properties from + * the user space. + * + * Returns: EU stall data stream fd on success or a negative error code. + */ +int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *file) +{ + struct xe_device *xe = to_xe_device(dev); + struct eu_stall_open_properties props = {}; + int ret, stream_fd; + + if (!has_eu_stall_sampling_support(xe)) { + drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n"); + return -ENODEV; + } + + if (xe_observation_paranoid && !perfmon_capable()) { + drm_dbg(&xe->drm, "Insufficient privileges for EU stall monitoring\n"); + return -EACCES; + } + + ret = xe_eu_stall_user_extensions(xe, data, 0, &props); + if (ret) + return ret; + + if (!props.gt) { + drm_dbg(&xe->drm, "GT ID not provided for EU stall sampling\n"); + return -EINVAL; + } + + stream_fd = anon_inode_getfd("[xe_eu_stall]", &fops_eu_stall, NULL, 0); + if (stream_fd < 0) + xe_gt_dbg(props.gt, "EU stall inode get fd failed : %d\n", stream_fd); + + return stream_fd; +} diff --git a/drivers/gpu/drm/xe/xe_eu_stall.h b/drivers/gpu/drm/xe/xe_eu_stall.h new file mode 100644 index 000000000000..c1aef8adac6e --- /dev/null +++ b/drivers/gpu/drm/xe/xe_eu_stall.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef __XE_EU_STALL_H__ +#define __XE_EU_STALL_H__ + +#include "xe_gt_types.h" + +int xe_eu_stall_stream_open(struct drm_device *dev, + u64 data, + struct drm_file *file); +#endif diff --git a/drivers/gpu/drm/xe/xe_observation.c b/drivers/gpu/drm/xe/xe_observation.c index 8ec1b84cbb9e..5011d0736644 100644 --- a/drivers/gpu/drm/xe/xe_observation.c +++ b/drivers/gpu/drm/xe/xe_observation.c @@ -8,6 +8,7 @@ #include +#include "xe_eu_stall.h" #include "xe_oa.h" #include "xe_observation.h" @@ -29,6 +30,17 @@ static int xe_oa_ioctl(struct drm_device *dev, struct drm_xe_observation_param * } } +static int xe_eu_stall_ioctl(struct drm_device *dev, struct drm_xe_observation_param *arg, + struct drm_file *file) +{ + switch (arg->observation_op) { + case DRM_XE_OBSERVATION_OP_STREAM_OPEN: + return xe_eu_stall_stream_open(dev, arg->param, file); + default: + return -EINVAL; + } +} + /** * xe_observation_ioctl - The top level observation layer ioctl * @dev: @drm_device @@ -51,6 +63,8 @@ int xe_observation_ioctl(struct drm_device *dev, void *data, struct drm_file *fi switch (arg->observation_type) { case DRM_XE_OBSERVATION_TYPE_OA: return xe_oa_ioctl(dev, arg, file); + case DRM_XE_OBSERVATION_TYPE_EU_STALL: + return xe_eu_stall_ioctl(dev, arg, file); default: return -EINVAL; } diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 892f54d3aa09..95cb9e65540b 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -1496,6 +1496,8 @@ struct drm_xe_wait_user_fence { enum drm_xe_observation_type { /** @DRM_XE_OBSERVATION_TYPE_OA: OA observation stream type */ DRM_XE_OBSERVATION_TYPE_OA, + /** @DRM_XE_OBSERVATION_TYPE_EU_STALL: EU stall sampling observation stream type */ + DRM_XE_OBSERVATION_TYPE_EU_STALL, }; /** @@ -1848,6 +1850,42 @@ enum drm_xe_pxp_session_type { /* ID of the protected content session managed by Xe when PXP is active */ #define DRM_XE_PXP_HWDRM_DEFAULT_SESSION 0xf +/** + * enum drm_xe_eu_stall_property_id - EU stall sampling input property ids. + * + * These properties are passed to the driver at open as a chain of + * @drm_xe_ext_set_property structures with @property set to these + * properties' enums and @value set to the corresponding values of these + * properties. @drm_xe_user_extension base.name should be set to + * @DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY. + * + * With the file descriptor obtained from open, user space must enable + * the EU stall stream fd with @DRM_XE_OBSERVATION_IOCTL_ENABLE before + * calling read(). EIO errno from read() indicates HW dropped data + * due to full buffer. + */ +enum drm_xe_eu_stall_property_id { +#define DRM_XE_EU_STALL_EXTENSION_SET_PROPERTY 0 + /** + * @DRM_XE_EU_STALL_PROP_GT_ID: @gt_id of the GT on which + * EU stall data will be captured. + */ + DRM_XE_EU_STALL_PROP_GT_ID = 1, + + /** + * @DRM_XE_EU_STALL_PROP_SAMPLE_RATE: Sampling rate + * in GPU cycles. + */ + DRM_XE_EU_STALL_PROP_SAMPLE_RATE, + + /** + * @DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS: Minimum number of + * EU stall data reports to be present in the kernel buffer + * before unblocking a blocked poll or read. + */ + DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS, +}; + #if defined(__cplusplus) } #endif -- cgit v1.3 From cd5bbb2532f276626d1416b237676772bb61f11e Mon Sep 17 00:00:00 2001 From: Harish Chegondi Date: Tue, 25 Feb 2025 17:47:11 -0800 Subject: drm/xe/uapi: Add a device query to get EU stall sampling information User space can get the EU stall data record size, EU stall capabilities, EU stall sampling rates, and per XeCore buffer size with query IOCTL DRM_IOCTL_XE_DEVICE_QUERY with .query set to DRM_XE_DEVICE_QUERY_EU_STALL. A struct drm_xe_query_eu_stall will be returned to the user space along with an array of supported sampling rates sorted in the fastest sampling rate first order. sampling_rates in struct drm_xe_query_eu_stall will point to the array of sampling rates. Any capabilities in EU stall sampling as of this patch are considered as base capabilities. New capability bits will be added for any new functionality added later. v12: Rename has_eu_stall_sampling_support() to xe_eu_stall_supported_on_platform() and move it to header file. v11: Check if EU stall sampling is supported on the platform. v10: Change comments and variable names as per feedback v9: Move reserved fields above num_sampling_rates in struct drm_xe_query_eu_stall. v7: Change sampling_rates from a pointer to flexible array. v6: Include EU stall sampling rates information and per XeCore buffer size in the query information. Reviewed-by: Ashutosh Dixit Signed-off-by: Harish Chegondi Signed-off-by: Ashutosh Dixit Link: https://patchwork.freedesktop.org/patch/msgid/67ba42796a5a99d648239c315694cd222812a49b.1740533885.git.harish.chegondi@intel.com --- drivers/gpu/drm/xe/xe_eu_stall.c | 48 ++++++++++++++++++++++++++++++++++------ drivers/gpu/drm/xe/xe_eu_stall.h | 9 ++++++++ drivers/gpu/drm/xe/xe_query.c | 43 +++++++++++++++++++++++++++++++++++ include/uapi/drm/xe_drm.h | 40 +++++++++++++++++++++++++++++++-- 4 files changed, 131 insertions(+), 9 deletions(-) (limited to 'include/uapi/drm') diff --git a/drivers/gpu/drm/xe/xe_eu_stall.c b/drivers/gpu/drm/xe/xe_eu_stall.c index 87f978373bd0..54a0d1f6a491 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.c +++ b/drivers/gpu/drm/xe/xe_eu_stall.c @@ -120,7 +120,46 @@ struct xe_eu_stall_data_xe2 { __u64 unused[6]; } __packed; -static size_t xe_eu_stall_data_record_size(struct xe_device *xe) +const u64 eu_stall_sampling_rates[] = {251, 251 * 2, 251 * 3, 251 * 4, 251 * 5, 251 * 6, 251 * 7}; + +/** + * xe_eu_stall_get_sampling_rates - get EU stall sampling rates information. + * + * @num_rates: Pointer to a u32 to return the number of sampling rates. + * @rates: double u64 pointer to point to an array of sampling rates. + * + * Stores the number of sampling rates and pointer to the array of + * sampling rates in the input pointers. + * + * Returns: Size of the EU stall sampling rates array. + */ +size_t xe_eu_stall_get_sampling_rates(u32 *num_rates, const u64 **rates) +{ + *num_rates = ARRAY_SIZE(eu_stall_sampling_rates); + *rates = eu_stall_sampling_rates; + + return sizeof(eu_stall_sampling_rates); +} + +/** + * xe_eu_stall_get_per_xecore_buf_size - get per XeCore buffer size. + * + * Returns: The per XeCore buffer size used to allocate the per GT + * EU stall data buffer. + */ +size_t xe_eu_stall_get_per_xecore_buf_size(void) +{ + return per_xecore_buf_size; +} + +/** + * xe_eu_stall_data_record_size - get EU stall data record size. + * + * @xe: Pointer to a Xe device. + * + * Returns: EU stall data record size. + */ +size_t xe_eu_stall_data_record_size(struct xe_device *xe) { size_t record_size = 0; @@ -812,11 +851,6 @@ static const struct file_operations fops_eu_stall = { .compat_ioctl = xe_eu_stall_stream_ioctl, }; -static inline bool has_eu_stall_sampling_support(struct xe_device *xe) -{ - return xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20; -} - static int xe_eu_stall_stream_open_locked(struct drm_device *dev, struct eu_stall_open_properties *props, struct drm_file *file) @@ -885,7 +919,7 @@ int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *f struct eu_stall_open_properties props = {}; int ret; - if (!has_eu_stall_sampling_support(xe)) { + if (!xe_eu_stall_supported_on_platform(xe)) { drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n"); return -ENODEV; } diff --git a/drivers/gpu/drm/xe/xe_eu_stall.h b/drivers/gpu/drm/xe/xe_eu_stall.h index 24e215b840c0..ed9d0f233566 100644 --- a/drivers/gpu/drm/xe/xe_eu_stall.h +++ b/drivers/gpu/drm/xe/xe_eu_stall.h @@ -8,8 +8,17 @@ #include "xe_gt_types.h" +size_t xe_eu_stall_get_per_xecore_buf_size(void); +size_t xe_eu_stall_data_record_size(struct xe_device *xe); +size_t xe_eu_stall_get_sampling_rates(u32 *num_rates, const u64 **rates); + int xe_eu_stall_init(struct xe_gt *gt); int xe_eu_stall_stream_open(struct drm_device *dev, u64 data, struct drm_file *file); + +static inline bool xe_eu_stall_supported_on_platform(struct xe_device *xe) +{ + return xe->info.platform == XE_PVC || GRAPHICS_VER(xe) >= 20; +} #endif diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index ebfae746f861..781dd21682e5 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -16,6 +16,7 @@ #include "regs/xe_gt_regs.h" #include "xe_bo.h" #include "xe_device.h" +#include "xe_eu_stall.h" #include "xe_exec_queue.h" #include "xe_force_wake.h" #include "xe_ggtt.h" @@ -729,6 +730,47 @@ static int query_pxp_status(struct xe_device *xe, struct drm_xe_device_query *qu return 0; } +static int query_eu_stall(struct xe_device *xe, + struct drm_xe_device_query *query) +{ + void __user *query_ptr = u64_to_user_ptr(query->data); + struct drm_xe_query_eu_stall *info; + size_t size, array_size; + const u64 *rates; + u32 num_rates; + int ret; + + if (!xe_eu_stall_supported_on_platform(xe)) { + drm_dbg(&xe->drm, "EU stall monitoring is not supported on this platform\n"); + return -ENODEV; + } + + array_size = xe_eu_stall_get_sampling_rates(&num_rates, &rates); + size = sizeof(struct drm_xe_query_eu_stall) + array_size; + + if (query->size == 0) { + query->size = size; + return 0; + } else if (XE_IOCTL_DBG(xe, query->size != size)) { + return -EINVAL; + } + + info = kzalloc(size, GFP_KERNEL); + if (!info) + return -ENOMEM; + + info->num_sampling_rates = num_rates; + info->capabilities = DRM_XE_EU_STALL_CAPS_BASE; + info->record_size = xe_eu_stall_data_record_size(xe); + info->per_xecore_buf_size = xe_eu_stall_get_per_xecore_buf_size(); + memcpy(info->sampling_rates, rates, array_size); + + ret = copy_to_user(query_ptr, info, size); + kfree(info); + + return ret ? -EFAULT : 0; +} + static int (* const xe_query_funcs[])(struct xe_device *xe, struct drm_xe_device_query *query) = { query_engines, @@ -741,6 +783,7 @@ static int (* const xe_query_funcs[])(struct xe_device *xe, query_uc_fw_version, query_oa_units, query_pxp_status, + query_eu_stall, }; int xe_query_ioctl(struct drm_device *dev, void *data, struct drm_file *file) diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 95cb9e65540b..76a462fae05f 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -735,6 +735,7 @@ struct drm_xe_device_query { #define DRM_XE_DEVICE_QUERY_UC_FW_VERSION 7 #define DRM_XE_DEVICE_QUERY_OA_UNITS 8 #define DRM_XE_DEVICE_QUERY_PXP_STATUS 9 +#define DRM_XE_DEVICE_QUERY_EU_STALL 10 /** @query: The type of data to query */ __u32 query; @@ -1873,8 +1874,8 @@ enum drm_xe_eu_stall_property_id { DRM_XE_EU_STALL_PROP_GT_ID = 1, /** - * @DRM_XE_EU_STALL_PROP_SAMPLE_RATE: Sampling rate - * in GPU cycles. + * @DRM_XE_EU_STALL_PROP_SAMPLE_RATE: Sampling rate in + * GPU cycles from @sampling_rates in struct @drm_xe_query_eu_stall */ DRM_XE_EU_STALL_PROP_SAMPLE_RATE, @@ -1886,6 +1887,41 @@ enum drm_xe_eu_stall_property_id { DRM_XE_EU_STALL_PROP_WAIT_NUM_REPORTS, }; +/** + * struct drm_xe_query_eu_stall - Information about EU stall sampling. + * + * If a query is made with a struct @drm_xe_device_query where .query + * is equal to @DRM_XE_DEVICE_QUERY_EU_STALL, then the reply uses + * struct @drm_xe_query_eu_stall in .data. + */ +struct drm_xe_query_eu_stall { + /** @extensions: Pointer to the first extension struct, if any */ + __u64 extensions; + + /** @capabilities: EU stall capabilities bit-mask */ + __u64 capabilities; +#define DRM_XE_EU_STALL_CAPS_BASE (1 << 0) + + /** @record_size: size of each EU stall data record */ + __u64 record_size; + + /** @per_xecore_buf_size: internal per XeCore buffer size */ + __u64 per_xecore_buf_size; + + /** @reserved: Reserved */ + __u64 reserved[5]; + + /** @num_sampling_rates: Number of sampling rates in @sampling_rates array */ + __u64 num_sampling_rates; + + /** + * @sampling_rates: Flexible array of sampling rates + * sorted in the fastest to slowest order. + * Sampling rates are specified in GPU clock cycles. + */ + __u64 sampling_rates[]; +}; + #if defined(__cplusplus) } #endif -- cgit v1.3 From 5488bec96bccbd87335921338f8dc38b87db7d2c Mon Sep 17 00:00:00 2001 From: Tejas Upadhyay Date: Fri, 28 Feb 2025 12:32:24 +0530 Subject: drm/xe/uapi: Use hint for guc to set GT frequency Allow user to provide a low latency hint. When set, KMD sends a hint to GuC which results in special handling for that process. SLPC will ramp the GT frequency aggressively every time it switches to this process. We need to enable the use of SLPC Compute strategy during init, but it will apply only to processes that set this bit during process creation. Improvement with this approach as below: Before, :~$ NEOReadDebugKeys=1 EnableDirectSubmission=0 clpeak --kernel-latency Platform: Intel(R) OpenCL Graphics Device: Intel(R) Graphics [0xe20b] Driver version : 24.52.0 (Linux x64) Compute units : 160 Clock frequency : 2850 MHz Kernel launch latency : 283.16 us After, :~$ NEOReadDebugKeys=1 EnableDirectSubmission=0 clpeak --kernel-latency Platform: Intel(R) OpenCL Graphics Device: Intel(R) Graphics [0xe20b] Driver version : 24.52.0 (Linux x64) Compute units : 160 Clock frequency : 2850 MHz Kernel launch latency : 63.38 us Compute PR: https://github.com/intel/compute-runtime/pull/794 Mesa PR: https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/33214 IGT PR: https://patchwork.freedesktop.org/patch/639989/ V10(Lucas): - Remove doc from drm-uapi.rst v9(Vinay): - remove extra line, align commit message v8(Vinay): - Add separate example for using low latency hint v7(Jose): - Update UMD PR - applicable to all gpus V6: - init flags, remove redundant flags check (MAuld) V5: - Move uapi doc to documentation and GuC ABI specific change (Rodrigo) - Modify logic to restrict exec queue flags (MAuld) V4: - To make it clear, dont use exec queue word (Vinay) - Correct typo in description of flag (Jose/Vinay) - rename set_strategy api and replace ctx with exec queue(Vinay) - Start with 0th bit to indentify user flags (Jose) V3: - Conver user flag to kernel internal flag and use (Oak) - Support query config for use to check kernel support (Jose) - Dont need to take runtime pm (Vinay) V2: - DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT 1 planned for other hint(Szymon) - Add motivation to description (Lucas) Acked-by: Lucas De Marchi Reviewed-by: Vinay Belgaumkar Link: https://patchwork.freedesktop.org/patch/msgid/20250228070224.739295-2-tejas.upadhyay@intel.com Signed-off-by: Tejas Upadhyay --- drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h | 3 +++ drivers/gpu/drm/xe/xe_exec_queue.c | 10 +++++++--- drivers/gpu/drm/xe/xe_exec_queue_types.h | 2 ++ drivers/gpu/drm/xe/xe_guc_pc.c | 16 ++++++++++++++++ drivers/gpu/drm/xe/xe_guc_submit.c | 8 ++++++++ drivers/gpu/drm/xe/xe_query.c | 2 ++ include/uapi/drm/xe_drm.h | 21 ++++++++++++++++++++- 7 files changed, 58 insertions(+), 4 deletions(-) (limited to 'include/uapi/drm') diff --git a/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h b/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h index 85abe4f09ae2..b28c8fa061f7 100644 --- a/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h +++ b/drivers/gpu/drm/xe/abi/guc_actions_slpc_abi.h @@ -174,6 +174,9 @@ struct slpc_task_state_data { }; } __packed; +#define SLPC_CTX_FREQ_REQ_IS_COMPUTE REG_BIT(28) +#define SLPC_OPTIMIZED_STRATEGY_COMPUTE REG_BIT(0) + struct slpc_shared_data_header { /* Total size in bytes of this shared buffer. */ u32 size; diff --git a/drivers/gpu/drm/xe/xe_exec_queue.c b/drivers/gpu/drm/xe/xe_exec_queue.c index 23a9f519ce1c..7c5c003d3c40 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue.c +++ b/drivers/gpu/drm/xe/xe_exec_queue.c @@ -604,11 +604,12 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, struct xe_tile *tile; struct xe_exec_queue *q = NULL; u32 logical_mask; + u32 flags = 0; u32 id; u32 len; int err; - if (XE_IOCTL_DBG(xe, args->flags) || + if (XE_IOCTL_DBG(xe, args->flags & ~DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) || XE_IOCTL_DBG(xe, args->reserved[0] || args->reserved[1])) return -EINVAL; @@ -625,6 +626,9 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, if (XE_IOCTL_DBG(xe, eci[0].gt_id >= xe->info.gt_count)) return -EINVAL; + if (args->flags & DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT) + flags |= EXEC_QUEUE_FLAG_LOW_LATENCY; + if (eci[0].engine_class == DRM_XE_ENGINE_CLASS_VM_BIND) { if (XE_IOCTL_DBG(xe, args->width != 1) || XE_IOCTL_DBG(xe, args->num_placements != 1) || @@ -633,8 +637,8 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, for_each_tile(tile, xe, id) { struct xe_exec_queue *new; - u32 flags = EXEC_QUEUE_FLAG_VM; + flags |= EXEC_QUEUE_FLAG_VM; if (id) flags |= EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD; @@ -680,7 +684,7 @@ int xe_exec_queue_create_ioctl(struct drm_device *dev, void *data, } q = xe_exec_queue_create(xe, vm, logical_mask, - args->width, hwe, 0, + args->width, hwe, flags, args->extensions); up_read(&vm->lock); xe_vm_put(vm); diff --git a/drivers/gpu/drm/xe/xe_exec_queue_types.h b/drivers/gpu/drm/xe/xe_exec_queue_types.h index 6eb7ff091534..cc1cffb5c87f 100644 --- a/drivers/gpu/drm/xe/xe_exec_queue_types.h +++ b/drivers/gpu/drm/xe/xe_exec_queue_types.h @@ -85,6 +85,8 @@ struct xe_exec_queue { #define EXEC_QUEUE_FLAG_BIND_ENGINE_CHILD BIT(3) /* kernel exec_queue only, set priority to highest level */ #define EXEC_QUEUE_FLAG_HIGH_PRIORITY BIT(4) +/* flag to indicate low latency hint to guc */ +#define EXEC_QUEUE_FLAG_LOW_LATENCY BIT(5) /** * @flags: flags for this exec queue, should statically setup aside from ban diff --git a/drivers/gpu/drm/xe/xe_guc_pc.c b/drivers/gpu/drm/xe/xe_guc_pc.c index 02409eedb914..25040efa043f 100644 --- a/drivers/gpu/drm/xe/xe_guc_pc.c +++ b/drivers/gpu/drm/xe/xe_guc_pc.c @@ -995,6 +995,17 @@ out: return ret; } +static int pc_action_set_strategy(struct xe_guc_pc *pc, u32 val) +{ + int ret = 0; + + ret = pc_action_set_param(pc, + SLPC_PARAM_STRATEGIES, + val); + + return ret; +} + /** * xe_guc_pc_start - Start GuC's Power Conservation component * @pc: Xe_GuC_PC instance @@ -1054,6 +1065,11 @@ int xe_guc_pc_start(struct xe_guc_pc *pc) } ret = pc_action_setup_gucrc(pc, GUCRC_FIRMWARE_CONTROL); + if (ret) + goto out; + + /* Enable SLPC Optimized Strategy for compute */ + ret = pc_action_set_strategy(pc, SLPC_OPTIMIZED_STRATEGY_COMPUTE); out: xe_force_wake_put(gt_to_fw(gt), fw_ref); diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index b6a2dd742ebd..b95934055f72 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -15,6 +15,7 @@ #include #include "abi/guc_actions_abi.h" +#include "abi/guc_actions_slpc_abi.h" #include "abi/guc_klvs_abi.h" #include "regs/xe_lrc_layout.h" #include "xe_assert.h" @@ -400,6 +401,7 @@ static void __guc_exec_queue_policy_add_##func(struct exec_queue_policy *policy, MAKE_EXEC_QUEUE_POLICY_ADD(execution_quantum, EXECUTION_QUANTUM) MAKE_EXEC_QUEUE_POLICY_ADD(preemption_timeout, PREEMPTION_TIMEOUT) MAKE_EXEC_QUEUE_POLICY_ADD(priority, SCHEDULING_PRIORITY) +MAKE_EXEC_QUEUE_POLICY_ADD(slpc_exec_queue_freq_req, SLPM_GT_FREQUENCY) #undef MAKE_EXEC_QUEUE_POLICY_ADD static const int xe_exec_queue_prio_to_guc[] = { @@ -414,14 +416,20 @@ static void init_policies(struct xe_guc *guc, struct xe_exec_queue *q) struct exec_queue_policy policy; enum xe_exec_queue_priority prio = q->sched_props.priority; u32 timeslice_us = q->sched_props.timeslice_us; + u32 slpc_exec_queue_freq_req = 0; u32 preempt_timeout_us = q->sched_props.preempt_timeout_us; xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); + if (q->flags & EXEC_QUEUE_FLAG_LOW_LATENCY) + slpc_exec_queue_freq_req |= SLPC_CTX_FREQ_REQ_IS_COMPUTE; + __guc_exec_queue_policy_start_klv(&policy, q->guc->id); __guc_exec_queue_policy_add_priority(&policy, xe_exec_queue_prio_to_guc[prio]); __guc_exec_queue_policy_add_execution_quantum(&policy, timeslice_us); __guc_exec_queue_policy_add_preemption_timeout(&policy, preempt_timeout_us); + __guc_exec_queue_policy_add_slpc_exec_queue_freq_req(&policy, + slpc_exec_queue_freq_req); xe_guc_ct_send(&guc->ct, (u32 *)&policy.h2g, __guc_exec_queue_policy_action_size(&policy), 0, 0); diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index 781dd21682e5..ce2a2767de1a 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -340,6 +340,8 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) if (xe_device_get_root_tile(xe)->mem.vram.usable_size) config->info[DRM_XE_QUERY_CONFIG_FLAGS] = DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM; + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= + DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY; config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] = xe->info.vram_flags & XE_VRAM_FLAGS_NEED64K ? SZ_64K : SZ_4K; config->info[DRM_XE_QUERY_CONFIG_VA_BITS] = xe->info.va_bits; diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index 76a462fae05f..d1f0018342b6 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -393,6 +393,8 @@ struct drm_xe_query_mem_regions { * * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM - Flag is set if the device * has usable VRAM + * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device + * has low latency hint support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address @@ -409,6 +411,7 @@ struct drm_xe_query_config { #define DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID 0 #define DRM_XE_QUERY_CONFIG_FLAGS 1 #define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) + #define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) #define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2 #define DRM_XE_QUERY_CONFIG_VA_BITS 3 #define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4 @@ -1205,6 +1208,21 @@ struct drm_xe_vm_bind { * }; * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); * + * Allow users to provide a hint to kernel for cases demanding low latency + * profile. Please note it will have impact on power consumption. User can + * indicate low latency hint with flag while creating exec queue as + * mentioned below, + * + * struct drm_xe_exec_queue_create exec_queue_create = { + * .flags = DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT, + * .extensions = 0, + * .vm_id = vm, + * .num_bb_per_exec = 1, + * .num_eng_per_bb = 1, + * .instances = to_user_pointer(&instance), + * }; + * ioctl(fd, DRM_IOCTL_XE_EXEC_QUEUE_CREATE, &exec_queue_create); + * */ struct drm_xe_exec_queue_create { #define DRM_XE_EXEC_QUEUE_EXTENSION_SET_PROPERTY 0 @@ -1223,7 +1241,8 @@ struct drm_xe_exec_queue_create { /** @vm_id: VM to use for this exec queue */ __u32 vm_id; - /** @flags: MBZ */ +#define DRM_XE_EXEC_QUEUE_LOW_LATENCY_HINT (1 << 0) + /** @flags: flags to use for this exec queue */ __u32 flags; /** @exec_queue_id: Returned exec queue ID */ -- cgit v1.3 From b43e864af0d4e74636c0e1dee857ce3275a84829 Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 5 Mar 2025 17:26:33 -0800 Subject: drm/xe/uapi: Add DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR flag, which is used to create unpopulated virtual memory areas (VMAs) without memory backing or GPU page tables. These VMAs are referred to as CPU address mirror VMAs. The idea is that upon a page fault or prefetch, the memory backing and GPU page tables will be populated. CPU address mirror VMAs only update GPUVM state; they do not have an internal page table (PT) state, nor do they have GPU mappings. It is expected that CPU address mirror VMAs will be mixed with buffer object (BO) VMAs within a single VM. In other words, system allocations and runtime allocations can be mixed within a single user-mode driver (UMD) program. Expected usage: - Bind the entire virtual address (VA) space upon program load using the DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR flag. - If a buffer object (BO) requires GPU mapping (runtime allocation), allocate a CPU address using mmap(PROT_NONE), bind the BO to the mmapped address using existing bind IOCTLs. If a CPU map of the BO is needed, mmap it again to the same CPU address using mmap(MAP_FIXED) - If a BO no longer requires GPU mapping, munmap it from the CPU address space and them bind the mapping address with the DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR flag. - Any malloc'd or mmapped CPU address accessed by the GPU will be faulted in via the SVM implementation (system allocation). - Upon freeing any mmapped or malloc'd data, the SVM implementation will remove GPU mappings. Only supporting 1 to 1 mapping between user address space and GPU address space at the moment as that is the expected use case. uAPI defines interface for non 1 to 1 but enforces 1 to 1, this restriction can be lifted if use cases arrise for non 1 to 1 mappings. This patch essentially short-circuits the code in the existing VM bind paths to avoid populating page tables when the DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR flag is set. v3: - Call vm_bind_ioctl_ops_fini on -ENODATA - Don't allow DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR on non-faulting VMs - s/DRM_XE_VM_BIND_FLAG_SYSTEM_ALLOCATOR/DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (Thomas) - Rework commit message for expected usage (Thomas) - Describe state of code after patch in commit message (Thomas) v4: - Fix alignment (Checkpatch) Signed-off-by: Matthew Brost Reviewed-by: Thomas Hellström Reviewed-by: Himal Prasad Ghimiray Link: https://patchwork.freedesktop.org/patch/msgid/20250306012657.3505757-9-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_pt.c | 76 ++++++++++++++++---- drivers/gpu/drm/xe/xe_vm.c | 145 +++++++++++++++++++++++---------------- drivers/gpu/drm/xe/xe_vm.h | 8 ++- drivers/gpu/drm/xe/xe_vm_types.h | 3 + include/uapi/drm/xe_drm.h | 19 ++++- 5 files changed, 176 insertions(+), 75 deletions(-) (limited to 'include/uapi/drm') diff --git a/drivers/gpu/drm/xe/xe_pt.c b/drivers/gpu/drm/xe/xe_pt.c index dc24baa84092..651512023829 100644 --- a/drivers/gpu/drm/xe/xe_pt.c +++ b/drivers/gpu/drm/xe/xe_pt.c @@ -1090,6 +1090,11 @@ static int op_add_deps(struct xe_vm *vm, struct xe_vma_op *op, { int err = 0; + /* + * No need to check for is_cpu_addr_mirror here as vma_add_deps is a + * NOP if VMA is_cpu_addr_mirror + */ + switch (op->base.op) { case DRM_GPUVA_OP_MAP: if (!op->map.immediate && xe_vm_in_fault_mode(vm)) @@ -1648,6 +1653,7 @@ static int bind_op_prepare(struct xe_vm *vm, struct xe_tile *tile, struct xe_vm_pgtable_update_op *pt_op = &pt_update_ops->ops[current_op]; int err; + xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); xe_bo_assert_held(xe_vma_bo(vma)); vm_dbg(&xe_vma_vm(vma)->xe->drm, @@ -1715,6 +1721,7 @@ static int unbind_op_prepare(struct xe_tile *tile, if (!((vma->tile_present | vma->tile_staged) & BIT(tile->id))) return 0; + xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); xe_bo_assert_held(xe_vma_bo(vma)); vm_dbg(&xe_vma_vm(vma)->xe->drm, @@ -1761,15 +1768,21 @@ static int op_prepare(struct xe_vm *vm, switch (op->base.op) { case DRM_GPUVA_OP_MAP: - if (!op->map.immediate && xe_vm_in_fault_mode(vm)) + if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) || + op->map.is_cpu_addr_mirror) break; err = bind_op_prepare(vm, tile, pt_update_ops, op->map.vma); pt_update_ops->wait_vm_kernel = true; break; case DRM_GPUVA_OP_REMAP: - err = unbind_op_prepare(tile, pt_update_ops, - gpuva_to_vma(op->base.remap.unmap->va)); + { + struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va); + + if (xe_vma_is_cpu_addr_mirror(old)) + break; + + err = unbind_op_prepare(tile, pt_update_ops, old); if (!err && op->remap.prev) { err = bind_op_prepare(vm, tile, pt_update_ops, @@ -1782,15 +1795,28 @@ static int op_prepare(struct xe_vm *vm, pt_update_ops->wait_vm_bookkeep = true; } break; + } case DRM_GPUVA_OP_UNMAP: - err = unbind_op_prepare(tile, pt_update_ops, - gpuva_to_vma(op->base.unmap.va)); + { + struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); + + if (xe_vma_is_cpu_addr_mirror(vma)) + break; + + err = unbind_op_prepare(tile, pt_update_ops, vma); break; + } case DRM_GPUVA_OP_PREFETCH: - err = bind_op_prepare(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.prefetch.va)); + { + struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); + + if (xe_vma_is_cpu_addr_mirror(vma)) + break; + + err = bind_op_prepare(vm, tile, pt_update_ops, vma); pt_update_ops->wait_vm_kernel = true; break; + } default: drm_warn(&vm->xe->drm, "NOT POSSIBLE"); } @@ -1860,6 +1886,8 @@ static void bind_op_commit(struct xe_vm *vm, struct xe_tile *tile, struct xe_vma *vma, struct dma_fence *fence, struct dma_fence *fence2) { + xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, pt_update_ops->wait_vm_bookkeep ? @@ -1893,6 +1921,8 @@ static void unbind_op_commit(struct xe_vm *vm, struct xe_tile *tile, struct xe_vma *vma, struct dma_fence *fence, struct dma_fence *fence2) { + xe_tile_assert(tile, !xe_vma_is_cpu_addr_mirror(vma)); + if (!xe_vma_has_no_bo(vma) && !xe_vma_bo(vma)->vm) { dma_resv_add_fence(xe_vma_bo(vma)->ttm.base.resv, fence, pt_update_ops->wait_vm_bookkeep ? @@ -1927,16 +1957,21 @@ static void op_commit(struct xe_vm *vm, switch (op->base.op) { case DRM_GPUVA_OP_MAP: - if (!op->map.immediate && xe_vm_in_fault_mode(vm)) + if ((!op->map.immediate && xe_vm_in_fault_mode(vm)) || + op->map.is_cpu_addr_mirror) break; bind_op_commit(vm, tile, pt_update_ops, op->map.vma, fence, fence2); break; case DRM_GPUVA_OP_REMAP: - unbind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.remap.unmap->va), fence, - fence2); + { + struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va); + + if (xe_vma_is_cpu_addr_mirror(old)) + break; + + unbind_op_commit(vm, tile, pt_update_ops, old, fence, fence2); if (op->remap.prev) bind_op_commit(vm, tile, pt_update_ops, op->remap.prev, @@ -1945,14 +1980,25 @@ static void op_commit(struct xe_vm *vm, bind_op_commit(vm, tile, pt_update_ops, op->remap.next, fence, fence2); break; + } case DRM_GPUVA_OP_UNMAP: - unbind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.unmap.va), fence, fence2); + { + struct xe_vma *vma = gpuva_to_vma(op->base.unmap.va); + + if (!xe_vma_is_cpu_addr_mirror(vma)) + unbind_op_commit(vm, tile, pt_update_ops, vma, fence, + fence2); break; + } case DRM_GPUVA_OP_PREFETCH: - bind_op_commit(vm, tile, pt_update_ops, - gpuva_to_vma(op->base.prefetch.va), fence, fence2); + { + struct xe_vma *vma = gpuva_to_vma(op->base.prefetch.va); + + if (!xe_vma_is_cpu_addr_mirror(vma)) + bind_op_commit(vm, tile, pt_update_ops, vma, fence, + fence2); break; + } default: drm_warn(&vm->xe->drm, "NOT POSSIBLE"); } diff --git a/drivers/gpu/drm/xe/xe_vm.c b/drivers/gpu/drm/xe/xe_vm.c index 338d98533fae..6d1730902c3e 100644 --- a/drivers/gpu/drm/xe/xe_vm.c +++ b/drivers/gpu/drm/xe/xe_vm.c @@ -956,9 +956,10 @@ static void xe_vma_free(struct xe_vma *vma) kfree(vma); } -#define VMA_CREATE_FLAG_READ_ONLY BIT(0) -#define VMA_CREATE_FLAG_IS_NULL BIT(1) -#define VMA_CREATE_FLAG_DUMPABLE BIT(2) +#define VMA_CREATE_FLAG_READ_ONLY BIT(0) +#define VMA_CREATE_FLAG_IS_NULL BIT(1) +#define VMA_CREATE_FLAG_DUMPABLE BIT(2) +#define VMA_CREATE_FLAG_IS_SYSTEM_ALLOCATOR BIT(3) static struct xe_vma *xe_vma_create(struct xe_vm *vm, struct xe_bo *bo, @@ -972,6 +973,8 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, bool read_only = (flags & VMA_CREATE_FLAG_READ_ONLY); bool is_null = (flags & VMA_CREATE_FLAG_IS_NULL); bool dumpable = (flags & VMA_CREATE_FLAG_DUMPABLE); + bool is_cpu_addr_mirror = + (flags & VMA_CREATE_FLAG_IS_SYSTEM_ALLOCATOR); xe_assert(vm->xe, start < end); xe_assert(vm->xe, end < vm->size); @@ -980,7 +983,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, * Allocate and ensure that the xe_vma_is_userptr() return * matches what was allocated. */ - if (!bo && !is_null) { + if (!bo && !is_null && !is_cpu_addr_mirror) { struct xe_userptr_vma *uvma = kzalloc(sizeof(*uvma), GFP_KERNEL); if (!uvma) @@ -992,6 +995,8 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, if (!vma) return ERR_PTR(-ENOMEM); + if (is_cpu_addr_mirror) + vma->gpuva.flags |= XE_VMA_SYSTEM_ALLOCATOR; if (is_null) vma->gpuva.flags |= DRM_GPUVA_SPARSE; if (bo) @@ -1034,7 +1039,7 @@ static struct xe_vma *xe_vma_create(struct xe_vm *vm, drm_gpuva_link(&vma->gpuva, vm_bo); drm_gpuvm_bo_put(vm_bo); } else /* userptr or null */ { - if (!is_null) { + if (!is_null && !is_cpu_addr_mirror) { struct xe_userptr *userptr = &to_userptr_vma(vma)->userptr; u64 size = end - start + 1; int err; @@ -1086,7 +1091,7 @@ static void xe_vma_destroy_late(struct xe_vma *vma) mmu_interval_notifier_remove(&userptr->notifier); mutex_destroy(&userptr->unmap_mutex); xe_vm_put(vm); - } else if (xe_vma_is_null(vma)) { + } else if (xe_vma_is_null(vma) || xe_vma_is_cpu_addr_mirror(vma)) { xe_vm_put(vm); } else { xe_bo_put(xe_vma_bo(vma)); @@ -1126,7 +1131,7 @@ static void xe_vma_destroy(struct xe_vma *vma, struct dma_fence *fence) xe_assert(vm->xe, list_empty(&to_userptr_vma(vma)->userptr.repin_link)); list_del(&to_userptr_vma(vma)->userptr.invalidate_link); spin_unlock(&vm->userptr.invalidated_lock); - } else if (!xe_vma_is_null(vma)) { + } else if (!xe_vma_is_null(vma) && !xe_vma_is_cpu_addr_mirror(vma)) { xe_bo_assert_held(xe_vma_bo(vma)); drm_gpuva_unlink(&vma->gpuva); @@ -2046,6 +2051,8 @@ vm_bind_ioctl_ops_create(struct xe_vm *vm, struct xe_bo *bo, op->map.read_only = flags & DRM_XE_VM_BIND_FLAG_READONLY; op->map.is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; + op->map.is_cpu_addr_mirror = flags & + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR; op->map.dumpable = flags & DRM_XE_VM_BIND_FLAG_DUMPABLE; op->map.pat_index = pat_index; } else if (__op->op == DRM_GPUVA_OP_PREFETCH) { @@ -2238,6 +2245,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, VMA_CREATE_FLAG_IS_NULL : 0; flags |= op->map.dumpable ? VMA_CREATE_FLAG_DUMPABLE : 0; + flags |= op->map.is_cpu_addr_mirror ? + VMA_CREATE_FLAG_IS_SYSTEM_ALLOCATOR : 0; vma = new_vma(vm, &op->base.map, op->map.pat_index, flags); @@ -2245,7 +2254,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, return PTR_ERR(vma); op->map.vma = vma; - if (op->map.immediate || !xe_vm_in_fault_mode(vm)) + if ((op->map.immediate || !xe_vm_in_fault_mode(vm)) && + !op->map.is_cpu_addr_mirror) xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask); break; @@ -2254,21 +2264,24 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, { struct xe_vma *old = gpuva_to_vma(op->base.remap.unmap->va); + bool skip = xe_vma_is_cpu_addr_mirror(old); op->remap.start = xe_vma_start(old); op->remap.range = xe_vma_size(old); - if (op->base.remap.prev) { - flags |= op->base.remap.unmap->va->flags & - XE_VMA_READ_ONLY ? - VMA_CREATE_FLAG_READ_ONLY : 0; - flags |= op->base.remap.unmap->va->flags & - DRM_GPUVA_SPARSE ? - VMA_CREATE_FLAG_IS_NULL : 0; - flags |= op->base.remap.unmap->va->flags & - XE_VMA_DUMPABLE ? - VMA_CREATE_FLAG_DUMPABLE : 0; + flags |= op->base.remap.unmap->va->flags & + XE_VMA_READ_ONLY ? + VMA_CREATE_FLAG_READ_ONLY : 0; + flags |= op->base.remap.unmap->va->flags & + DRM_GPUVA_SPARSE ? + VMA_CREATE_FLAG_IS_NULL : 0; + flags |= op->base.remap.unmap->va->flags & + XE_VMA_DUMPABLE ? + VMA_CREATE_FLAG_DUMPABLE : 0; + flags |= xe_vma_is_cpu_addr_mirror(old) ? + VMA_CREATE_FLAG_IS_SYSTEM_ALLOCATOR : 0; + if (op->base.remap.prev) { vma = new_vma(vm, op->base.remap.prev, old->pat_index, flags); if (IS_ERR(vma)) @@ -2280,9 +2293,10 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, * Userptr creates a new SG mapping so * we must also rebind. */ - op->remap.skip_prev = !xe_vma_is_userptr(old) && + op->remap.skip_prev = skip || + (!xe_vma_is_userptr(old) && IS_ALIGNED(xe_vma_end(vma), - xe_vma_max_pte_size(old)); + xe_vma_max_pte_size(old))); if (op->remap.skip_prev) { xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); op->remap.range -= @@ -2298,16 +2312,6 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, } if (op->base.remap.next) { - flags |= op->base.remap.unmap->va->flags & - XE_VMA_READ_ONLY ? - VMA_CREATE_FLAG_READ_ONLY : 0; - flags |= op->base.remap.unmap->va->flags & - DRM_GPUVA_SPARSE ? - VMA_CREATE_FLAG_IS_NULL : 0; - flags |= op->base.remap.unmap->va->flags & - XE_VMA_DUMPABLE ? - VMA_CREATE_FLAG_DUMPABLE : 0; - vma = new_vma(vm, op->base.remap.next, old->pat_index, flags); if (IS_ERR(vma)) @@ -2319,9 +2323,10 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, * Userptr creates a new SG mapping so * we must also rebind. */ - op->remap.skip_next = !xe_vma_is_userptr(old) && + op->remap.skip_next = skip || + (!xe_vma_is_userptr(old) && IS_ALIGNED(xe_vma_start(vma), - xe_vma_max_pte_size(old)); + xe_vma_max_pte_size(old))); if (op->remap.skip_next) { xe_vma_set_pte_size(vma, xe_vma_max_pte_size(old)); op->remap.range -= @@ -2334,11 +2339,15 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask); } } - xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask); + if (!skip) + xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask); break; } case DRM_GPUVA_OP_UNMAP: - xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask); + vma = gpuva_to_vma(op->base.unmap.va); + + if (!xe_vma_is_cpu_addr_mirror(vma)) + xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask); break; case DRM_GPUVA_OP_PREFETCH: vma = gpuva_to_vma(op->base.prefetch.va); @@ -2349,7 +2358,8 @@ static int vm_bind_ioctl_ops_parse(struct xe_vm *vm, struct drm_gpuva_ops *ops, return err; } - xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask); + if (!xe_vma_is_cpu_addr_mirror(vma)) + xe_vma_ops_incr_pt_update_ops(vops, op->tile_mask); break; default: drm_warn(&vm->xe->drm, "NOT POSSIBLE"); @@ -2752,9 +2762,11 @@ static void vm_bind_ioctl_ops_fini(struct xe_vm *vm, struct xe_vma_ops *vops, } if (ufence) xe_sync_ufence_put(ufence); - for (i = 0; i < vops->num_syncs; i++) - xe_sync_entry_signal(vops->syncs + i, fence); - xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); + if (fence) { + for (i = 0; i < vops->num_syncs; i++) + xe_sync_entry_signal(vops->syncs + i, fence); + xe_exec_queue_last_fence_set(wait_exec_queue, vm, fence); + } } static struct dma_fence *vm_bind_ioctl_ops_execute(struct xe_vm *vm, @@ -2777,8 +2789,11 @@ static struct dma_fence *vm_bind_ioctl_ops_execute(struct xe_vm *vm, } fence = ops_execute(vm, vops); - if (IS_ERR(fence)) + if (IS_ERR(fence)) { + if (PTR_ERR(fence) == -ENODATA) + vm_bind_ioctl_ops_fini(vm, vops, NULL); goto unlock; + } vm_bind_ioctl_ops_fini(vm, vops, fence); } @@ -2794,7 +2809,8 @@ ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_execute, ERRNO); DRM_XE_VM_BIND_FLAG_IMMEDIATE | \ DRM_XE_VM_BIND_FLAG_NULL | \ DRM_XE_VM_BIND_FLAG_DUMPABLE | \ - DRM_XE_VM_BIND_FLAG_CHECK_PXP) + DRM_XE_VM_BIND_FLAG_CHECK_PXP | \ + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR) #ifdef TEST_VM_OPS_ERROR #define SUPPORTED_FLAGS (SUPPORTED_FLAGS_STUB | FORCE_OP_ERROR) @@ -2805,7 +2821,7 @@ ALLOW_ERROR_INJECTION(vm_bind_ioctl_ops_execute, ERRNO); #define XE_64K_PAGE_MASK 0xffffull #define ALL_DRM_XE_SYNCS_FLAGS (DRM_XE_SYNCS_FLAG_WAIT_FOR_OP) -static int vm_bind_ioctl_check_args(struct xe_device *xe, +static int vm_bind_ioctl_check_args(struct xe_device *xe, struct xe_vm *vm, struct drm_xe_vm_bind *args, struct drm_xe_vm_bind_op **bind_ops) { @@ -2850,9 +2866,23 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, u64 obj_offset = (*bind_ops)[i].obj_offset; u32 prefetch_region = (*bind_ops)[i].prefetch_mem_region_instance; bool is_null = flags & DRM_XE_VM_BIND_FLAG_NULL; + bool is_cpu_addr_mirror = flags & + DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR; u16 pat_index = (*bind_ops)[i].pat_index; u16 coh_mode; + /* FIXME: Disabling CPU address mirror for now */ + if (XE_IOCTL_DBG(xe, is_cpu_addr_mirror)) { + err = -EOPNOTSUPP; + goto free_bind_ops; + } + + if (XE_IOCTL_DBG(xe, is_cpu_addr_mirror && + !xe_vm_in_fault_mode(vm))) { + err = -EINVAL; + goto free_bind_ops; + } + if (XE_IOCTL_DBG(xe, pat_index >= xe->pat.n_entries)) { err = -EINVAL; goto free_bind_ops; @@ -2873,13 +2903,14 @@ static int vm_bind_ioctl_check_args(struct xe_device *xe, if (XE_IOCTL_DBG(xe, op > DRM_XE_VM_BIND_OP_PREFETCH) || XE_IOCTL_DBG(xe, flags & ~SUPPORTED_FLAGS) || - XE_IOCTL_DBG(xe, obj && is_null) || - XE_IOCTL_DBG(xe, obj_offset && is_null) || + XE_IOCTL_DBG(xe, obj && (is_null || is_cpu_addr_mirror)) || + XE_IOCTL_DBG(xe, obj_offset && (is_null || + is_cpu_addr_mirror)) || XE_IOCTL_DBG(xe, op != DRM_XE_VM_BIND_OP_MAP && - is_null) || + (is_null || is_cpu_addr_mirror)) || XE_IOCTL_DBG(xe, !obj && op == DRM_XE_VM_BIND_OP_MAP && - !is_null) || + !is_null && !is_cpu_addr_mirror) || XE_IOCTL_DBG(xe, !obj && op == DRM_XE_VM_BIND_OP_UNMAP_ALL) || XE_IOCTL_DBG(xe, addr && @@ -3028,15 +3059,19 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) int err; int i; - err = vm_bind_ioctl_check_args(xe, args, &bind_ops); + vm = xe_vm_lookup(xef, args->vm_id); + if (XE_IOCTL_DBG(xe, !vm)) + return -EINVAL; + + err = vm_bind_ioctl_check_args(xe, vm, args, &bind_ops); if (err) - return err; + goto put_vm; if (args->exec_queue_id) { q = xe_exec_queue_lookup(xef, args->exec_queue_id); if (XE_IOCTL_DBG(xe, !q)) { err = -ENOENT; - goto free_objs; + goto put_vm; } if (XE_IOCTL_DBG(xe, !(q->flags & EXEC_QUEUE_FLAG_VM))) { @@ -3045,15 +3080,9 @@ int xe_vm_bind_ioctl(struct drm_device *dev, void *data, struct drm_file *file) } } - vm = xe_vm_lookup(xef, args->vm_id); - if (XE_IOCTL_DBG(xe, !vm)) { - err = -EINVAL; - goto put_exec_queue; - } - err = down_write_killable(&vm->lock); if (err) - goto put_vm; + goto put_exec_queue; if (XE_IOCTL_DBG(xe, xe_vm_is_closed_or_banned(vm))) { err = -ENOENT; @@ -3217,12 +3246,11 @@ put_obj: xe_bo_put(bos[i]); release_vm_lock: up_write(&vm->lock); -put_vm: - xe_vm_put(vm); put_exec_queue: if (q) xe_exec_queue_put(q); -free_objs: +put_vm: + xe_vm_put(vm); kvfree(bos); kvfree(ops); if (args->num_binds > 1) @@ -3354,6 +3382,7 @@ int xe_vm_invalidate_vma(struct xe_vma *vma) int ret = 0; xe_assert(xe, !xe_vma_is_null(vma)); + xe_assert(xe, !xe_vma_is_cpu_addr_mirror(vma)); trace_xe_vma_invalidate(vma); vm_dbg(&xe_vma_vm(vma)->xe->drm, diff --git a/drivers/gpu/drm/xe/xe_vm.h b/drivers/gpu/drm/xe/xe_vm.h index f5d835271350..2148303a9035 100644 --- a/drivers/gpu/drm/xe/xe_vm.h +++ b/drivers/gpu/drm/xe/xe_vm.h @@ -152,6 +152,11 @@ static inline bool xe_vma_is_null(struct xe_vma *vma) return vma->gpuva.flags & DRM_GPUVA_SPARSE; } +static inline bool xe_vma_is_cpu_addr_mirror(struct xe_vma *vma) +{ + return vma->gpuva.flags & XE_VMA_SYSTEM_ALLOCATOR; +} + static inline bool xe_vma_has_no_bo(struct xe_vma *vma) { return !xe_vma_bo(vma); @@ -159,7 +164,8 @@ static inline bool xe_vma_has_no_bo(struct xe_vma *vma) static inline bool xe_vma_is_userptr(struct xe_vma *vma) { - return xe_vma_has_no_bo(vma) && !xe_vma_is_null(vma); + return xe_vma_has_no_bo(vma) && !xe_vma_is_null(vma) && + !xe_vma_is_cpu_addr_mirror(vma); } /** diff --git a/drivers/gpu/drm/xe/xe_vm_types.h b/drivers/gpu/drm/xe/xe_vm_types.h index eca73c4197d4..db7107e784c8 100644 --- a/drivers/gpu/drm/xe/xe_vm_types.h +++ b/drivers/gpu/drm/xe/xe_vm_types.h @@ -42,6 +42,7 @@ struct xe_vm_pgtable_update_op; #define XE_VMA_PTE_64K (DRM_GPUVA_USERBITS << 6) #define XE_VMA_PTE_COMPACT (DRM_GPUVA_USERBITS << 7) #define XE_VMA_DUMPABLE (DRM_GPUVA_USERBITS << 8) +#define XE_VMA_SYSTEM_ALLOCATOR (DRM_GPUVA_USERBITS << 9) /** struct xe_userptr - User pointer */ struct xe_userptr { @@ -299,6 +300,8 @@ struct xe_vma_op_map { bool read_only; /** @is_null: is NULL binding */ bool is_null; + /** @is_cpu_addr_mirror: is CPU address mirror binding */ + bool is_cpu_addr_mirror; /** @dumpable: whether BO is dumped on GPU hang */ bool dumpable; /** @pat_index: The pat index to use for this operation. */ diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index d1f0018342b6..acf92a367e3d 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -990,6 +990,12 @@ struct drm_xe_vm_destroy { * - %DRM_XE_VM_BIND_FLAG_CHECK_PXP - If the object is encrypted via PXP, * reject the binding if the encryption key is no longer valid. This * flag has no effect on BOs that are not marked as using PXP. + * - %DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR - When the CPU address mirror flag is + * set, no mappings are created rather the range is reserved for CPU address + * mirroring which will be populated on GPU page faults or prefetches. Only + * valid on VMs with DRM_XE_VM_CREATE_FLAG_FAULT_MODE set. The CPU address + * mirror flag are only valid for DRM_XE_VM_BIND_OP_MAP operations, the BO + * handle MBZ, and the BO offset MBZ. */ struct drm_xe_vm_bind_op { /** @extensions: Pointer to the first extension struct, if any */ @@ -1042,7 +1048,9 @@ struct drm_xe_vm_bind_op { * on the @pat_index. For such mappings there is no actual memory being * mapped (the address in the PTE is invalid), so the various PAT memory * attributes likely do not apply. Simply leaving as zero is one - * option (still a valid pat_index). + * option (still a valid pat_index). Same applies to + * DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR bindings as for such mapping + * there is no actual memory being mapped. */ __u16 pat_index; @@ -1058,6 +1066,14 @@ struct drm_xe_vm_bind_op { /** @userptr: user pointer to bind on */ __u64 userptr; + + /** + * @cpu_addr_mirror_offset: Offset from GPU @addr to create + * CPU address mirror mappings. MBZ with current level of + * support (e.g. 1 to 1 mapping between GPU and CPU mappings + * only supported). + */ + __s64 cpu_addr_mirror_offset; }; /** @@ -1081,6 +1097,7 @@ struct drm_xe_vm_bind_op { #define DRM_XE_VM_BIND_FLAG_NULL (1 << 2) #define DRM_XE_VM_BIND_FLAG_DUMPABLE (1 << 3) #define DRM_XE_VM_BIND_FLAG_CHECK_PXP (1 << 4) +#define DRM_XE_VM_BIND_FLAG_CPU_ADDR_MIRROR (1 << 5) /** @flags: Bind flags */ __u32 flags; -- cgit v1.3 From 77613a2e10087b1e613649ecb337c4922900421c Mon Sep 17 00:00:00 2001 From: Matthew Brost Date: Wed, 5 Mar 2025 17:26:44 -0800 Subject: drm/xe/uapi: Add DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR device query flag, which indicates whether the device supports CPU address mirroring. The intent is for UMDs to use this query to determine if a VM can be set up with CPU address mirroring. This flag is implemented by checking if the device supports GPU faults. v7: - Only report enabled if CONFIG_DRM_GPUSVM is selected (CI) Signed-off-by: Matthew Brost Reviewed-by: Himal Prasad Ghimiray Reviewed-by: Thomas Hellström Reviewed-by: Tejas Upadhyay Link: https://patchwork.freedesktop.org/patch/msgid/20250306012657.3505757-20-matthew.brost@intel.com --- drivers/gpu/drm/xe/xe_query.c | 5 ++++- include/uapi/drm/xe_drm.h | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) (limited to 'include/uapi/drm') diff --git a/drivers/gpu/drm/xe/xe_query.c b/drivers/gpu/drm/xe/xe_query.c index ce2a2767de1a..5e65830dad25 100644 --- a/drivers/gpu/drm/xe/xe_query.c +++ b/drivers/gpu/drm/xe/xe_query.c @@ -338,8 +338,11 @@ static int query_config(struct xe_device *xe, struct drm_xe_device_query *query) config->info[DRM_XE_QUERY_CONFIG_REV_AND_DEVICE_ID] = xe->info.devid | (xe->info.revid << 16); if (xe_device_get_root_tile(xe)->mem.vram.usable_size) - config->info[DRM_XE_QUERY_CONFIG_FLAGS] = + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM; + if (xe->info.has_usm && IS_ENABLED(CONFIG_DRM_GPUSVM)) + config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= + DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR; config->info[DRM_XE_QUERY_CONFIG_FLAGS] |= DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY; config->info[DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT] = diff --git a/include/uapi/drm/xe_drm.h b/include/uapi/drm/xe_drm.h index acf92a367e3d..616916985e3f 100644 --- a/include/uapi/drm/xe_drm.h +++ b/include/uapi/drm/xe_drm.h @@ -395,6 +395,8 @@ struct drm_xe_query_mem_regions { * has usable VRAM * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY - Flag is set if the device * has low latency hint support + * - %DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR - Flag is set if the + * device has CPU address mirroring support * - %DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT - Minimal memory alignment * required by this device, typically SZ_4K or SZ_64K * - %DRM_XE_QUERY_CONFIG_VA_BITS - Maximum bits of a virtual address @@ -412,6 +414,7 @@ struct drm_xe_query_config { #define DRM_XE_QUERY_CONFIG_FLAGS 1 #define DRM_XE_QUERY_CONFIG_FLAG_HAS_VRAM (1 << 0) #define DRM_XE_QUERY_CONFIG_FLAG_HAS_LOW_LATENCY (1 << 1) + #define DRM_XE_QUERY_CONFIG_FLAG_HAS_CPU_ADDR_MIRROR (1 << 2) #define DRM_XE_QUERY_CONFIG_MIN_ALIGNMENT 2 #define DRM_XE_QUERY_CONFIG_VA_BITS 3 #define DRM_XE_QUERY_CONFIG_MAX_EXEC_QUEUE_PRIORITY 4 -- cgit v1.3