/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (C) 2021 Intel Corporation * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES */ #ifndef __LINUX_IOMMUFD_H #define __LINUX_IOMMUFD_H #include #include #include #include #include #include #include struct device; struct file; struct iommu_group; struct iommu_user_data; struct iommu_user_data_array; struct iommufd_access; struct iommufd_ctx; struct iommufd_device; struct iommufd_viommu_ops; struct page; enum iommufd_object_type { IOMMUFD_OBJ_NONE, IOMMUFD_OBJ_ANY = IOMMUFD_OBJ_NONE, IOMMUFD_OBJ_DEVICE, IOMMUFD_OBJ_HWPT_PAGING, IOMMUFD_OBJ_HWPT_NESTED, IOMMUFD_OBJ_IOAS, IOMMUFD_OBJ_ACCESS, IOMMUFD_OBJ_FAULT, IOMMUFD_OBJ_VIOMMU, IOMMUFD_OBJ_VDEVICE, IOMMUFD_OBJ_VEVENTQ, IOMMUFD_OBJ_HW_QUEUE, #ifdef CONFIG_IOMMUFD_TEST IOMMUFD_OBJ_SELFTEST, #endif IOMMUFD_OBJ_MAX, }; /* Base struct for all objects with a userspace ID handle. */ struct iommufd_object { /* * Destroy will sleep and wait for wait_cnt to go to zero. This allows * concurrent users of the ID to reliably avoid causing a spurious * destroy failure. Incrementing this count should either be short * lived or be revoked and blocked during pre_destroy(). */ refcount_t wait_cnt; refcount_t users; enum iommufd_object_type type; unsigned int id; }; struct iommufd_device *iommufd_device_bind(struct iommufd_ctx *ictx, struct device *dev, u32 *id); void iommufd_device_unbind(struct iommufd_device *idev); int iommufd_device_attach(struct iommufd_device *idev, ioasid_t pasid, u32 *pt_id); int iommufd_device_replace(struct iommufd_device *idev, ioasid_t pasid, u32 *pt_id); void iommufd_device_detach(struct iommufd_device *idev, ioasid_t pasid); struct iommufd_ctx *iommufd_device_to_ictx(struct iommufd_device *idev); u32 iommufd_device_to_id(struct iommufd_device *idev); struct iommufd_access_ops { u8 needs_pin_pages : 1; void (*unmap)(void *data, unsigned long iova, unsigned long length); }; enum { IOMMUFD_ACCESS_RW_READ = 0, IOMMUFD_ACCESS_RW_WRITE = 1 << 0, /* Set if the caller is in a kthread then rw will use kthread_use_mm() */ IOMMUFD_ACCESS_RW_KTHREAD = 1 << 1, /* Only for use by selftest */ __IOMMUFD_ACCESS_RW_SLOW_PATH = 1 << 2, }; struct iommufd_access * iommufd_access_create(struct iommufd_ctx *ictx, const struct iommufd_access_ops *ops, void *data, u32 *id); void iommufd_access_destroy(struct iommufd_access *access); int iommufd_access_attach(struct iommufd_access *access, u32 ioas_id); int iommufd_access_replace(struct iommufd_access *access, u32 ioas_id); void iommufd_access_detach(struct iommufd_access *access); void iommufd_ctx_get(struct iommufd_ctx *ictx); struct iommufd_viommu { struct iommufd_object obj; struct iommufd_ctx *ictx; struct iommu_device *iommu_dev; struct iommufd_hwpt_paging *hwpt; const struct iommufd_viommu_ops *ops; struct xarray vdevs; struct list_head veventqs; struct rw_semaphore veventqs_rwsem; enum iommu_viommu_type type; }; struct iommufd_vdevice { struct iommufd_object obj; struct iommufd_viommu *viommu; struct iommufd_device *idev; /* * Virtual device ID per vIOMMU, e.g. vSID of ARM SMMUv3, vDeviceID of * AMD IOMMU, and vRID of Intel VT-d */ u64 virt_id; /* Clean up all driver-specific parts of an iommufd_vdevice */ void (*destroy)(struct iommufd_vdevice *vdev); }; struct iommufd_hw_queue { struct iommufd_object obj; struct iommufd_viommu *viommu; struct iommufd_access *access; u64 base_addr; /* in guest physical address space */ size_t length; enum iommu_hw_queue_type type; /* Clean up all driver-specific parts of an iommufd_hw_queue */ void (*destroy)(struct iommufd_hw_queue *hw_queue); }; /** * struct iommufd_viommu_ops - vIOMMU specific operations * @destroy: Clean up all driver-specific parts of an iommufd_viommu. The memory * of the vIOMMU will be free-ed by iommufd core after calling this op * @alloc_domain_nested: Allocate a IOMMU_DOMAIN_NESTED on a vIOMMU that holds a * nesting parent domain (IOMMU_DOMAIN_PAGING). @user_data * must be defined in include/uapi/linux/iommufd.h. * It must fully initialize the new iommu_domain before * returning. Upon failure, ERR_PTR must be returned. * @cache_invalidate: Flush hardware cache used by a vIOMMU. It can be used for * any IOMMU hardware specific cache: TLB and device cache. * The @array passes in the cache invalidation requests, in * form of a driver data structure. A driver must update the * array->entry_num to report the number of handled requests. * The data structure of the array entry must be defined in * include/uapi/linux/iommufd.h * @vdevice_size: Size of the driver-defined vDEVICE structure per this vIOMMU * @vdevice_init: Initialize the driver-level structure of a vDEVICE object, or * related HW procedure. @vdev is already initialized by iommufd * core: vdev->dev and vdev->viommu pointers; vdev->id carries a * per-vIOMMU virtual ID (refer to struct iommu_vdevice_alloc in * include/uapi/linux/iommufd.h) * If driver has a deinit function to revert what vdevice_init op * does, it should set it to the @vdev->destroy function pointer * @get_hw_queue_size: Get the size of a driver-defined HW queue structure for a * given @viommu corresponding to @queue_type. Driver should * return 0 if HW queue aren't supported accordingly. It is * required for driver to use the HW_QUEUE_STRUCT_SIZE macro * to sanitize the driver-level HW queue structure related * to the core one * @hw_queue_init_phys: Initialize the driver-level structure of a HW queue that * is initialized with its core-level structure that holds * all the info about a guest queue memory. * Driver providing this op indicates that HW accesses the * guest queue memory via physical addresses. * @index carries the logical HW QUEUE ID per vIOMMU in a * guest VM, for a multi-queue model. @base_addr_pa carries * the physical location of the guest queue * If driver has a deinit function to revert what this op * does, it should set it to the @hw_queue->destroy pointer */ struct iommufd_viommu_ops { void (*destroy)(struct iommufd_viommu *viommu); struct iommu_domain *(*alloc_domain_nested)( struct iommufd_viommu *viommu, u32 flags, const struct iommu_user_data *user_data); int (*cache_invalidate)(struct iommufd_viommu *viommu, struct iommu_user_data_array *array); const size_t vdevice_size; int (*vdevice_init)(struct iommufd_vdevice *vdev); size_t (*get_hw_queue_size)(struct iommufd_viommu *viommu, enum iommu_hw_queue_type queue_type); /* AMD's HW will add hw_queue_init simply using @hw_queue->base_addr */ int (*hw_queue_init_phys)(struct iommufd_hw_queue *hw_queue, u32 index, phys_addr_t base_addr_pa); }; #if IS_ENABLED(CONFIG_IOMMUFD) struct iommufd_ctx *iommufd_ctx_from_file(struct file *file); struct iommufd_ctx *iommufd_ctx_from_fd(int fd); void iommufd_ctx_put(struct iommufd_ctx *ictx); bool iommufd_ctx_has_group(struct iommufd_ctx *ictx, struct iommu_group *group); int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length, struct page **out_pages, unsigned int flags); void iommufd_access_unpin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length); int iommufd_access_rw(struct iommufd_access *access, unsigned long iova, void *data, size_t len, unsigned int flags); int iommufd_vfio_compat_ioas_get_id(struct iommufd_ctx *ictx, u32 *out_ioas_id); int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx); int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx); #else /* !CONFIG_IOMMUFD */ static inline struct iommufd_ctx *iommufd_ctx_from_file(struct file *file) { return ERR_PTR(-EOPNOTSUPP); } static inline void iommufd_ctx_put(struct iommufd_ctx *ictx) { } static inline int iommufd_access_pin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length, struct page **out_pages, unsigned int flags) { return -EOPNOTSUPP; } static inline void iommufd_access_unpin_pages(struct iommufd_access *access, unsigned long iova, unsigned long length) { } static inline int iommufd_access_rw(struct iommufd_access *access, unsigned long iova, void *data, size_t len, unsigned int flags) { return -EOPNOTSUPP; } static inline int iommufd_vfio_compat_ioas_create(struct iommufd_ctx *ictx) { return -EOPNOTSUPP; } static inline int iommufd_vfio_compat_set_no_iommu(struct iommufd_ctx *ictx) { return -EOPNOTSUPP; } #endif /* CONFIG_IOMMUFD */ #if IS_ENABLED(CONFIG_IOMMUFD_DRIVER_CORE) int _iommufd_object_depend(struct iommufd_object *obj_dependent, struct iommufd_object *obj_depended); void _iommufd_object_undepend(struct iommufd_object *obj_dependent, struct iommufd_object *obj_depended); int _iommufd_alloc_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner, phys_addr_t mmio_addr, size_t length, unsigned long *offset); void _iommufd_destroy_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner, unsigned long offset); struct device *iommufd_vdevice_to_device(struct iommufd_vdevice *vdev); struct device *iommufd_viommu_find_dev(struct iommufd_viommu *viommu, unsigned long vdev_id); int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu, struct device *dev, unsigned long *vdev_id); int iommufd_viommu_report_event(struct iommufd_viommu *viommu, enum iommu_veventq_type type, void *event_data, size_t data_len); #else /* !CONFIG_IOMMUFD_DRIVER_CORE */ static inline int _iommufd_object_depend(struct iommufd_object *obj_dependent, struct iommufd_object *obj_depended) { return -EOPNOTSUPP; } static inline void _iommufd_object_undepend(struct iommufd_object *obj_dependent, struct iommufd_object *obj_depended) { } static inline int _iommufd_alloc_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner, phys_addr_t mmio_addr, size_t length, unsigned long *offset) { return -EOPNOTSUPP; } static inline void _iommufd_destroy_mmap(struct iommufd_ctx *ictx, struct iommufd_object *owner, unsigned long offset) { } static inline struct device * iommufd_vdevice_to_device(struct iommufd_vdevice *vdev) { return NULL; } static inline struct device * iommufd_viommu_find_dev(struct iommufd_viommu *viommu, unsigned long vdev_id) { return NULL; } static inline int iommufd_viommu_get_vdev_id(struct iommufd_viommu *viommu, struct device *dev, unsigned long *vdev_id) { return -ENOENT; } static inline int iommufd_viommu_report_event(struct iommufd_viommu *viommu, enum iommu_veventq_type type, void *event_data, size_t data_len) { return -EOPNOTSUPP; } #endif /* CONFIG_IOMMUFD_DRIVER_CORE */ #define VIOMMU_STRUCT_SIZE(drv_struct, member) \ (sizeof(drv_struct) + \ BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \ BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_viommu, \ ((drv_struct *)NULL)->member))) #define VDEVICE_STRUCT_SIZE(drv_struct, member) \ (sizeof(drv_struct) + \ BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \ BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_vdevice, \ ((drv_struct *)NULL)->member))) #define HW_QUEUE_STRUCT_SIZE(drv_struct, member) \ (sizeof(drv_struct) + \ BUILD_BUG_ON_ZERO(offsetof(drv_struct, member)) + \ BUILD_BUG_ON_ZERO(!__same_type(struct iommufd_hw_queue, \ ((drv_struct *)NULL)->member))) /* * Helpers for IOMMU driver to build/destroy a dependency between two sibling * structures created by one of the allocators above */ #define iommufd_hw_queue_depend(dependent, depended, member) \ ({ \ int ret = -EINVAL; \ \ static_assert(__same_type(struct iommufd_hw_queue, \ dependent->member)); \ static_assert(__same_type(typeof(*dependent), *depended)); \ if (!WARN_ON_ONCE(dependent->member.viommu != \ depended->member.viommu)) \ ret = _iommufd_object_depend(&dependent->member.obj, \ &depended->member.obj); \ ret; \ }) #define iommufd_hw_queue_undepend(dependent, depended, member) \ ({ \ static_assert(__same_type(struct iommufd_hw_queue, \ dependent->member)); \ static_assert(__same_type(typeof(*dependent), *depended)); \ WARN_ON_ONCE(dependent->member.viommu != \ depended->member.viommu); \ _iommufd_object_undepend(&dependent->member.obj, \ &depended->member.obj); \ }) /* * Helpers for IOMMU driver to alloc/destroy an mmapable area for a structure. * * To support an mmappable MMIO region, kernel driver must first register it to * iommufd core to allocate an @offset, during a driver-structure initialization * (e.g. viommu_init op). Then, it should report to user space this @offset and * the @length of the MMIO region for mmap syscall. */ static inline int iommufd_viommu_alloc_mmap(struct iommufd_viommu *viommu, phys_addr_t mmio_addr, size_t length, unsigned long *offset) { return _iommufd_alloc_mmap(viommu->ictx, &viommu->obj, mmio_addr, length, offset); } static inline void iommufd_viommu_destroy_mmap(struct iommufd_viommu *viommu, unsigned long offset) { _iommufd_destroy_mmap(viommu->ictx, &viommu->obj, offset); } #endif