diff options
Diffstat (limited to 'drivers/gpu/drm/panfrost')
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_devfreq.c | 6 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_device.c | 68 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_device.h | 24 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_drv.c | 243 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_dump.c | 8 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_gem.c | 9 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c | 4 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_gpu.c | 66 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_job.c | 336 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_job.h | 38 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_mmu.c | 115 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_mmu.h | 3 | ||||
| -rw-r--r-- | drivers/gpu/drm/panfrost/panfrost_perfcnt.c | 26 |
13 files changed, 658 insertions, 288 deletions
diff --git a/drivers/gpu/drm/panfrost/panfrost_devfreq.c b/drivers/gpu/drm/panfrost/panfrost_devfreq.c index 5d0dce10336b..b51c30778811 100644 --- a/drivers/gpu/drm/panfrost/panfrost_devfreq.c +++ b/drivers/gpu/drm/panfrost/panfrost_devfreq.c @@ -8,6 +8,8 @@ #include <linux/platform_device.h> #include <linux/pm_opp.h> +#include <drm/drm_print.h> + #include "panfrost_device.h" #include "panfrost_devfreq.h" @@ -74,7 +76,7 @@ static int panfrost_devfreq_get_dev_status(struct device *dev, spin_unlock_irqrestore(&pfdevfreq->lock, irqflags); - dev_dbg(pfdev->dev, "busy %lu total %lu %lu %% freq %lu MHz\n", + dev_dbg(pfdev->base.dev, "busy %lu total %lu %lu %% freq %lu MHz\n", status->busy_time, status->total_time, status->busy_time / (status->total_time / 100), status->current_frequency / 1000 / 1000); @@ -119,7 +121,7 @@ int panfrost_devfreq_init(struct panfrost_device *pfdev) int ret; struct dev_pm_opp *opp; unsigned long cur_freq; - struct device *dev = &pfdev->pdev->dev; + struct device *dev = pfdev->base.dev; struct devfreq *devfreq; struct thermal_cooling_device *cooling; struct panfrost_devfreq *pfdevfreq = &pfdev->pfdevfreq; diff --git a/drivers/gpu/drm/panfrost/panfrost_device.c b/drivers/gpu/drm/panfrost/panfrost_device.c index 04bec27449cb..c61b97af120c 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.c +++ b/drivers/gpu/drm/panfrost/panfrost_device.c @@ -20,9 +20,9 @@ static int panfrost_reset_init(struct panfrost_device *pfdev) { - pfdev->rstc = devm_reset_control_array_get_optional_exclusive(pfdev->dev); + pfdev->rstc = devm_reset_control_array_get_optional_exclusive(pfdev->base.dev); if (IS_ERR(pfdev->rstc)) { - dev_err(pfdev->dev, "get reset failed %ld\n", PTR_ERR(pfdev->rstc)); + dev_err(pfdev->base.dev, "get reset failed %ld\n", PTR_ERR(pfdev->rstc)); return PTR_ERR(pfdev->rstc); } @@ -39,22 +39,22 @@ static int panfrost_clk_init(struct panfrost_device *pfdev) int err; unsigned long rate; - pfdev->clock = devm_clk_get(pfdev->dev, NULL); + pfdev->clock = devm_clk_get(pfdev->base.dev, NULL); if (IS_ERR(pfdev->clock)) { - dev_err(pfdev->dev, "get clock failed %ld\n", PTR_ERR(pfdev->clock)); + dev_err(pfdev->base.dev, "get clock failed %ld\n", PTR_ERR(pfdev->clock)); return PTR_ERR(pfdev->clock); } rate = clk_get_rate(pfdev->clock); - dev_info(pfdev->dev, "clock rate = %lu\n", rate); + dev_info(pfdev->base.dev, "clock rate = %lu\n", rate); err = clk_prepare_enable(pfdev->clock); if (err) return err; - pfdev->bus_clock = devm_clk_get_optional(pfdev->dev, "bus"); + pfdev->bus_clock = devm_clk_get_optional(pfdev->base.dev, "bus"); if (IS_ERR(pfdev->bus_clock)) { - dev_err(pfdev->dev, "get bus_clock failed %ld\n", + dev_err(pfdev->base.dev, "get bus_clock failed %ld\n", PTR_ERR(pfdev->bus_clock)); err = PTR_ERR(pfdev->bus_clock); goto disable_clock; @@ -62,7 +62,7 @@ static int panfrost_clk_init(struct panfrost_device *pfdev) if (pfdev->bus_clock) { rate = clk_get_rate(pfdev->bus_clock); - dev_info(pfdev->dev, "bus_clock rate = %lu\n", rate); + dev_info(pfdev->base.dev, "bus_clock rate = %lu\n", rate); err = clk_prepare_enable(pfdev->bus_clock); if (err) @@ -87,7 +87,7 @@ static int panfrost_regulator_init(struct panfrost_device *pfdev) { int ret, i; - pfdev->regulators = devm_kcalloc(pfdev->dev, pfdev->comp->num_supplies, + pfdev->regulators = devm_kcalloc(pfdev->base.dev, pfdev->comp->num_supplies, sizeof(*pfdev->regulators), GFP_KERNEL); if (!pfdev->regulators) @@ -96,12 +96,12 @@ static int panfrost_regulator_init(struct panfrost_device *pfdev) for (i = 0; i < pfdev->comp->num_supplies; i++) pfdev->regulators[i].supply = pfdev->comp->supply_names[i]; - ret = devm_regulator_bulk_get(pfdev->dev, + ret = devm_regulator_bulk_get(pfdev->base.dev, pfdev->comp->num_supplies, pfdev->regulators); if (ret < 0) { if (ret != -EPROBE_DEFER) - dev_err(pfdev->dev, "failed to get regulators: %d\n", + dev_err(pfdev->base.dev, "failed to get regulators: %d\n", ret); return ret; } @@ -109,7 +109,7 @@ static int panfrost_regulator_init(struct panfrost_device *pfdev) ret = regulator_bulk_enable(pfdev->comp->num_supplies, pfdev->regulators); if (ret < 0) { - dev_err(pfdev->dev, "failed to enable regulators: %d\n", ret); + dev_err(pfdev->base.dev, "failed to enable regulators: %d\n", ret); return ret; } @@ -144,7 +144,7 @@ static int panfrost_pm_domain_init(struct panfrost_device *pfdev) int err; int i, num_domains; - num_domains = of_count_phandle_with_args(pfdev->dev->of_node, + num_domains = of_count_phandle_with_args(pfdev->base.dev->of_node, "power-domains", "#power-domain-cells"); @@ -156,7 +156,7 @@ static int panfrost_pm_domain_init(struct panfrost_device *pfdev) return 0; if (num_domains != pfdev->comp->num_pm_domains) { - dev_err(pfdev->dev, + dev_err(pfdev->base.dev, "Incorrect number of power domains: %d provided, %d needed\n", num_domains, pfdev->comp->num_pm_domains); return -EINVAL; @@ -168,20 +168,21 @@ static int panfrost_pm_domain_init(struct panfrost_device *pfdev) for (i = 0; i < num_domains; i++) { pfdev->pm_domain_devs[i] = - dev_pm_domain_attach_by_name(pfdev->dev, - pfdev->comp->pm_domain_names[i]); + dev_pm_domain_attach_by_name(pfdev->base.dev, + pfdev->comp->pm_domain_names[i]); if (IS_ERR_OR_NULL(pfdev->pm_domain_devs[i])) { err = PTR_ERR(pfdev->pm_domain_devs[i]) ? : -ENODATA; pfdev->pm_domain_devs[i] = NULL; - dev_err(pfdev->dev, + dev_err(pfdev->base.dev, "failed to get pm-domain %s(%d): %d\n", pfdev->comp->pm_domain_names[i], i, err); goto err; } - pfdev->pm_domain_links[i] = device_link_add(pfdev->dev, - pfdev->pm_domain_devs[i], DL_FLAG_PM_RUNTIME | - DL_FLAG_STATELESS | DL_FLAG_RPM_ACTIVE); + pfdev->pm_domain_links[i] = + device_link_add(pfdev->base.dev, + pfdev->pm_domain_devs[i], DL_FLAG_PM_RUNTIME | + DL_FLAG_STATELESS | DL_FLAG_RPM_ACTIVE); if (!pfdev->pm_domain_links[i]) { dev_err(pfdev->pm_domain_devs[i], "adding device link failed!\n"); @@ -220,20 +221,20 @@ int panfrost_device_init(struct panfrost_device *pfdev) err = panfrost_reset_init(pfdev); if (err) { - dev_err(pfdev->dev, "reset init failed %d\n", err); + dev_err(pfdev->base.dev, "reset init failed %d\n", err); goto out_pm_domain; } err = panfrost_clk_init(pfdev); if (err) { - dev_err(pfdev->dev, "clk init failed %d\n", err); + dev_err(pfdev->base.dev, "clk init failed %d\n", err); goto out_reset; } err = panfrost_devfreq_init(pfdev); if (err) { if (err != -EPROBE_DEFER) - dev_err(pfdev->dev, "devfreq init failed %d\n", err); + dev_err(pfdev->base.dev, "devfreq init failed %d\n", err); goto out_clk; } @@ -244,7 +245,7 @@ int panfrost_device_init(struct panfrost_device *pfdev) goto out_devfreq; } - pfdev->iomem = devm_platform_ioremap_resource(pfdev->pdev, 0); + pfdev->iomem = devm_platform_ioremap_resource(to_platform_device(pfdev->base.dev), 0); if (IS_ERR(pfdev->iomem)) { err = PTR_ERR(pfdev->iomem); goto out_regulator; @@ -258,7 +259,7 @@ int panfrost_device_init(struct panfrost_device *pfdev) if (err) goto out_gpu; - err = panfrost_job_init(pfdev); + err = panfrost_jm_init(pfdev); if (err) goto out_mmu; @@ -268,7 +269,7 @@ int panfrost_device_init(struct panfrost_device *pfdev) return 0; out_job: - panfrost_job_fini(pfdev); + panfrost_jm_fini(pfdev); out_mmu: panfrost_mmu_fini(pfdev); out_gpu: @@ -289,7 +290,7 @@ out_pm_domain: void panfrost_device_fini(struct panfrost_device *pfdev) { panfrost_perfcnt_fini(pfdev); - panfrost_job_fini(pfdev); + panfrost_jm_fini(pfdev); panfrost_mmu_fini(pfdev); panfrost_gpu_fini(pfdev); panfrost_devfreq_fini(pfdev); @@ -399,13 +400,16 @@ bool panfrost_exception_needs_reset(const struct panfrost_device *pfdev, return false; } -void panfrost_device_reset(struct panfrost_device *pfdev) +void panfrost_device_reset(struct panfrost_device *pfdev, bool enable_job_int) { panfrost_gpu_soft_reset(pfdev); panfrost_gpu_power_on(pfdev); panfrost_mmu_reset(pfdev); - panfrost_job_enable_interrupts(pfdev); + + panfrost_jm_reset_interrupts(pfdev); + if (enable_job_int) + panfrost_jm_enable_interrupts(pfdev); } static int panfrost_device_runtime_resume(struct device *dev) @@ -429,7 +433,7 @@ static int panfrost_device_runtime_resume(struct device *dev) } } - panfrost_device_reset(pfdev); + panfrost_device_reset(pfdev, true); panfrost_devfreq_resume(pfdev); return 0; @@ -447,11 +451,11 @@ static int panfrost_device_runtime_suspend(struct device *dev) { struct panfrost_device *pfdev = dev_get_drvdata(dev); - if (!panfrost_job_is_idle(pfdev)) + if (!panfrost_jm_is_idle(pfdev)) return -EBUSY; panfrost_devfreq_suspend(pfdev); - panfrost_job_suspend_irq(pfdev); + panfrost_jm_suspend_irq(pfdev); panfrost_mmu_suspend_irq(pfdev); panfrost_gpu_suspend_irq(pfdev); panfrost_gpu_power_off(pfdev); diff --git a/drivers/gpu/drm/panfrost/panfrost_device.h b/drivers/gpu/drm/panfrost/panfrost_device.h index 077525a3ad68..e61c4329fd07 100644 --- a/drivers/gpu/drm/panfrost/panfrost_device.h +++ b/drivers/gpu/drm/panfrost/panfrost_device.h @@ -10,11 +10,13 @@ #include <linux/pm.h> #include <linux/regulator/consumer.h> #include <linux/spinlock.h> +#include <drm/drm_auth.h> #include <drm/drm_device.h> #include <drm/drm_mm.h> #include <drm/gpu_scheduler.h> #include "panfrost_devfreq.h" +#include "panfrost_job.h" struct panfrost_device; struct panfrost_mmu; @@ -22,9 +24,12 @@ struct panfrost_job_slot; struct panfrost_job; struct panfrost_perfcnt; -#define NUM_JOB_SLOTS 3 #define MAX_PM_DOMAINS 5 +#define ALL_JS_INT_MASK \ + (GENMASK(16 + NUM_JOB_SLOTS - 1, 16) | \ + GENMASK(NUM_JOB_SLOTS - 1, 0)) + enum panfrost_drv_comp_bits { PANFROST_COMP_BIT_GPU, PANFROST_COMP_BIT_JOB, @@ -123,9 +128,7 @@ struct panfrost_device_debugfs { }; struct panfrost_device { - struct device *dev; - struct drm_device *ddev; - struct platform_device *pdev; + struct drm_device base; int gpu_irq; int mmu_irq; @@ -144,7 +147,6 @@ struct panfrost_device { DECLARE_BITMAP(is_suspended, PANFROST_COMP_BIT_MAX); spinlock_t as_lock; - unsigned long as_in_use_mask; unsigned long as_alloc_mask; unsigned long as_faulty_mask; struct list_head as_lru_list; @@ -206,16 +208,22 @@ struct panfrost_engine_usage { struct panfrost_file_priv { struct panfrost_device *pfdev; - struct drm_sched_entity sched_entity[NUM_JOB_SLOTS]; + struct xarray jm_ctxs; struct panfrost_mmu *mmu; struct panfrost_engine_usage engine_usage; }; +static inline bool panfrost_high_prio_allowed(struct drm_file *file) +{ + /* Higher priorities require CAP_SYS_NICE or DRM_MASTER */ + return (capable(CAP_SYS_NICE) || drm_is_current_master(file)); +} + static inline struct panfrost_device *to_panfrost_device(struct drm_device *ddev) { - return ddev->dev_private; + return container_of(ddev, struct panfrost_device, base); } static inline int panfrost_model_cmp(struct panfrost_device *pfdev, s32 id) @@ -241,7 +249,7 @@ int panfrost_unstable_ioctl_check(void); int panfrost_device_init(struct panfrost_device *pfdev); void panfrost_device_fini(struct panfrost_device *pfdev); -void panfrost_device_reset(struct panfrost_device *pfdev); +void panfrost_device_reset(struct panfrost_device *pfdev, bool enable_job_int); extern const struct dev_pm_ops panfrost_pm_ops; diff --git a/drivers/gpu/drm/panfrost/panfrost_drv.c b/drivers/gpu/drm/panfrost/panfrost_drv.c index 1ea6c509a5d5..7d8c7c337606 100644 --- a/drivers/gpu/drm/panfrost/panfrost_drv.c +++ b/drivers/gpu/drm/panfrost/panfrost_drv.c @@ -16,6 +16,7 @@ #include <drm/drm_debugfs.h> #include <drm/drm_drv.h> #include <drm/drm_ioctl.h> +#include <drm/drm_print.h> #include <drm/drm_syncobj.h> #include <drm/drm_utils.h> @@ -36,7 +37,7 @@ static int panfrost_ioctl_query_timestamp(struct panfrost_device *pfdev, { int ret; - ret = pm_runtime_resume_and_get(pfdev->dev); + ret = pm_runtime_resume_and_get(pfdev->base.dev); if (ret) return ret; @@ -44,14 +45,14 @@ static int panfrost_ioctl_query_timestamp(struct panfrost_device *pfdev, *arg = panfrost_timestamp_read(pfdev); panfrost_cycle_counter_put(pfdev); - pm_runtime_put(pfdev->dev); + pm_runtime_put(pfdev->base.dev); return 0; } static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct drm_file *file) { struct drm_panfrost_get_param *param = data; - struct panfrost_device *pfdev = ddev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(ddev); int ret; if (param->pad != 0) @@ -109,6 +110,14 @@ static int panfrost_ioctl_get_param(struct drm_device *ddev, void *data, struct #endif break; + case DRM_PANFROST_PARAM_ALLOWED_JM_CTX_PRIORITIES: + param->value = BIT(PANFROST_JM_CTX_PRIORITY_LOW) | + BIT(PANFROST_JM_CTX_PRIORITY_MEDIUM); + + if (panfrost_high_prio_allowed(file)) + param->value |= BIT(PANFROST_JM_CTX_PRIORITY_HIGH); + break; + default: return -EINVAL; } @@ -275,13 +284,17 @@ fail: static int panfrost_ioctl_submit(struct drm_device *dev, void *data, struct drm_file *file) { - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(dev); struct panfrost_file_priv *file_priv = file->driver_priv; struct drm_panfrost_submit *args = data; struct drm_syncobj *sync_out = NULL; + struct panfrost_jm_ctx *jm_ctx; struct panfrost_job *job; int ret = 0, slot; + if (args->pad) + return -EINVAL; + if (!args->jc) return -EINVAL; @@ -294,10 +307,16 @@ static int panfrost_ioctl_submit(struct drm_device *dev, void *data, return -ENODEV; } + jm_ctx = panfrost_jm_ctx_from_handle(file, args->jm_ctx_handle); + if (!jm_ctx) { + ret = -EINVAL; + goto out_put_syncout; + } + job = kzalloc(sizeof(*job), GFP_KERNEL); if (!job) { ret = -ENOMEM; - goto out_put_syncout; + goto out_put_jm_ctx; } kref_init(&job->refcount); @@ -307,12 +326,13 @@ static int panfrost_ioctl_submit(struct drm_device *dev, void *data, job->requirements = args->requirements; job->flush_id = panfrost_gpu_get_latest_flush_id(pfdev); job->mmu = file_priv->mmu; + job->ctx = panfrost_jm_ctx_get(jm_ctx); job->engine_usage = &file_priv->engine_usage; slot = panfrost_job_get_slot(job); ret = drm_sched_job_init(&job->base, - &file_priv->sched_entity[slot], + &jm_ctx->slot_entity[slot], 1, NULL, file->client_id); if (ret) goto out_put_job; @@ -338,6 +358,8 @@ out_cleanup_job: drm_sched_job_cleanup(&job->base); out_put_job: panfrost_job_put(job); +out_put_jm_ctx: + panfrost_jm_ctx_put(jm_ctx); out_put_syncout: if (sync_out) drm_syncobj_put(sync_out); @@ -436,7 +458,7 @@ static int panfrost_ioctl_madvise(struct drm_device *dev, void *data, { struct panfrost_file_priv *priv = file_priv->driver_priv; struct drm_panfrost_madvise *args = data; - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(dev); struct drm_gem_object *gem_obj; struct panfrost_gem_object *bo; int ret = 0; @@ -536,6 +558,27 @@ err_put_obj: return ret; } +static int panfrost_ioctl_jm_ctx_create(struct drm_device *dev, void *data, + struct drm_file *file) +{ + return panfrost_jm_ctx_create(file, data); +} + +static int panfrost_ioctl_jm_ctx_destroy(struct drm_device *dev, void *data, + struct drm_file *file) +{ + const struct drm_panfrost_jm_ctx_destroy *args = data; + + if (args->pad) + return -EINVAL; + + /* We can't destroy the default context created when the file is opened. */ + if (!args->handle) + return -EINVAL; + + return panfrost_jm_ctx_destroy(file, args->handle); +} + int panfrost_unstable_ioctl_check(void) { if (!unstable_ioctls) @@ -548,7 +591,7 @@ static int panfrost_open(struct drm_device *dev, struct drm_file *file) { int ret; - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(dev); struct panfrost_file_priv *panfrost_priv; panfrost_priv = kzalloc(sizeof(*panfrost_priv), GFP_KERNEL); @@ -564,7 +607,7 @@ panfrost_open(struct drm_device *dev, struct drm_file *file) goto err_free; } - ret = panfrost_job_open(panfrost_priv); + ret = panfrost_jm_open(file); if (ret) goto err_job; @@ -583,7 +626,7 @@ panfrost_postclose(struct drm_device *dev, struct drm_file *file) struct panfrost_file_priv *panfrost_priv = file->driver_priv; panfrost_perfcnt_close(file); - panfrost_job_close(panfrost_priv); + panfrost_jm_close(file); panfrost_mmu_ctx_put(panfrost_priv->mmu); kfree(panfrost_priv); @@ -603,6 +646,8 @@ static const struct drm_ioctl_desc panfrost_drm_driver_ioctls[] = { PANFROST_IOCTL(PERFCNT_DUMP, perfcnt_dump, DRM_RENDER_ALLOW), PANFROST_IOCTL(MADVISE, madvise, DRM_RENDER_ALLOW), PANFROST_IOCTL(SET_LABEL_BO, set_label_bo, DRM_RENDER_ALLOW), + PANFROST_IOCTL(JM_CTX_CREATE, jm_ctx_create, DRM_RENDER_ALLOW), + PANFROST_IOCTL(JM_CTX_DESTROY, jm_ctx_destroy, DRM_RENDER_ALLOW), }; static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev, @@ -624,30 +669,25 @@ static void panfrost_gpu_show_fdinfo(struct panfrost_device *pfdev, * job spent on the GPU. */ - static const char * const engine_names[] = { - "fragment", "vertex-tiler", "compute-only" - }; - - BUILD_BUG_ON(ARRAY_SIZE(engine_names) != NUM_JOB_SLOTS); - for (i = 0; i < NUM_JOB_SLOTS - 1; i++) { if (pfdev->profile_mode) { drm_printf(p, "drm-engine-%s:\t%llu ns\n", - engine_names[i], panfrost_priv->engine_usage.elapsed_ns[i]); + panfrost_engine_names[i], + panfrost_priv->engine_usage.elapsed_ns[i]); drm_printf(p, "drm-cycles-%s:\t%llu\n", - engine_names[i], panfrost_priv->engine_usage.cycles[i]); + panfrost_engine_names[i], + panfrost_priv->engine_usage.cycles[i]); } drm_printf(p, "drm-maxfreq-%s:\t%lu Hz\n", - engine_names[i], pfdev->pfdevfreq.fast_rate); + panfrost_engine_names[i], pfdev->pfdevfreq.fast_rate); drm_printf(p, "drm-curfreq-%s:\t%lu Hz\n", - engine_names[i], pfdev->pfdevfreq.current_frequency); + panfrost_engine_names[i], pfdev->pfdevfreq.current_frequency); } } static void panfrost_show_fdinfo(struct drm_printer *p, struct drm_file *file) { - struct drm_device *dev = file->minor->dev; - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(file->minor->dev); panfrost_gpu_show_fdinfo(pfdev, file->driver_priv, p); @@ -664,16 +704,57 @@ static const struct file_operations panfrost_drm_driver_fops = { static int panthor_gems_show(struct seq_file *m, void *data) { struct drm_info_node *node = m->private; - struct drm_device *dev = node->minor->dev; - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(node->minor->dev); panfrost_gem_debugfs_print_bos(pfdev, m); return 0; } +static void show_panfrost_jm_ctx(struct panfrost_jm_ctx *jm_ctx, u32 handle, + struct seq_file *m) +{ + struct drm_device *ddev = ((struct drm_info_node *)m->private)->minor->dev; + const char *prio = "UNKNOWN"; + + static const char * const prios[] = { + [DRM_SCHED_PRIORITY_HIGH] = "HIGH", + [DRM_SCHED_PRIORITY_NORMAL] = "NORMAL", + [DRM_SCHED_PRIORITY_LOW] = "LOW", + }; + + if (jm_ctx->slot_entity[0].priority != + jm_ctx->slot_entity[1].priority) + drm_warn(ddev, "Slot priorities should be the same in a single context"); + + if (jm_ctx->slot_entity[0].priority < ARRAY_SIZE(prios)) + prio = prios[jm_ctx->slot_entity[0].priority]; + + seq_printf(m, " JM context %u: priority %s\n", handle, prio); +} + +static int show_file_jm_ctxs(struct panfrost_file_priv *pfile, + struct seq_file *m) +{ + struct panfrost_jm_ctx *jm_ctx; + unsigned long i; + + xa_lock(&pfile->jm_ctxs); + xa_for_each(&pfile->jm_ctxs, i, jm_ctx) { + jm_ctx = panfrost_jm_ctx_get(jm_ctx); + xa_unlock(&pfile->jm_ctxs); + show_panfrost_jm_ctx(jm_ctx, i, m); + panfrost_jm_ctx_put(jm_ctx); + xa_lock(&pfile->jm_ctxs); + } + xa_unlock(&pfile->jm_ctxs); + + return 0; +} + static struct drm_info_list panthor_debugfs_list[] = { - {"gems", panthor_gems_show, 0, NULL}, + {"gems", + panthor_gems_show, 0, NULL}, }; static int panthor_gems_debugfs_init(struct drm_minor *minor) @@ -685,9 +766,64 @@ static int panthor_gems_debugfs_init(struct drm_minor *minor) return 0; } +static int show_each_file(struct seq_file *m, void *arg) +{ + struct drm_info_node *node = (struct drm_info_node *)m->private; + struct drm_device *ddev = node->minor->dev; + int (*show)(struct panfrost_file_priv *, struct seq_file *) = + node->info_ent->data; + struct drm_file *file; + int ret; + + ret = mutex_lock_interruptible(&ddev->filelist_mutex); + if (ret) + return ret; + + list_for_each_entry(file, &ddev->filelist, lhead) { + struct task_struct *task; + struct panfrost_file_priv *pfile = file->driver_priv; + struct pid *pid; + + /* + * Although we have a valid reference on file->pid, that does + * not guarantee that the task_struct who called get_pid() is + * still alive (e.g. get_pid(current) => fork() => exit()). + * Therefore, we need to protect this ->comm access using RCU. + */ + rcu_read_lock(); + pid = rcu_dereference(file->pid); + task = pid_task(pid, PIDTYPE_TGID); + seq_printf(m, "client_id %8llu pid %8d command %s:\n", + file->client_id, pid_nr(pid), + task ? task->comm : "<unknown>"); + rcu_read_unlock(); + + ret = show(pfile, m); + if (ret < 0) + break; + + seq_puts(m, "\n"); + } + + mutex_unlock(&ddev->filelist_mutex); + return ret; +} + +static struct drm_info_list panfrost_sched_debugfs_list[] = { + { "sched_ctxs", show_each_file, 0, show_file_jm_ctxs }, +}; + +static void panfrost_sched_debugfs_init(struct drm_minor *minor) +{ + drm_debugfs_create_files(panfrost_sched_debugfs_list, + ARRAY_SIZE(panfrost_sched_debugfs_list), + minor->debugfs_root, minor); +} + static void panfrost_debugfs_init(struct drm_minor *minor) { panthor_gems_debugfs_init(minor); + panfrost_sched_debugfs_init(minor); } #endif @@ -699,6 +835,8 @@ static void panfrost_debugfs_init(struct drm_minor *minor) * - 1.3 - adds JD_REQ_CYCLE_COUNT job requirement for SUBMIT * - adds SYSTEM_TIMESTAMP and SYSTEM_TIMESTAMP_FREQUENCY queries * - 1.4 - adds SET_LABEL_BO + * - 1.5 - adds JM_CTX_{CREATE,DESTROY} ioctls and extend SUBMIT to allow + * context creation with configurable priorities/affinity */ static const struct drm_driver panfrost_drm_driver = { .driver_features = DRIVER_RENDER | DRIVER_GEM | DRIVER_SYNCOBJ, @@ -711,7 +849,7 @@ static const struct drm_driver panfrost_drm_driver = { .name = "panfrost", .desc = "panfrost DRM", .major = 1, - .minor = 4, + .minor = 5, .gem_create_object = panfrost_gem_create_object, .gem_prime_import_sg_table = panfrost_gem_prime_import_sg_table, @@ -723,15 +861,12 @@ static const struct drm_driver panfrost_drm_driver = { static int panfrost_probe(struct platform_device *pdev) { struct panfrost_device *pfdev; - struct drm_device *ddev; int err; - pfdev = devm_kzalloc(&pdev->dev, sizeof(*pfdev), GFP_KERNEL); - if (!pfdev) - return -ENOMEM; - - pfdev->pdev = pdev; - pfdev->dev = &pdev->dev; + pfdev = devm_drm_dev_alloc(&pdev->dev, &panfrost_drm_driver, + struct panfrost_device, base); + if (IS_ERR(pfdev)) + return PTR_ERR(pfdev); platform_set_drvdata(pdev, pfdev); @@ -741,14 +876,6 @@ static int panfrost_probe(struct platform_device *pdev) pfdev->coherent = device_get_dma_attr(&pdev->dev) == DEV_DMA_COHERENT; - /* Allocate and initialize the DRM device. */ - ddev = drm_dev_alloc(&panfrost_drm_driver, &pdev->dev); - if (IS_ERR(ddev)) - return PTR_ERR(ddev); - - ddev->dev_private = pfdev; - pfdev->ddev = ddev; - mutex_init(&pfdev->shrinker_lock); INIT_LIST_HEAD(&pfdev->shrinker_list); @@ -759,51 +886,47 @@ static int panfrost_probe(struct platform_device *pdev) goto err_out0; } - pm_runtime_set_active(pfdev->dev); - pm_runtime_mark_last_busy(pfdev->dev); - pm_runtime_enable(pfdev->dev); - pm_runtime_set_autosuspend_delay(pfdev->dev, 50); /* ~3 frames */ - pm_runtime_use_autosuspend(pfdev->dev); + pm_runtime_set_active(pfdev->base.dev); + pm_runtime_mark_last_busy(pfdev->base.dev); + pm_runtime_enable(pfdev->base.dev); + pm_runtime_set_autosuspend_delay(pfdev->base.dev, 50); /* ~3 frames */ + pm_runtime_use_autosuspend(pfdev->base.dev); /* * Register the DRM device with the core and the connectors with * sysfs */ - err = drm_dev_register(ddev, 0); + err = drm_dev_register(&pfdev->base, 0); if (err < 0) goto err_out1; - err = panfrost_gem_shrinker_init(ddev); + err = panfrost_gem_shrinker_init(&pfdev->base); if (err) goto err_out2; return 0; err_out2: - drm_dev_unregister(ddev); + drm_dev_unregister(&pfdev->base); err_out1: - pm_runtime_disable(pfdev->dev); + pm_runtime_disable(pfdev->base.dev); panfrost_device_fini(pfdev); - pm_runtime_set_suspended(pfdev->dev); + pm_runtime_set_suspended(pfdev->base.dev); err_out0: - drm_dev_put(ddev); return err; } static void panfrost_remove(struct platform_device *pdev) { struct panfrost_device *pfdev = platform_get_drvdata(pdev); - struct drm_device *ddev = pfdev->ddev; - drm_dev_unregister(ddev); - panfrost_gem_shrinker_cleanup(ddev); + drm_dev_unregister(&pfdev->base); + panfrost_gem_shrinker_cleanup(&pfdev->base); - pm_runtime_get_sync(pfdev->dev); - pm_runtime_disable(pfdev->dev); + pm_runtime_get_sync(pfdev->base.dev); + pm_runtime_disable(pfdev->base.dev); panfrost_device_fini(pfdev); - pm_runtime_set_suspended(pfdev->dev); - - drm_dev_put(ddev); + pm_runtime_set_suspended(pfdev->base.dev); } static ssize_t profiling_show(struct device *dev, diff --git a/drivers/gpu/drm/panfrost/panfrost_dump.c b/drivers/gpu/drm/panfrost/panfrost_dump.c index 4042afe2fbf4..3ed6c902d0a1 100644 --- a/drivers/gpu/drm/panfrost/panfrost_dump.c +++ b/drivers/gpu/drm/panfrost/panfrost_dump.c @@ -163,7 +163,7 @@ void panfrost_core_dump(struct panfrost_job *job) iter.start = __vmalloc(file_size, GFP_KERNEL | __GFP_NOWARN | __GFP_NORETRY); if (!iter.start) { - dev_warn(pfdev->dev, "failed to allocate devcoredump file\n"); + dev_warn(pfdev->base.dev, "failed to allocate devcoredump file\n"); return; } @@ -204,14 +204,14 @@ void panfrost_core_dump(struct panfrost_job *job) mapping = job->mappings[i]; if (!bo->base.sgt) { - dev_err(pfdev->dev, "Panfrost Dump: BO has no sgt, cannot dump\n"); + dev_err(pfdev->base.dev, "Panfrost Dump: BO has no sgt, cannot dump\n"); iter.hdr->bomap.valid = 0; goto dump_header; } ret = drm_gem_vmap(&bo->base.base, &map); if (ret) { - dev_err(pfdev->dev, "Panfrost Dump: couldn't map Buffer Object\n"); + dev_err(pfdev->base.dev, "Panfrost Dump: couldn't map Buffer Object\n"); iter.hdr->bomap.valid = 0; goto dump_header; } @@ -237,5 +237,5 @@ dump_header: panfrost_core_dump_header(&iter, PANFROSTDUMP_BUF_BO, iter.data + } panfrost_core_dump_header(&iter, PANFROSTDUMP_BUF_TRAILER, iter.data); - dev_coredumpv(pfdev->dev, iter.start, iter.data - iter.start, GFP_KERNEL); + dev_coredumpv(pfdev->base.dev, iter.start, iter.data - iter.start, GFP_KERNEL); } diff --git a/drivers/gpu/drm/panfrost/panfrost_gem.c b/drivers/gpu/drm/panfrost/panfrost_gem.c index 85d6289a6eda..8041b65c6609 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem.c +++ b/drivers/gpu/drm/panfrost/panfrost_gem.c @@ -8,6 +8,7 @@ #include <linux/dma-mapping.h> #include <drm/panfrost_drm.h> +#include <drm/drm_print.h> #include "panfrost_device.h" #include "panfrost_gem.h" #include "panfrost_mmu.h" @@ -26,7 +27,7 @@ static void panfrost_gem_debugfs_bo_add(struct panfrost_device *pfdev, static void panfrost_gem_debugfs_bo_rm(struct panfrost_gem_object *bo) { - struct panfrost_device *pfdev = bo->base.base.dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(bo->base.base.dev); if (list_empty(&bo->debugfs.node)) return; @@ -48,7 +49,7 @@ static void panfrost_gem_debugfs_bo_rm(struct panfrost_gem_object *bo) {} static void panfrost_gem_free_object(struct drm_gem_object *obj) { struct panfrost_gem_object *bo = to_panfrost_bo(obj); - struct panfrost_device *pfdev = obj->dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(obj->dev); /* * Make sure the BO is no longer inserted in the shrinker list before @@ -76,7 +77,7 @@ static void panfrost_gem_free_object(struct drm_gem_object *obj) for (i = 0; i < n_sgt; i++) { if (bo->sgts[i].sgl) { - dma_unmap_sgtable(pfdev->dev, &bo->sgts[i], + dma_unmap_sgtable(pfdev->base.dev, &bo->sgts[i], DMA_BIDIRECTIONAL, 0); sg_free_table(&bo->sgts[i]); } @@ -284,7 +285,7 @@ static const struct drm_gem_object_funcs panfrost_gem_funcs = { */ struct drm_gem_object *panfrost_gem_create_object(struct drm_device *dev, size_t size) { - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(dev); struct panfrost_gem_object *obj; obj = kzalloc(sizeof(*obj), GFP_KERNEL); diff --git a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c index 02b60ea1433a..2fe967a90bcb 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c +++ b/drivers/gpu/drm/panfrost/panfrost_gem_shrinker.c @@ -97,7 +97,7 @@ panfrost_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) */ int panfrost_gem_shrinker_init(struct drm_device *dev) { - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(dev); pfdev->shrinker = shrinker_alloc(0, "drm-panfrost"); if (!pfdev->shrinker) @@ -120,7 +120,7 @@ int panfrost_gem_shrinker_init(struct drm_device *dev) */ void panfrost_gem_shrinker_cleanup(struct drm_device *dev) { - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(dev); if (pfdev->shrinker) shrinker_free(pfdev->shrinker); diff --git a/drivers/gpu/drm/panfrost/panfrost_gpu.c b/drivers/gpu/drm/panfrost/panfrost_gpu.c index 174e190ba40f..483d278eb154 100644 --- a/drivers/gpu/drm/panfrost/panfrost_gpu.c +++ b/drivers/gpu/drm/panfrost/panfrost_gpu.c @@ -12,6 +12,8 @@ #include <linux/platform_device.h> #include <linux/pm_runtime.h> +#include <drm/drm_print.h> + #include "panfrost_device.h" #include "panfrost_features.h" #include "panfrost_issues.h" @@ -36,12 +38,12 @@ static irqreturn_t panfrost_gpu_irq_handler(int irq, void *data) u64 address = (u64) gpu_read(pfdev, GPU_FAULT_ADDRESS_HI) << 32; address |= gpu_read(pfdev, GPU_FAULT_ADDRESS_LO); - dev_warn(pfdev->dev, "GPU Fault 0x%08x (%s) at 0x%016llx\n", + dev_warn(pfdev->base.dev, "GPU Fault 0x%08x (%s) at 0x%016llx\n", fault_status, panfrost_exception_name(fault_status & 0xFF), address); if (state & GPU_IRQ_MULTIPLE_FAULT) - dev_warn(pfdev->dev, "There were multiple GPU faults - some have not been reported\n"); + dev_warn(pfdev->base.dev, "There were multiple GPU faults - some have not been reported\n"); gpu_write(pfdev, GPU_INT_MASK, 0); } @@ -72,13 +74,13 @@ int panfrost_gpu_soft_reset(struct panfrost_device *pfdev) val, val & GPU_IRQ_RESET_COMPLETED, 10, 10000); if (ret) { - dev_err(pfdev->dev, "gpu soft reset timed out, attempting hard reset\n"); + dev_err(pfdev->base.dev, "gpu soft reset timed out, attempting hard reset\n"); gpu_write(pfdev, GPU_CMD, GPU_CMD_HARD_RESET); ret = readl_relaxed_poll_timeout(pfdev->iomem + GPU_INT_RAWSTAT, val, val & GPU_IRQ_RESET_COMPLETED, 100, 10000); if (ret) { - dev_err(pfdev->dev, "gpu hard reset timed out\n"); + dev_err(pfdev->base.dev, "gpu hard reset timed out\n"); return ret; } } @@ -95,7 +97,7 @@ int panfrost_gpu_soft_reset(struct panfrost_device *pfdev) * All in-flight jobs should have released their cycle * counter references upon reset, but let us make sure */ - if (drm_WARN_ON(pfdev->ddev, atomic_read(&pfdev->cycle_counter.use_count) != 0)) + if (drm_WARN_ON(&pfdev->base, atomic_read(&pfdev->cycle_counter.use_count) != 0)) atomic_set(&pfdev->cycle_counter.use_count, 0); return 0; @@ -240,9 +242,10 @@ static const struct panfrost_model gpu_models[] = { /* MediaTek MT8188 Mali-G57 MC3 */ GPU_MODEL(g57, 0x9093, GPU_REV(g57, 0, 0)), + {0}, }; -static void panfrost_gpu_init_features(struct panfrost_device *pfdev) +static int panfrost_gpu_init_features(struct panfrost_device *pfdev) { u32 gpu_id, num_js, major, minor, status, rev; const char *name = "unknown"; @@ -327,16 +330,22 @@ static void panfrost_gpu_init_features(struct panfrost_device *pfdev) break; } + if (!model->name) { + dev_err(pfdev->base.dev, "GPU model not found: mali-%s id rev %#x %#x\n", + name, gpu_id, rev); + return -ENODEV; + } + bitmap_from_u64(pfdev->features.hw_features, hw_feat); bitmap_from_u64(pfdev->features.hw_issues, hw_issues); - dev_info(pfdev->dev, "mali-%s id 0x%x major 0x%x minor 0x%x status 0x%x", + dev_info(pfdev->base.dev, "mali-%s id 0x%x major 0x%x minor 0x%x status 0x%x", name, gpu_id, major, minor, status); - dev_info(pfdev->dev, "features: %64pb, issues: %64pb", + dev_info(pfdev->base.dev, "features: %64pb, issues: %64pb", pfdev->features.hw_features, pfdev->features.hw_issues); - dev_info(pfdev->dev, "Features: L2:0x%08x Shader:0x%08x Tiler:0x%08x Mem:0x%0x MMU:0x%08x AS:0x%x JS:0x%x", + dev_info(pfdev->base.dev, "Features: L2:0x%08x Shader:0x%08x Tiler:0x%08x Mem:0x%0x MMU:0x%08x AS:0x%x JS:0x%x", pfdev->features.l2_features, pfdev->features.core_features, pfdev->features.tiler_features, @@ -345,8 +354,10 @@ static void panfrost_gpu_init_features(struct panfrost_device *pfdev) pfdev->features.as_present, pfdev->features.js_present); - dev_info(pfdev->dev, "shader_present=0x%0llx l2_present=0x%0llx", + dev_info(pfdev->base.dev, "shader_present=0x%0llx l2_present=0x%0llx", pfdev->features.shader_present, pfdev->features.l2_present); + + return 0; } void panfrost_cycle_counter_get(struct panfrost_device *pfdev) @@ -411,7 +422,7 @@ static u64 panfrost_get_core_mask(struct panfrost_device *pfdev) */ core_mask = ~(pfdev->features.l2_present - 1) & (pfdev->features.l2_present - 2); - dev_info_once(pfdev->dev, "using only 1st core group (%lu cores from %lu)\n", + dev_info_once(pfdev->base.dev, "using only 1st core group (%lu cores from %lu)\n", hweight64(core_mask), hweight64(pfdev->features.shader_present)); @@ -432,7 +443,7 @@ void panfrost_gpu_power_on(struct panfrost_device *pfdev) val, val == (pfdev->features.l2_present & core_mask), 10, 20000); if (ret) - dev_err(pfdev->dev, "error powering up gpu L2"); + dev_err(pfdev->base.dev, "error powering up gpu L2"); gpu_write(pfdev, SHADER_PWRON_LO, pfdev->features.shader_present & core_mask); @@ -440,13 +451,13 @@ void panfrost_gpu_power_on(struct panfrost_device *pfdev) val, val == (pfdev->features.shader_present & core_mask), 10, 20000); if (ret) - dev_err(pfdev->dev, "error powering up gpu shader"); + dev_err(pfdev->base.dev, "error powering up gpu shader"); gpu_write(pfdev, TILER_PWRON_LO, pfdev->features.tiler_present); ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_READY_LO, val, val == pfdev->features.tiler_present, 10, 1000); if (ret) - dev_err(pfdev->dev, "error powering up gpu tiler"); + dev_err(pfdev->base.dev, "error powering up gpu tiler"); } void panfrost_gpu_power_off(struct panfrost_device *pfdev) @@ -458,19 +469,19 @@ void panfrost_gpu_power_off(struct panfrost_device *pfdev) ret = readl_relaxed_poll_timeout(pfdev->iomem + SHADER_PWRTRANS_LO, val, !val, 1, 2000); if (ret) - dev_err(pfdev->dev, "shader power transition timeout"); + dev_err(pfdev->base.dev, "shader power transition timeout"); gpu_write(pfdev, TILER_PWROFF_LO, pfdev->features.tiler_present); ret = readl_relaxed_poll_timeout(pfdev->iomem + TILER_PWRTRANS_LO, val, !val, 1, 2000); if (ret) - dev_err(pfdev->dev, "tiler power transition timeout"); + dev_err(pfdev->base.dev, "tiler power transition timeout"); gpu_write(pfdev, L2_PWROFF_LO, pfdev->features.l2_present); ret = readl_poll_timeout(pfdev->iomem + L2_PWRTRANS_LO, val, !val, 0, 2000); if (ret) - dev_err(pfdev->dev, "l2 power transition timeout"); + dev_err(pfdev->base.dev, "l2 power transition timeout"); } void panfrost_gpu_suspend_irq(struct panfrost_device *pfdev) @@ -489,23 +500,26 @@ int panfrost_gpu_init(struct panfrost_device *pfdev) if (err) return err; - panfrost_gpu_init_features(pfdev); + err = panfrost_gpu_init_features(pfdev); + if (err) + return err; - err = dma_set_mask_and_coherent(pfdev->dev, - DMA_BIT_MASK(FIELD_GET(0xff00, pfdev->features.mmu_features))); + err = dma_set_mask_and_coherent(pfdev->base.dev, + DMA_BIT_MASK(FIELD_GET(0xff00, + pfdev->features.mmu_features))); if (err) return err; - dma_set_max_seg_size(pfdev->dev, UINT_MAX); + dma_set_max_seg_size(pfdev->base.dev, UINT_MAX); - pfdev->gpu_irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "gpu"); + pfdev->gpu_irq = platform_get_irq_byname(to_platform_device(pfdev->base.dev), "gpu"); if (pfdev->gpu_irq < 0) return pfdev->gpu_irq; - err = devm_request_irq(pfdev->dev, pfdev->gpu_irq, panfrost_gpu_irq_handler, + err = devm_request_irq(pfdev->base.dev, pfdev->gpu_irq, panfrost_gpu_irq_handler, IRQF_SHARED, KBUILD_MODNAME "-gpu", pfdev); if (err) { - dev_err(pfdev->dev, "failed to request gpu irq"); + dev_err(pfdev->base.dev, "failed to request gpu irq"); return err; } @@ -525,9 +539,9 @@ u32 panfrost_gpu_get_latest_flush_id(struct panfrost_device *pfdev) if (panfrost_has_hw_feature(pfdev, HW_FEATURE_FLUSH_REDUCTION)) { /* Flush reduction only makes sense when the GPU is kept powered on between jobs */ - if (pm_runtime_get_if_in_use(pfdev->dev)) { + if (pm_runtime_get_if_in_use(pfdev->base.dev)) { flush_id = gpu_read(pfdev, GPU_LATEST_FLUSH_ID); - pm_runtime_put(pfdev->dev); + pm_runtime_put(pfdev->base.dev); return flush_id; } } diff --git a/drivers/gpu/drm/panfrost/panfrost_job.c b/drivers/gpu/drm/panfrost/panfrost_job.c index 82acabb21b27..11894a6b9fcc 100644 --- a/drivers/gpu/drm/panfrost/panfrost_job.c +++ b/drivers/gpu/drm/panfrost/panfrost_job.c @@ -22,11 +22,16 @@ #include "panfrost_mmu.h" #include "panfrost_dump.h" +#define MAX_JM_CTX_PER_FILE 64 #define JOB_TIMEOUT_MS 500 #define job_write(dev, reg, data) writel(data, dev->iomem + (reg)) #define job_read(dev, reg) readl(dev->iomem + (reg)) +const char * const panfrost_engine_names[] = { + "fragment", "vertex-tiler", "compute-only" +}; + struct panfrost_queue_state { struct drm_gpu_scheduler sched; u64 fence_context; @@ -94,7 +99,7 @@ static struct dma_fence *panfrost_fence_create(struct panfrost_device *pfdev, in if (!fence) return ERR_PTR(-ENOMEM); - fence->dev = pfdev->ddev; + fence->dev = &pfdev->base; fence->queue = js_num; fence->seqno = ++js->queue[js_num].emit_seqno; dma_fence_init(&fence->base, &panfrost_fence_ops, &js->job_lock, @@ -195,7 +200,7 @@ panfrost_enqueue_job(struct panfrost_device *pfdev, int slot, return 1; } -static void panfrost_job_hw_submit(struct panfrost_job *job, int js) +static int panfrost_job_hw_submit(struct panfrost_job *job, int js) { struct panfrost_device *pfdev = job->pfdev; unsigned int subslot; @@ -203,17 +208,22 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js) u64 jc_head = job->jc; int ret; - panfrost_devfreq_record_busy(&pfdev->pfdevfreq); - - ret = pm_runtime_get_sync(pfdev->dev); + ret = pm_runtime_get_sync(pfdev->base.dev); if (ret < 0) - return; + goto err_hwsubmit; if (WARN_ON(job_read(pfdev, JS_COMMAND_NEXT(js)))) { - return; + ret = -EINVAL; + goto err_hwsubmit; } - cfg = panfrost_mmu_as_get(pfdev, job->mmu); + ret = panfrost_mmu_as_get(pfdev, job->mmu); + if (ret < 0) + goto err_hwsubmit; + + cfg = ret; + + panfrost_devfreq_record_busy(&pfdev->pfdevfreq); job_write(pfdev, JS_HEAD_NEXT_LO(js), lower_32_bits(jc_head)); job_write(pfdev, JS_HEAD_NEXT_HI(js), upper_32_bits(jc_head)); @@ -256,11 +266,17 @@ static void panfrost_job_hw_submit(struct panfrost_job *job, int js) } job_write(pfdev, JS_COMMAND_NEXT(js), JS_COMMAND_START); - dev_dbg(pfdev->dev, + dev_dbg(pfdev->base.dev, "JS: Submitting atom %p to js[%d][%d] with head=0x%llx AS %d", job, js, subslot, jc_head, cfg & 0xf); } spin_unlock(&pfdev->js->job_lock); + + return 0; + +err_hwsubmit: + pm_runtime_put_autosuspend(pfdev->base.dev); + return ret; } static int panfrost_acquire_object_fences(struct drm_gem_object **bos, @@ -359,6 +375,7 @@ static void panfrost_job_cleanup(struct kref *ref) kvfree(job->bos); } + panfrost_jm_ctx_put(job->ctx); kfree(job); } @@ -382,6 +399,10 @@ static struct dma_fence *panfrost_job_run(struct drm_sched_job *sched_job) struct panfrost_device *pfdev = job->pfdev; int slot = panfrost_job_get_slot(job); struct dma_fence *fence = NULL; + int ret; + + if (job->ctx->destroyed) + return ERR_PTR(-ECANCELED); if (unlikely(job->base.s_fence->finished.error)) return NULL; @@ -400,27 +421,27 @@ static struct dma_fence *panfrost_job_run(struct drm_sched_job *sched_job) dma_fence_put(job->done_fence); job->done_fence = dma_fence_get(fence); - panfrost_job_hw_submit(job, slot); + ret = panfrost_job_hw_submit(job, slot); + if (ret) { + dma_fence_put(fence); + return ERR_PTR(ret); + } return fence; } -void panfrost_job_enable_interrupts(struct panfrost_device *pfdev) +void panfrost_jm_reset_interrupts(struct panfrost_device *pfdev) { - int j; - u32 irq_mask = 0; + job_write(pfdev, JOB_INT_CLEAR, ALL_JS_INT_MASK); +} +void panfrost_jm_enable_interrupts(struct panfrost_device *pfdev) +{ clear_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended); - - for (j = 0; j < NUM_JOB_SLOTS; j++) { - irq_mask |= MK_JS_MASK(j); - } - - job_write(pfdev, JOB_INT_CLEAR, irq_mask); - job_write(pfdev, JOB_INT_MASK, irq_mask); + job_write(pfdev, JOB_INT_MASK, ALL_JS_INT_MASK); } -void panfrost_job_suspend_irq(struct panfrost_device *pfdev) +void panfrost_jm_suspend_irq(struct panfrost_device *pfdev) { set_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended); @@ -437,12 +458,12 @@ static void panfrost_job_handle_err(struct panfrost_device *pfdev, bool signal_fence = true; if (!panfrost_exception_is_fault(js_status)) { - dev_dbg(pfdev->dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x", + dev_dbg(pfdev->base.dev, "js event, js=%d, status=%s, head=0x%x, tail=0x%x", js, exception_name, job_read(pfdev, JS_HEAD_LO(js)), job_read(pfdev, JS_TAIL_LO(js))); } else { - dev_err(pfdev->dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x", + dev_err(pfdev->base.dev, "js fault, js=%d, status=%s, head=0x%x, tail=0x%x", js, exception_name, job_read(pfdev, JS_HEAD_LO(js)), job_read(pfdev, JS_TAIL_LO(js))); @@ -474,7 +495,7 @@ static void panfrost_job_handle_err(struct panfrost_device *pfdev, if (signal_fence) dma_fence_signal_locked(job->done_fence); - pm_runtime_put_autosuspend(pfdev->dev); + pm_runtime_put_autosuspend(pfdev->base.dev); if (panfrost_exception_needs_reset(pfdev, js_status)) { atomic_set(&pfdev->reset.pending, 1); @@ -482,8 +503,8 @@ static void panfrost_job_handle_err(struct panfrost_device *pfdev, } } -static void panfrost_job_handle_done(struct panfrost_device *pfdev, - struct panfrost_job *job) +static void panfrost_jm_handle_done(struct panfrost_device *pfdev, + struct panfrost_job *job) { /* Set ->jc to 0 to avoid re-submitting an already finished job (can * happen when we receive the DONE interrupt while doing a GPU reset). @@ -493,10 +514,10 @@ static void panfrost_job_handle_done(struct panfrost_device *pfdev, panfrost_devfreq_record_idle(&pfdev->pfdevfreq); dma_fence_signal_locked(job->done_fence); - pm_runtime_put_autosuspend(pfdev->dev); + pm_runtime_put_autosuspend(pfdev->base.dev); } -static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status) +static void panfrost_jm_handle_irq(struct panfrost_device *pfdev, u32 status) { struct panfrost_job *done[NUM_JOB_SLOTS][2] = {}; struct panfrost_job *failed[NUM_JOB_SLOTS] = {}; @@ -571,7 +592,7 @@ static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status) } for (i = 0; i < ARRAY_SIZE(done[0]) && done[j][i]; i++) - panfrost_job_handle_done(pfdev, done[j][i]); + panfrost_jm_handle_done(pfdev, done[j][i]); } /* And finally we requeue jobs that were waiting in the second slot @@ -589,7 +610,7 @@ static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status) struct panfrost_job *canceled = panfrost_dequeue_job(pfdev, j); dma_fence_set_error(canceled->done_fence, -ECANCELED); - panfrost_job_handle_done(pfdev, canceled); + panfrost_jm_handle_done(pfdev, canceled); } else if (!atomic_read(&pfdev->reset.pending)) { /* Requeue the job we removed if no reset is pending */ job_write(pfdev, JS_COMMAND_NEXT(j), JS_COMMAND_START); @@ -597,15 +618,15 @@ static void panfrost_job_handle_irq(struct panfrost_device *pfdev, u32 status) } } -static void panfrost_job_handle_irqs(struct panfrost_device *pfdev) +static void panfrost_jm_handle_irqs(struct panfrost_device *pfdev) { u32 status = job_read(pfdev, JOB_INT_RAWSTAT); while (status) { - pm_runtime_mark_last_busy(pfdev->dev); + pm_runtime_mark_last_busy(pfdev->base.dev); spin_lock(&pfdev->js->job_lock); - panfrost_job_handle_irq(pfdev, status); + panfrost_jm_handle_irq(pfdev, status); spin_unlock(&pfdev->js->job_lock); status = job_read(pfdev, JOB_INT_RAWSTAT); } @@ -683,10 +704,10 @@ panfrost_reset(struct panfrost_device *pfdev, 10, 10000); if (ret) - dev_err(pfdev->dev, "Soft-stop failed\n"); + dev_err(pfdev->base.dev, "Soft-stop failed\n"); /* Handle the remaining interrupts before we reset. */ - panfrost_job_handle_irqs(pfdev); + panfrost_jm_handle_irqs(pfdev); /* Remaining interrupts have been handled, but we might still have * stuck jobs. Let's make sure the PM counters stay balanced by @@ -701,7 +722,7 @@ panfrost_reset(struct panfrost_device *pfdev, if (pfdev->jobs[i][j]->requirements & PANFROST_JD_REQ_CYCLE_COUNT || pfdev->jobs[i][j]->is_profiled) panfrost_cycle_counter_put(pfdev->jobs[i][j]->pfdev); - pm_runtime_put_noidle(pfdev->dev); + pm_runtime_put_noidle(pfdev->base.dev); panfrost_devfreq_record_idle(&pfdev->pfdevfreq); } } @@ -709,12 +730,7 @@ panfrost_reset(struct panfrost_device *pfdev, spin_unlock(&pfdev->js->job_lock); /* Proceed with reset now. */ - panfrost_device_reset(pfdev); - - /* panfrost_device_reset() unmasks job interrupts, but we want to - * keep them masked a bit longer. - */ - job_write(pfdev, JOB_INT_MASK, 0); + panfrost_device_reset(pfdev, false); /* GPU has been reset, we can clear the reset pending bit. */ atomic_set(&pfdev->reset.pending, 0); @@ -736,9 +752,7 @@ panfrost_reset(struct panfrost_device *pfdev, drm_sched_start(&pfdev->js->queue[i].sched, 0); /* Re-enable job interrupts now that everything has been restarted. */ - job_write(pfdev, JOB_INT_MASK, - GENMASK(16 + NUM_JOB_SLOTS - 1, 16) | - GENMASK(NUM_JOB_SLOTS - 1, 0)); + panfrost_jm_enable_interrupts(pfdev); dma_fence_end_signalling(cookie); } @@ -769,11 +783,11 @@ static enum drm_gpu_sched_stat panfrost_job_timedout(struct drm_sched_job synchronize_irq(pfdev->js->irq); if (dma_fence_is_signaled(job->done_fence)) { - dev_warn(pfdev->dev, "unexpectedly high interrupt latency\n"); + dev_warn(pfdev->base.dev, "unexpectedly high interrupt latency\n"); return DRM_GPU_SCHED_STAT_NO_HANG; } - dev_err(pfdev->dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p", + dev_err(pfdev->base.dev, "gpu sched timeout, js=%d, config=0x%x, status=0x%x, head=0x%x, tail=0x%x, sched_job=%p", js, job_read(pfdev, JS_CONFIG(js)), job_read(pfdev, JS_STATUS(js)), @@ -803,22 +817,20 @@ static const struct drm_sched_backend_ops panfrost_sched_ops = { .free_job = panfrost_job_free }; -static irqreturn_t panfrost_job_irq_handler_thread(int irq, void *data) +static irqreturn_t panfrost_jm_irq_handler_thread(int irq, void *data) { struct panfrost_device *pfdev = data; - panfrost_job_handle_irqs(pfdev); + panfrost_jm_handle_irqs(pfdev); /* Enable interrupts only if we're not about to get suspended */ if (!test_bit(PANFROST_COMP_BIT_JOB, pfdev->is_suspended)) - job_write(pfdev, JOB_INT_MASK, - GENMASK(16 + NUM_JOB_SLOTS - 1, 16) | - GENMASK(NUM_JOB_SLOTS - 1, 0)); + job_write(pfdev, JOB_INT_MASK, ALL_JS_INT_MASK); return IRQ_HANDLED; } -static irqreturn_t panfrost_job_irq_handler(int irq, void *data) +static irqreturn_t panfrost_jm_irq_handler(int irq, void *data) { struct panfrost_device *pfdev = data; u32 status; @@ -834,19 +846,20 @@ static irqreturn_t panfrost_job_irq_handler(int irq, void *data) return IRQ_WAKE_THREAD; } -int panfrost_job_init(struct panfrost_device *pfdev) +int panfrost_jm_init(struct panfrost_device *pfdev) { struct drm_sched_init_args args = { .ops = &panfrost_sched_ops, .num_rqs = DRM_SCHED_PRIORITY_COUNT, .credit_limit = 2, .timeout = msecs_to_jiffies(JOB_TIMEOUT_MS), - .name = "pan_js", - .dev = pfdev->dev, + .dev = pfdev->base.dev, }; struct panfrost_job_slot *js; int ret, j; + BUILD_BUG_ON(ARRAY_SIZE(panfrost_engine_names) != NUM_JOB_SLOTS); + /* All GPUs have two entries per queue, but without jobchain * disambiguation stopping the right job in the close path is tricky, * so let's just advertise one entry in that case. @@ -854,24 +867,25 @@ int panfrost_job_init(struct panfrost_device *pfdev) if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_JOBCHAIN_DISAMBIGUATION)) args.credit_limit = 1; - pfdev->js = js = devm_kzalloc(pfdev->dev, sizeof(*js), GFP_KERNEL); + js = devm_kzalloc(pfdev->base.dev, sizeof(*js), GFP_KERNEL); if (!js) return -ENOMEM; + pfdev->js = js; INIT_WORK(&pfdev->reset.work, panfrost_reset_work); spin_lock_init(&js->job_lock); - js->irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "job"); + js->irq = platform_get_irq_byname(to_platform_device(pfdev->base.dev), "job"); if (js->irq < 0) return js->irq; - ret = devm_request_threaded_irq(pfdev->dev, js->irq, - panfrost_job_irq_handler, - panfrost_job_irq_handler_thread, + ret = devm_request_threaded_irq(pfdev->base.dev, js->irq, + panfrost_jm_irq_handler, + panfrost_jm_irq_handler_thread, IRQF_SHARED, KBUILD_MODNAME "-job", pfdev); if (ret) { - dev_err(pfdev->dev, "failed to request job irq"); + dev_err(pfdev->base.dev, "failed to request job irq"); return ret; } @@ -882,15 +896,17 @@ int panfrost_job_init(struct panfrost_device *pfdev) for (j = 0; j < NUM_JOB_SLOTS; j++) { js->queue[j].fence_context = dma_fence_context_alloc(1); + args.name = panfrost_engine_names[j]; ret = drm_sched_init(&js->queue[j].sched, &args); if (ret) { - dev_err(pfdev->dev, "Failed to create scheduler: %d.", ret); + dev_err(pfdev->base.dev, "Failed to create scheduler: %d.", ret); goto err_sched; } } - panfrost_job_enable_interrupts(pfdev); + panfrost_jm_reset_interrupts(pfdev); + panfrost_jm_enable_interrupts(pfdev); return 0; @@ -902,7 +918,7 @@ err_sched: return ret; } -void panfrost_job_fini(struct panfrost_device *pfdev) +void panfrost_jm_fini(struct panfrost_device *pfdev) { struct panfrost_job_slot *js = pfdev->js; int j; @@ -917,39 +933,176 @@ void panfrost_job_fini(struct panfrost_device *pfdev) destroy_workqueue(pfdev->reset.wq); } -int panfrost_job_open(struct panfrost_file_priv *panfrost_priv) +int panfrost_jm_open(struct drm_file *file) +{ + struct panfrost_file_priv *panfrost_priv = file->driver_priv; + int ret; + + struct drm_panfrost_jm_ctx_create default_jm_ctx = { + .priority = PANFROST_JM_CTX_PRIORITY_MEDIUM, + }; + + xa_init_flags(&panfrost_priv->jm_ctxs, XA_FLAGS_ALLOC); + + ret = panfrost_jm_ctx_create(file, &default_jm_ctx); + if (ret) + return ret; + + /* We expect the default context to be assigned handle 0. */ + if (WARN_ON(default_jm_ctx.handle)) + return -EINVAL; + + return 0; +} + +void panfrost_jm_close(struct drm_file *file) +{ + struct panfrost_file_priv *panfrost_priv = file->driver_priv; + struct panfrost_jm_ctx *jm_ctx; + unsigned long i; + + xa_for_each(&panfrost_priv->jm_ctxs, i, jm_ctx) + panfrost_jm_ctx_destroy(file, i); + + xa_destroy(&panfrost_priv->jm_ctxs); +} + +int panfrost_jm_is_idle(struct panfrost_device *pfdev) { - struct panfrost_device *pfdev = panfrost_priv->pfdev; struct panfrost_job_slot *js = pfdev->js; - struct drm_gpu_scheduler *sched; - int ret, i; + int i; for (i = 0; i < NUM_JOB_SLOTS; i++) { - sched = &js->queue[i].sched; - ret = drm_sched_entity_init(&panfrost_priv->sched_entity[i], - DRM_SCHED_PRIORITY_NORMAL, &sched, - 1, NULL); - if (WARN_ON(ret)) - return ret; + /* If there are any jobs in the HW queue, we're not idle */ + if (atomic_read(&js->queue[i].sched.credit_count)) + return false; + } + + return true; +} + +static void panfrost_jm_ctx_release(struct kref *kref) +{ + struct panfrost_jm_ctx *jm_ctx = container_of(kref, struct panfrost_jm_ctx, refcnt); + + WARN_ON(!jm_ctx->destroyed); + + for (u32 i = 0; i < ARRAY_SIZE(jm_ctx->slot_entity); i++) + drm_sched_entity_destroy(&jm_ctx->slot_entity[i]); + + kfree(jm_ctx); +} + +void +panfrost_jm_ctx_put(struct panfrost_jm_ctx *jm_ctx) +{ + if (jm_ctx) + kref_put(&jm_ctx->refcnt, panfrost_jm_ctx_release); +} + +struct panfrost_jm_ctx * +panfrost_jm_ctx_get(struct panfrost_jm_ctx *jm_ctx) +{ + if (jm_ctx) + kref_get(&jm_ctx->refcnt); + + return jm_ctx; +} + +struct panfrost_jm_ctx * +panfrost_jm_ctx_from_handle(struct drm_file *file, u32 handle) +{ + struct panfrost_file_priv *priv = file->driver_priv; + struct panfrost_jm_ctx *jm_ctx; + + xa_lock(&priv->jm_ctxs); + jm_ctx = panfrost_jm_ctx_get(xa_load(&priv->jm_ctxs, handle)); + xa_unlock(&priv->jm_ctxs); + + return jm_ctx; +} + +static int jm_ctx_prio_to_drm_sched_prio(struct drm_file *file, + enum drm_panfrost_jm_ctx_priority in, + enum drm_sched_priority *out) +{ + switch (in) { + case PANFROST_JM_CTX_PRIORITY_LOW: + *out = DRM_SCHED_PRIORITY_LOW; + return 0; + case PANFROST_JM_CTX_PRIORITY_MEDIUM: + *out = DRM_SCHED_PRIORITY_NORMAL; + return 0; + case PANFROST_JM_CTX_PRIORITY_HIGH: + if (!panfrost_high_prio_allowed(file)) + return -EACCES; + + *out = DRM_SCHED_PRIORITY_HIGH; + return 0; + default: + return -EINVAL; + } +} + +int panfrost_jm_ctx_create(struct drm_file *file, + struct drm_panfrost_jm_ctx_create *args) +{ + struct panfrost_file_priv *priv = file->driver_priv; + struct panfrost_device *pfdev = priv->pfdev; + enum drm_sched_priority sched_prio; + struct panfrost_jm_ctx *jm_ctx; + int ret; + + jm_ctx = kzalloc(sizeof(*jm_ctx), GFP_KERNEL); + if (!jm_ctx) + return -ENOMEM; + + kref_init(&jm_ctx->refcnt); + + ret = jm_ctx_prio_to_drm_sched_prio(file, args->priority, &sched_prio); + if (ret) + goto err_put_jm_ctx; + + for (u32 i = 0; i < NUM_JOB_SLOTS; i++) { + struct drm_gpu_scheduler *sched = &pfdev->js->queue[i].sched; + + ret = drm_sched_entity_init(&jm_ctx->slot_entity[i], sched_prio, + &sched, 1, NULL); + if (ret) + goto err_put_jm_ctx; } + + ret = xa_alloc(&priv->jm_ctxs, &args->handle, jm_ctx, + XA_LIMIT(0, MAX_JM_CTX_PER_FILE), GFP_KERNEL); + if (ret) + goto err_put_jm_ctx; + return 0; + +err_put_jm_ctx: + jm_ctx->destroyed = true; + panfrost_jm_ctx_put(jm_ctx); + return ret; } -void panfrost_job_close(struct panfrost_file_priv *panfrost_priv) +int panfrost_jm_ctx_destroy(struct drm_file *file, u32 handle) { - struct panfrost_device *pfdev = panfrost_priv->pfdev; - int i; + struct panfrost_file_priv *priv = file->driver_priv; + struct panfrost_device *pfdev = priv->pfdev; + struct panfrost_jm_ctx *jm_ctx; - for (i = 0; i < NUM_JOB_SLOTS; i++) - drm_sched_entity_destroy(&panfrost_priv->sched_entity[i]); + jm_ctx = xa_erase(&priv->jm_ctxs, handle); + if (!jm_ctx) + return -EINVAL; + + jm_ctx->destroyed = true; /* Kill in-flight jobs */ spin_lock(&pfdev->js->job_lock); - for (i = 0; i < NUM_JOB_SLOTS; i++) { - struct drm_sched_entity *entity = &panfrost_priv->sched_entity[i]; - int j; + for (u32 i = 0; i < ARRAY_SIZE(jm_ctx->slot_entity); i++) { + struct drm_sched_entity *entity = &jm_ctx->slot_entity[i]; - for (j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) { + for (int j = ARRAY_SIZE(pfdev->jobs[0]) - 1; j >= 0; j--) { struct panfrost_job *job = pfdev->jobs[i][j]; u32 cmd; @@ -980,18 +1133,7 @@ void panfrost_job_close(struct panfrost_file_priv *panfrost_priv) } } spin_unlock(&pfdev->js->job_lock); -} - -int panfrost_job_is_idle(struct panfrost_device *pfdev) -{ - struct panfrost_job_slot *js = pfdev->js; - int i; - - for (i = 0; i < NUM_JOB_SLOTS; i++) { - /* If there are any jobs in the HW queue, we're not idle */ - if (atomic_read(&js->queue[i].sched.credit_count)) - return false; - } - return true; + panfrost_jm_ctx_put(jm_ctx); + return 0; } diff --git a/drivers/gpu/drm/panfrost/panfrost_job.h b/drivers/gpu/drm/panfrost/panfrost_job.h index ec581b97852b..c3f57e41a571 100644 --- a/drivers/gpu/drm/panfrost/panfrost_job.h +++ b/drivers/gpu/drm/panfrost/panfrost_job.h @@ -18,6 +18,7 @@ struct panfrost_job { struct panfrost_device *pfdev; struct panfrost_mmu *mmu; + struct panfrost_jm_ctx *ctx; /* Fence to be signaled by IRQ handler when the job is complete. */ struct dma_fence *done_fence; @@ -39,15 +40,38 @@ struct panfrost_job { u64 start_cycles; }; -int panfrost_job_init(struct panfrost_device *pfdev); -void panfrost_job_fini(struct panfrost_device *pfdev); -int panfrost_job_open(struct panfrost_file_priv *panfrost_priv); -void panfrost_job_close(struct panfrost_file_priv *panfrost_priv); +struct panfrost_js_ctx { + struct drm_sched_entity sched_entity; + bool enabled; +}; + +#define NUM_JOB_SLOTS 3 + +struct panfrost_jm_ctx { + struct kref refcnt; + bool destroyed; + struct drm_sched_entity slot_entity[NUM_JOB_SLOTS]; +}; + +extern const char * const panfrost_engine_names[]; + +int panfrost_jm_ctx_create(struct drm_file *file, + struct drm_panfrost_jm_ctx_create *args); +int panfrost_jm_ctx_destroy(struct drm_file *file, u32 handle); +void panfrost_jm_ctx_put(struct panfrost_jm_ctx *jm_ctx); +struct panfrost_jm_ctx *panfrost_jm_ctx_get(struct panfrost_jm_ctx *jm_ctx); +struct panfrost_jm_ctx *panfrost_jm_ctx_from_handle(struct drm_file *file, u32 handle); + +int panfrost_jm_init(struct panfrost_device *pfdev); +void panfrost_jm_fini(struct panfrost_device *pfdev); +int panfrost_jm_open(struct drm_file *file); +void panfrost_jm_close(struct drm_file *file); +void panfrost_jm_reset_interrupts(struct panfrost_device *pfdev); +void panfrost_jm_enable_interrupts(struct panfrost_device *pfdev); +void panfrost_jm_suspend_irq(struct panfrost_device *pfdev); +int panfrost_jm_is_idle(struct panfrost_device *pfdev); int panfrost_job_get_slot(struct panfrost_job *job); int panfrost_job_push(struct panfrost_job *job); void panfrost_job_put(struct panfrost_job *job); -void panfrost_job_enable_interrupts(struct panfrost_device *pfdev); -void panfrost_job_suspend_irq(struct panfrost_device *pfdev); -int panfrost_job_is_idle(struct panfrost_device *pfdev); #endif diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.c b/drivers/gpu/drm/panfrost/panfrost_mmu.c index f6b91c052cfb..8f3b7a7b6ad0 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.c +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.c @@ -2,6 +2,7 @@ /* Copyright 2019 Linaro, Ltd, Rob Herring <robh@kernel.org> */ #include <drm/panfrost_drm.h> +#include <drm/drm_print.h> #include <linux/atomic.h> #include <linux/bitfield.h> @@ -81,7 +82,7 @@ static int wait_ready(struct panfrost_device *pfdev, u32 as_nr) if (ret) { /* The GPU hung, let's trigger a reset */ panfrost_device_schedule_reset(pfdev); - dev_err(pfdev->dev, "AS_ACTIVE bit stuck\n"); + dev_err(pfdev->base.dev, "AS_ACTIVE bit stuck\n"); } return ret; @@ -222,7 +223,7 @@ static int mmu_cfg_init_aarch64_4k(struct panfrost_mmu *mmu) struct io_pgtable_cfg *pgtbl_cfg = &mmu->pgtbl_cfg; struct panfrost_device *pfdev = mmu->pfdev; - if (drm_WARN_ON(pfdev->ddev, pgtbl_cfg->arm_lpae_s1_cfg.ttbr & + if (drm_WARN_ON(&pfdev->base, pgtbl_cfg->arm_lpae_s1_cfg.ttbr & ~AS_TRANSTAB_AARCH64_4K_ADDR_MASK)) return -EINVAL; @@ -253,12 +254,12 @@ static int panfrost_mmu_cfg_init(struct panfrost_mmu *mmu, return mmu_cfg_init_mali_lpae(mmu); default: /* This should never happen */ - drm_WARN(pfdev->ddev, 1, "Invalid pgtable format"); + drm_WARN(&pfdev->base, 1, "Invalid pgtable format"); return -EINVAL; } } -u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) +int panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) { int as; @@ -300,7 +301,10 @@ u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) if (!atomic_read(&lru_mmu->as_count)) break; } - WARN_ON(&lru_mmu->list == &pfdev->as_lru_list); + if (WARN_ON(&lru_mmu->list == &pfdev->as_lru_list)) { + as = -EBUSY; + goto out; + } list_del_init(&lru_mmu->list); as = lru_mmu->as; @@ -315,7 +319,9 @@ u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu) atomic_set(&mmu->as_count, 1); list_add(&mmu->list, &pfdev->as_lru_list); - dev_dbg(pfdev->dev, "Assigned AS%d to mmu %p, alloc_mask=%lx", as, mmu, pfdev->as_alloc_mask); + dev_dbg(pfdev->base.dev, + "Assigned AS%d to mmu %p, alloc_mask=%lx", + as, mmu, pfdev->as_alloc_mask); panfrost_mmu_enable(pfdev, mmu); @@ -381,13 +387,30 @@ static void panfrost_mmu_flush_range(struct panfrost_device *pfdev, if (mmu->as < 0) return; - pm_runtime_get_noresume(pfdev->dev); + pm_runtime_get_noresume(pfdev->base.dev); /* Flush the PTs only if we're already awake */ - if (pm_runtime_active(pfdev->dev)) + if (pm_runtime_active(pfdev->base.dev)) mmu_hw_do_operation(pfdev, mmu, iova, size, AS_COMMAND_FLUSH_PT); - pm_runtime_put_autosuspend(pfdev->dev); + pm_runtime_put_autosuspend(pfdev->base.dev); +} + +static void mmu_unmap_range(struct panfrost_mmu *mmu, u64 iova, size_t len) +{ + struct io_pgtable_ops *ops = mmu->pgtbl_ops; + size_t pgsize, unmapped_len = 0; + size_t unmapped_page, pgcount; + + while (unmapped_len < len) { + pgsize = get_pgsize(iova, len - unmapped_len, &pgcount); + + unmapped_page = ops->unmap_pages(ops, iova, pgsize, pgcount, NULL); + WARN_ON(unmapped_page != pgsize * pgcount); + + iova += pgsize * pgcount; + unmapped_len += pgsize * pgcount; + } } static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, @@ -396,22 +419,30 @@ static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, unsigned int count; struct scatterlist *sgl; struct io_pgtable_ops *ops = mmu->pgtbl_ops; + size_t total_mapped = 0; u64 start_iova = iova; + int ret; for_each_sgtable_dma_sg(sgt, sgl, count) { unsigned long paddr = sg_dma_address(sgl); size_t len = sg_dma_len(sgl); - dev_dbg(pfdev->dev, "map: as=%d, iova=%llx, paddr=%lx, len=%zx", mmu->as, iova, paddr, len); + dev_dbg(pfdev->base.dev, + "map: as=%d, iova=%llx, paddr=%lx, len=%zx", + mmu->as, iova, paddr, len); while (len) { size_t pgcount, mapped = 0; size_t pgsize = get_pgsize(iova | paddr, len, &pgcount); - ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, + ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, GFP_KERNEL, &mapped); + if (ret) + goto err_unmap_pages; + /* Don't get stuck if things have gone wrong */ mapped = max(mapped, pgsize); + total_mapped += mapped; iova += mapped; paddr += mapped; len -= mapped; @@ -421,6 +452,10 @@ static int mmu_map_sg(struct panfrost_device *pfdev, struct panfrost_mmu *mmu, panfrost_mmu_flush_range(pfdev, mmu, start_iova, iova - start_iova); return 0; + +err_unmap_pages: + mmu_unmap_range(mmu, start_iova, total_mapped); + return ret; } int panfrost_mmu_map(struct panfrost_gem_mapping *mapping) @@ -431,6 +466,7 @@ int panfrost_mmu_map(struct panfrost_gem_mapping *mapping) struct panfrost_device *pfdev = to_panfrost_device(obj->dev); struct sg_table *sgt; int prot = IOMMU_READ | IOMMU_WRITE | IOMMU_CACHE; + int ret; if (WARN_ON(mapping->active)) return 0; @@ -442,11 +478,18 @@ int panfrost_mmu_map(struct panfrost_gem_mapping *mapping) if (WARN_ON(IS_ERR(sgt))) return PTR_ERR(sgt); - mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT, - prot, sgt); + ret = mmu_map_sg(pfdev, mapping->mmu, mapping->mmnode.start << PAGE_SHIFT, + prot, sgt); + if (ret) + goto err_put_pages; + mapping->active = true; return 0; + +err_put_pages: + drm_gem_shmem_put_pages_locked(shmem); + return ret; } void panfrost_mmu_unmap(struct panfrost_gem_mapping *mapping) @@ -462,7 +505,7 @@ void panfrost_mmu_unmap(struct panfrost_gem_mapping *mapping) if (WARN_ON(!mapping->active)) return; - dev_dbg(pfdev->dev, "unmap: as=%d, iova=%llx, len=%zx", + dev_dbg(pfdev->base.dev, "unmap: as=%d, iova=%llx, len=%zx", mapping->mmu->as, iova, len); while (unmapped_len < len) { @@ -559,7 +602,7 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, bo = bomapping->obj; if (!bo->is_heap) { - dev_WARN(pfdev->dev, "matching BO is not heap type (GPU VA = %llx)", + dev_WARN(pfdev->base.dev, "matching BO is not heap type (GPU VA = %llx)", bomapping->mmnode.start << PAGE_SHIFT); ret = -EINVAL; goto err_bo; @@ -595,10 +638,12 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, refcount_set(&bo->base.pages_use_count, 1); } else { pages = bo->base.pages; - if (pages[page_offset]) { - /* Pages are already mapped, bail out. */ - goto out; - } + } + + sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)]; + if (sgt->sgl) { + /* Pages are already mapped, bail out. */ + goto out; } mapping = bo->base.base.filp->f_mapping; @@ -620,23 +665,24 @@ static int panfrost_mmu_map_fault_addr(struct panfrost_device *pfdev, int as, } } - sgt = &bo->sgts[page_offset / (SZ_2M / PAGE_SIZE)]; ret = sg_alloc_table_from_pages(sgt, pages + page_offset, NUM_FAULT_PAGES, 0, SZ_2M, GFP_KERNEL); if (ret) goto err_unlock; - ret = dma_map_sgtable(pfdev->dev, sgt, DMA_BIDIRECTIONAL, 0); + ret = dma_map_sgtable(pfdev->base.dev, sgt, DMA_BIDIRECTIONAL, 0); if (ret) goto err_map; - mmu_map_sg(pfdev, bomapping->mmu, addr, - IOMMU_WRITE | IOMMU_READ | IOMMU_CACHE | IOMMU_NOEXEC, sgt); + ret = mmu_map_sg(pfdev, bomapping->mmu, addr, + IOMMU_WRITE | IOMMU_READ | IOMMU_CACHE | IOMMU_NOEXEC, sgt); + if (ret) + goto err_mmu_map_sg; bomapping->active = true; bo->heap_rss_size += SZ_2M; - dev_dbg(pfdev->dev, "mapped page fault @ AS%d %llx", as, addr); + dev_dbg(pfdev->base.dev, "mapped page fault @ AS%d %llx", as, addr); out: dma_resv_unlock(obj->resv); @@ -645,6 +691,8 @@ out: return 0; +err_mmu_map_sg: + dma_unmap_sgtable(pfdev->base.dev, sgt, DMA_BIDIRECTIONAL, 0); err_map: sg_free_table(sgt); err_unlock: @@ -662,13 +710,12 @@ static void panfrost_mmu_release_ctx(struct kref *kref) spin_lock(&pfdev->as_lock); if (mmu->as >= 0) { - pm_runtime_get_noresume(pfdev->dev); - if (pm_runtime_active(pfdev->dev)) + pm_runtime_get_noresume(pfdev->base.dev); + if (pm_runtime_active(pfdev->base.dev)) panfrost_mmu_disable(pfdev, mmu->as); - pm_runtime_put_autosuspend(pfdev->dev); + pm_runtime_put_autosuspend(pfdev->base.dev); clear_bit(mmu->as, &pfdev->as_alloc_mask); - clear_bit(mmu->as, &pfdev->as_in_use_mask); list_del(&mmu->list); } spin_unlock(&pfdev->as_lock); @@ -726,7 +773,7 @@ struct panfrost_mmu *panfrost_mmu_ctx_create(struct panfrost_device *pfdev) if (pfdev->comp->gpu_quirks & BIT(GPU_QUIRK_FORCE_AARCH64_PGTABLE)) { if (!panfrost_has_hw_feature(pfdev, HW_FEATURE_AARCH64_MMU)) { - dev_err_once(pfdev->dev, + dev_err_once(pfdev->base.dev, "AARCH64_4K page table not supported\n"); return ERR_PTR(-EINVAL); } @@ -755,7 +802,7 @@ struct panfrost_mmu *panfrost_mmu_ctx_create(struct panfrost_device *pfdev) .oas = pa_bits, .coherent_walk = pfdev->coherent, .tlb = &mmu_tlb_ops, - .iommu_dev = pfdev->dev, + .iommu_dev = pfdev->base.dev, }; mmu->pgtbl_ops = alloc_io_pgtable_ops(fmt, &mmu->pgtbl_cfg, mmu); @@ -848,7 +895,7 @@ static irqreturn_t panfrost_mmu_irq_handler_thread(int irq, void *data) if (ret) { /* terminal fault, print info about the fault */ - dev_err(pfdev->dev, + dev_err(pfdev->base.dev, "Unhandled Page fault in AS%d at VA 0x%016llX\n" "Reason: %s\n" "raw fault status: 0x%X\n" @@ -896,18 +943,18 @@ int panfrost_mmu_init(struct panfrost_device *pfdev) { int err; - pfdev->mmu_irq = platform_get_irq_byname(to_platform_device(pfdev->dev), "mmu"); + pfdev->mmu_irq = platform_get_irq_byname(to_platform_device(pfdev->base.dev), "mmu"); if (pfdev->mmu_irq < 0) return pfdev->mmu_irq; - err = devm_request_threaded_irq(pfdev->dev, pfdev->mmu_irq, + err = devm_request_threaded_irq(pfdev->base.dev, pfdev->mmu_irq, panfrost_mmu_irq_handler, panfrost_mmu_irq_handler_thread, IRQF_SHARED, KBUILD_MODNAME "-mmu", pfdev); if (err) { - dev_err(pfdev->dev, "failed to request mmu irq"); + dev_err(pfdev->base.dev, "failed to request mmu irq"); return err; } diff --git a/drivers/gpu/drm/panfrost/panfrost_mmu.h b/drivers/gpu/drm/panfrost/panfrost_mmu.h index 022a9a74a114..27c3c65ed074 100644 --- a/drivers/gpu/drm/panfrost/panfrost_mmu.h +++ b/drivers/gpu/drm/panfrost/panfrost_mmu.h @@ -4,6 +4,7 @@ #ifndef __PANFROST_MMU_H__ #define __PANFROST_MMU_H__ +struct panfrost_device; struct panfrost_gem_mapping; struct panfrost_file_priv; struct panfrost_mmu; @@ -16,7 +17,7 @@ void panfrost_mmu_fini(struct panfrost_device *pfdev); void panfrost_mmu_reset(struct panfrost_device *pfdev); void panfrost_mmu_suspend_irq(struct panfrost_device *pfdev); -u32 panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu); +int panfrost_mmu_as_get(struct panfrost_device *pfdev, struct panfrost_mmu *mmu); void panfrost_mmu_as_put(struct panfrost_device *pfdev, struct panfrost_mmu *mmu); struct panfrost_mmu *panfrost_mmu_ctx_get(struct panfrost_mmu *mmu); diff --git a/drivers/gpu/drm/panfrost/panfrost_perfcnt.c b/drivers/gpu/drm/panfrost/panfrost_perfcnt.c index 0dd62e8b2fa7..7020c0192e18 100644 --- a/drivers/gpu/drm/panfrost/panfrost_perfcnt.c +++ b/drivers/gpu/drm/panfrost/panfrost_perfcnt.c @@ -84,11 +84,11 @@ static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev, else if (perfcnt->user) return -EBUSY; - ret = pm_runtime_get_sync(pfdev->dev); + ret = pm_runtime_get_sync(pfdev->base.dev); if (ret < 0) goto err_put_pm; - bo = drm_gem_shmem_create(pfdev->ddev, perfcnt->bosize); + bo = drm_gem_shmem_create(&pfdev->base, perfcnt->bosize); if (IS_ERR(bo)) { ret = PTR_ERR(bo); goto err_put_pm; @@ -130,9 +130,11 @@ static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev, goto err_vunmap; } - perfcnt->user = user; + ret = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu); + if (ret < 0) + goto err_vunmap; - as = panfrost_mmu_as_get(pfdev, perfcnt->mapping->mmu); + as = ret; cfg = GPU_PERFCNT_CFG_AS(as) | GPU_PERFCNT_CFG_MODE(GPU_PERFCNT_CFG_MODE_MANUAL); @@ -164,6 +166,8 @@ static int panfrost_perfcnt_enable_locked(struct panfrost_device *pfdev, /* The BO ref is retained by the mapping. */ drm_gem_object_put(&bo->base); + perfcnt->user = user; + return 0; err_vunmap: @@ -175,7 +179,7 @@ err_close_bo: err_put_bo: drm_gem_object_put(&bo->base); err_put_pm: - pm_runtime_put(pfdev->dev); + pm_runtime_put(pfdev->base.dev); return ret; } @@ -203,7 +207,7 @@ static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev, panfrost_mmu_as_put(pfdev, perfcnt->mapping->mmu); panfrost_gem_mapping_put(perfcnt->mapping); perfcnt->mapping = NULL; - pm_runtime_put_autosuspend(pfdev->dev); + pm_runtime_put_autosuspend(pfdev->base.dev); return 0; } @@ -211,7 +215,7 @@ static int panfrost_perfcnt_disable_locked(struct panfrost_device *pfdev, int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(dev); struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; struct drm_panfrost_perfcnt_enable *req = data; int ret; @@ -238,7 +242,7 @@ int panfrost_ioctl_perfcnt_enable(struct drm_device *dev, void *data, int panfrost_ioctl_perfcnt_dump(struct drm_device *dev, void *data, struct drm_file *file_priv) { - struct panfrost_device *pfdev = dev->dev_private; + struct panfrost_device *pfdev = to_panfrost_device(dev); struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; struct drm_panfrost_perfcnt_dump *req = data; void __user *user_ptr = (void __user *)(uintptr_t)req->buf_ptr; @@ -273,12 +277,12 @@ void panfrost_perfcnt_close(struct drm_file *file_priv) struct panfrost_device *pfdev = pfile->pfdev; struct panfrost_perfcnt *perfcnt = pfdev->perfcnt; - pm_runtime_get_sync(pfdev->dev); + pm_runtime_get_sync(pfdev->base.dev); mutex_lock(&perfcnt->lock); if (perfcnt->user == pfile) panfrost_perfcnt_disable_locked(pfdev, file_priv); mutex_unlock(&perfcnt->lock); - pm_runtime_put_autosuspend(pfdev->dev); + pm_runtime_put_autosuspend(pfdev->base.dev); } int panfrost_perfcnt_init(struct panfrost_device *pfdev) @@ -316,7 +320,7 @@ int panfrost_perfcnt_init(struct panfrost_device *pfdev) COUNTERS_PER_BLOCK * BYTES_PER_COUNTER; } - perfcnt = devm_kzalloc(pfdev->dev, sizeof(*perfcnt), GFP_KERNEL); + perfcnt = devm_kzalloc(pfdev->base.dev, sizeof(*perfcnt), GFP_KERNEL); if (!perfcnt) return -ENOMEM; |