diff options
Diffstat (limited to 'drivers')
62 files changed, 1696 insertions, 145 deletions
diff --git a/drivers/ata/libata-core.c b/drivers/ata/libata-core.c index f48fb63d7e85..462217935558 100644 --- a/drivers/ata/libata-core.c +++ b/drivers/ata/libata-core.c @@ -4216,6 +4216,10 @@ static const struct ata_dev_quirks_entry __ata_dev_quirks[] = { /* Apacer models with LPM issues */ { "Apacer AS340*", NULL, ATA_QUIRK_NOLPM }, + /* Silicon Motion models with LPM issues */ + { "MD619HXCLDE3TC", "TCVAID", ATA_QUIRK_NOLPM }, + { "MD619GXCLDE3TC", "TCV35D", ATA_QUIRK_NOLPM }, + /* These specific Samsung models/firmware-revs do not handle LPM well */ { "SAMSUNG MZMPC128HBFU-000MV", "CXM14M1Q", ATA_QUIRK_NOLPM }, { "SAMSUNG SSD PM830 mSATA *", "CXM13D1Q", ATA_QUIRK_NOLPM }, diff --git a/drivers/ata/libata-sff.c b/drivers/ata/libata-sff.c index 1e2a2c33cdc8..785b6e371abf 100644 --- a/drivers/ata/libata-sff.c +++ b/drivers/ata/libata-sff.c @@ -3191,7 +3191,8 @@ void ata_sff_port_init(struct ata_port *ap) int __init ata_sff_init(void) { - ata_sff_wq = alloc_workqueue("ata_sff", WQ_MEM_RECLAIM, WQ_MAX_ACTIVE); + ata_sff_wq = alloc_workqueue("ata_sff", WQ_MEM_RECLAIM | WQ_PERCPU, + WQ_MAX_ACTIVE); if (!ata_sff_wq) return -ENOMEM; diff --git a/drivers/ata/pata_it821x.c b/drivers/ata/pata_it821x.c index 042f6ad1f7c6..fc762dcc61bf 100644 --- a/drivers/ata/pata_it821x.c +++ b/drivers/ata/pata_it821x.c @@ -75,6 +75,7 @@ #include <linux/blkdev.h> #include <linux/delay.h> #include <linux/slab.h> +#include <linux/string.h> #include <scsi/scsi_host.h> #include <linux/libata.h> @@ -632,9 +633,9 @@ static void it821x_display_disk(struct ata_port *ap, int n, u8 *buf) cbl = ""; if (mode) - snprintf(mbuf, 8, "%5s%d", mtype, mode - 1); + snprintf(mbuf, sizeof(mbuf), "%5s%d", mtype, mode - 1); else - strcpy(mbuf, "PIO"); + strscpy(mbuf, "PIO"); if (buf[52] == 4) ata_port_info(ap, "%d: %-6s %-8s %s %s\n", n, mbuf, types[buf[52]], id, cbl); diff --git a/drivers/ata/pata_pcmcia.c b/drivers/ata/pata_pcmcia.c index cf3810933a27..caefcd8c4b3c 100644 --- a/drivers/ata/pata_pcmcia.c +++ b/drivers/ata/pata_pcmcia.c @@ -344,6 +344,7 @@ static const struct pcmcia_device_id pcmcia_devices[] = { PCMCIA_DEVICE_PROD_ID2("NinjaATA-", 0xebe0bd79), PCMCIA_DEVICE_PROD_ID12("PCMCIA", "CD-ROM", 0x281f1c5d, 0x66536591), PCMCIA_DEVICE_PROD_ID12("PCMCIA", "PnPIDE", 0x281f1c5d, 0x0c694728), + PCMCIA_DEVICE_PROD_ID2("PCMCIA ATA/ATAPI Adapter", 0x888d7b73), PCMCIA_DEVICE_PROD_ID12("SHUTTLE TECHNOLOGY LTD.", "PCCARD-IDE/ATAPI Adapter", 0x4a3f0ba0, 0x322560e1), PCMCIA_DEVICE_PROD_ID12("SEAGATE", "ST1", 0x87c1b330, 0xe1f30883), PCMCIA_DEVICE_PROD_ID12("SAMSUNG", "04/05/06", 0x43d74cb4, 0x6a22777d), diff --git a/drivers/char/tpm/tpm-chip.c b/drivers/char/tpm/tpm-chip.c index e25daf2396d3..082b910ddf0d 100644 --- a/drivers/char/tpm/tpm-chip.c +++ b/drivers/char/tpm/tpm-chip.c @@ -231,42 +231,6 @@ struct tpm_chip *tpm_default_chip(void) EXPORT_SYMBOL_GPL(tpm_default_chip); /** - * tpm_find_get_ops() - find and reserve a TPM chip - * @chip: a &struct tpm_chip instance, %NULL for the default chip - * - * Finds a TPM chip and reserves its class device and operations. The chip must - * be released with tpm_put_ops() after use. - * This function is for internal use only. It supports existing TPM callers - * by accepting NULL, but those callers should be converted to pass in a chip - * directly. - * - * Return: - * A reserved &struct tpm_chip instance. - * %NULL if a chip is not found. - * %NULL if the chip is not available. - */ -struct tpm_chip *tpm_find_get_ops(struct tpm_chip *chip) -{ - int rc; - - if (chip) { - if (!tpm_try_get_ops(chip)) - return chip; - return NULL; - } - - chip = tpm_default_chip(); - if (!chip) - return NULL; - rc = tpm_try_get_ops(chip); - /* release additional reference we got from tpm_default_chip() */ - put_device(&chip->dev); - if (rc) - return NULL; - return chip; -} - -/** * tpm_dev_release() - free chip memory and the device number * @dev: the character device for the TPM chip * @@ -282,7 +246,6 @@ static void tpm_dev_release(struct device *dev) kfree(chip->work_space.context_buf); kfree(chip->work_space.session_buf); - kfree(chip->allocated_banks); #ifdef CONFIG_TCG_TPM2_HMAC kfree(chip->auth); #endif diff --git a/drivers/char/tpm/tpm-dev-common.c b/drivers/char/tpm/tpm-dev-common.c index f2a5e09257dd..f942c0c8e402 100644 --- a/drivers/char/tpm/tpm-dev-common.c +++ b/drivers/char/tpm/tpm-dev-common.c @@ -275,7 +275,8 @@ void tpm_common_release(struct file *file, struct file_priv *priv) int __init tpm_dev_common_init(void) { - tpm_dev_wq = alloc_workqueue("tpm_dev_wq", WQ_MEM_RECLAIM, 0); + tpm_dev_wq = alloc_workqueue("tpm_dev_wq", WQ_MEM_RECLAIM | WQ_PERCPU, + 0); return !tpm_dev_wq ? -ENOMEM : 0; } diff --git a/drivers/char/tpm/tpm-interface.c b/drivers/char/tpm/tpm-interface.c index c9f173001d0e..f745a098908b 100644 --- a/drivers/char/tpm/tpm-interface.c +++ b/drivers/char/tpm/tpm-interface.c @@ -313,10 +313,13 @@ int tpm_is_tpm2(struct tpm_chip *chip) { int rc; - chip = tpm_find_get_ops(chip); if (!chip) return -ENODEV; + rc = tpm_try_get_ops(chip); + if (rc) + return rc; + rc = (chip->flags & TPM_CHIP_FLAG_TPM2) != 0; tpm_put_ops(chip); @@ -338,10 +341,13 @@ int tpm_pcr_read(struct tpm_chip *chip, u32 pcr_idx, { int rc; - chip = tpm_find_get_ops(chip); if (!chip) return -ENODEV; + rc = tpm_try_get_ops(chip); + if (rc) + return rc; + if (chip->flags & TPM_CHIP_FLAG_TPM2) rc = tpm2_pcr_read(chip, pcr_idx, digest, NULL); else @@ -369,10 +375,13 @@ int tpm_pcr_extend(struct tpm_chip *chip, u32 pcr_idx, int rc; int i; - chip = tpm_find_get_ops(chip); if (!chip) return -ENODEV; + rc = tpm_try_get_ops(chip); + if (rc) + return rc; + for (i = 0; i < chip->nr_allocated_banks; i++) { if (digests[i].alg_id != chip->allocated_banks[i].alg_id) { rc = -EINVAL; @@ -492,10 +501,13 @@ int tpm_get_random(struct tpm_chip *chip, u8 *out, size_t max) if (!out || max > TPM_MAX_RNG_DATA) return -EINVAL; - chip = tpm_find_get_ops(chip); if (!chip) return -ENODEV; + rc = tpm_try_get_ops(chip); + if (rc) + return rc; + if (chip->flags & TPM_CHIP_FLAG_TPM2) rc = tpm2_get_random(chip, out, max); else diff --git a/drivers/char/tpm/tpm.h b/drivers/char/tpm/tpm.h index 2726bd38e5ac..02c07fef41ba 100644 --- a/drivers/char/tpm/tpm.h +++ b/drivers/char/tpm/tpm.h @@ -267,7 +267,6 @@ static inline void tpm_msleep(unsigned int delay_msec) int tpm_chip_bootstrap(struct tpm_chip *chip); int tpm_chip_start(struct tpm_chip *chip); void tpm_chip_stop(struct tpm_chip *chip); -struct tpm_chip *tpm_find_get_ops(struct tpm_chip *chip); struct tpm_chip *tpm_chip_alloc(struct device *dev, const struct tpm_class_ops *ops); diff --git a/drivers/char/tpm/tpm1-cmd.c b/drivers/char/tpm/tpm1-cmd.c index cf64c7385105..b49a790f1bd5 100644 --- a/drivers/char/tpm/tpm1-cmd.c +++ b/drivers/char/tpm/tpm1-cmd.c @@ -799,11 +799,6 @@ int tpm1_pm_suspend(struct tpm_chip *chip, u32 tpm_suspend_pcr) */ int tpm1_get_pcr_allocation(struct tpm_chip *chip) { - chip->allocated_banks = kcalloc(1, sizeof(*chip->allocated_banks), - GFP_KERNEL); - if (!chip->allocated_banks) - return -ENOMEM; - chip->allocated_banks[0].alg_id = TPM_ALG_SHA1; chip->allocated_banks[0].digest_size = hash_digest_size[HASH_ALGO_SHA1]; chip->allocated_banks[0].crypto_id = HASH_ALGO_SHA1; diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index 5532e53a2dd3..dd502322f499 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -550,11 +550,9 @@ ssize_t tpm2_get_pcr_allocation(struct tpm_chip *chip) nr_possible_banks = be32_to_cpup( (__be32 *)&buf.data[TPM_HEADER_SIZE + 5]); - - chip->allocated_banks = kcalloc(nr_possible_banks, - sizeof(*chip->allocated_banks), - GFP_KERNEL); - if (!chip->allocated_banks) { + if (nr_possible_banks > TPM2_MAX_PCR_BANKS) { + pr_err("tpm: out of bank capacity: %u > %u\n", + nr_possible_banks, TPM2_MAX_PCR_BANKS); rc = -ENOMEM; goto out; } diff --git a/drivers/char/tpm/tpm_crb.c b/drivers/char/tpm/tpm_crb.c index c75a531cfb98..6c25305c256e 100644 --- a/drivers/char/tpm/tpm_crb.c +++ b/drivers/char/tpm/tpm_crb.c @@ -179,6 +179,7 @@ static int crb_try_pluton_doorbell(struct crb_priv *priv, bool wait_for_complete * * @dev: crb device * @priv: crb private data + * @loc: locality * * Write CRB_CTRL_REQ_GO_IDLE to TPM_CRB_CTRL_REQ * The device should respond within TIMEOUT_C by clearing the bit. @@ -233,6 +234,7 @@ static int crb_go_idle(struct tpm_chip *chip) * * @dev: crb device * @priv: crb private data + * @loc: locality * * Write CRB_CTRL_REQ_CMD_READY to TPM_CRB_CTRL_REQ * and poll till the device acknowledge it by clearing the bit. @@ -412,7 +414,7 @@ static int crb_do_acpi_start(struct tpm_chip *chip) #ifdef CONFIG_ARM64 /* * This is a TPM Command Response Buffer start method that invokes a - * Secure Monitor Call to requrest the firmware to execute or cancel + * Secure Monitor Call to request the firmware to execute or cancel * a TPM 2.0 command. */ static int tpm_crb_smc_start(struct device *dev, unsigned long func_id) diff --git a/drivers/char/tpm/tpm_tis_core.c b/drivers/char/tpm/tpm_tis_core.c index 8954a8660ffc..e2a1769081b1 100644 --- a/drivers/char/tpm/tpm_tis_core.c +++ b/drivers/char/tpm/tpm_tis_core.c @@ -265,8 +265,7 @@ static u8 tpm_tis_status(struct tpm_chip *chip) /* * Dump stack for forensics, as invalid TPM_STS.x could be - * potentially triggered by impaired tpm_try_get_ops() or - * tpm_find_get_ops(). + * potentially triggered by impaired tpm_try_get_ops(). */ dump_stack(); } diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile index 7c89e5e0a277..4db24050edb0 100644 --- a/drivers/gpu/drm/i915/Makefile +++ b/drivers/gpu/drm/i915/Makefile @@ -239,6 +239,8 @@ i915-y += \ display/intel_cdclk.o \ display/intel_cmtg.o \ display/intel_color.o \ + display/intel_colorop.o \ + display/intel_color_pipeline.o \ display/intel_combo_phy.o \ display/intel_connector.o \ display/intel_crtc.o \ diff --git a/drivers/gpu/drm/i915/display/intel_color.c b/drivers/gpu/drm/i915/display/intel_color.c index a217a67ceb43..e7950655434b 100644 --- a/drivers/gpu/drm/i915/display/intel_color.c +++ b/drivers/gpu/drm/i915/display/intel_color.c @@ -32,6 +32,8 @@ #include "intel_display_utils.h" #include "intel_dsb.h" #include "intel_vrr.h" +#include "skl_universal_plane.h" +#include "skl_universal_plane_regs.h" struct intel_color_funcs { int (*color_check)(struct intel_atomic_state *state, @@ -87,6 +89,14 @@ struct intel_color_funcs { * Read config other than LUTs and CSCs, before them. Optional. */ void (*get_config)(struct intel_crtc_state *crtc_state); + + /* Plane CSC*/ + void (*load_plane_csc_matrix)(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state); + + /* Plane Pre/Post CSC */ + void (*load_plane_luts)(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state); }; #define CTM_COEFF_SIGN (1ULL << 63) @@ -609,6 +619,8 @@ static u16 ctm_to_twos_complement(u64 coeff, int int_bits, int frac_bits) if (CTM_COEFF_NEGATIVE(coeff)) c = -c; + int_bits = max(int_bits, 1); + c = clamp(c, -(s64)BIT(int_bits + frac_bits - 1), (s64)(BIT(int_bits + frac_bits - 1) - 1)); @@ -3836,6 +3848,266 @@ static void icl_read_luts(struct intel_crtc_state *crtc_state) } } +static void +xelpd_load_plane_csc_matrix(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state) +{ + struct intel_display *display = to_intel_display(plane_state); + const struct drm_plane_state *state = &plane_state->uapi; + enum pipe pipe = to_intel_plane(state->plane)->pipe; + enum plane_id plane = to_intel_plane(state->plane)->id; + const struct drm_property_blob *blob = plane_state->hw.ctm; + struct drm_color_ctm_3x4 *ctm; + const u64 *input; + u16 coeffs[9] = {}; + int i, j; + + if (!icl_is_hdr_plane(display, plane) || !blob) + return; + + ctm = blob->data; + input = ctm->matrix; + + /* + * Convert fixed point S31.32 input to format supported by the + * hardware. + */ + for (i = 0, j = 0; i < ARRAY_SIZE(coeffs); i++) { + u64 abs_coeff = ((1ULL << 63) - 1) & input[j]; + + /* + * Clamp input value to min/max supported by + * hardware. + */ + abs_coeff = clamp_val(abs_coeff, 0, CTM_COEFF_4_0 - 1); + + /* sign bit */ + if (CTM_COEFF_NEGATIVE(input[j])) + coeffs[i] |= 1 << 15; + + if (abs_coeff < CTM_COEFF_0_125) + coeffs[i] |= (3 << 12) | + ILK_CSC_COEFF_FP(abs_coeff, 12); + else if (abs_coeff < CTM_COEFF_0_25) + coeffs[i] |= (2 << 12) | + ILK_CSC_COEFF_FP(abs_coeff, 11); + else if (abs_coeff < CTM_COEFF_0_5) + coeffs[i] |= (1 << 12) | + ILK_CSC_COEFF_FP(abs_coeff, 10); + else if (abs_coeff < CTM_COEFF_1_0) + coeffs[i] |= ILK_CSC_COEFF_FP(abs_coeff, 9); + else if (abs_coeff < CTM_COEFF_2_0) + coeffs[i] |= (7 << 12) | + ILK_CSC_COEFF_FP(abs_coeff, 8); + else + coeffs[i] |= (6 << 12) | + ILK_CSC_COEFF_FP(abs_coeff, 7); + + /* Skip postoffs */ + if (!((j + 2) % 4)) + j += 2; + else + j++; + } + + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 0), + coeffs[0] << 16 | coeffs[1]); + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 1), + coeffs[2] << 16); + + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 2), + coeffs[3] << 16 | coeffs[4]); + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 3), + coeffs[5] << 16); + + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 4), + coeffs[6] << 16 | coeffs[7]); + intel_de_write_dsb(display, dsb, PLANE_CSC_COEFF(pipe, plane, 5), + coeffs[8] << 16); + + intel_de_write_dsb(display, dsb, PLANE_CSC_PREOFF(pipe, plane, 0), 0); + intel_de_write_dsb(display, dsb, PLANE_CSC_PREOFF(pipe, plane, 1), 0); + intel_de_write_dsb(display, dsb, PLANE_CSC_PREOFF(pipe, plane, 2), 0); + + /* + * Conversion from S31.32 to S0.12. BIT[12] is the signed bit + */ + intel_de_write_dsb(display, dsb, + PLANE_CSC_POSTOFF(pipe, plane, 0), + ctm_to_twos_complement(input[3], 0, 12)); + intel_de_write_dsb(display, dsb, + PLANE_CSC_POSTOFF(pipe, plane, 1), + ctm_to_twos_complement(input[7], 0, 12)); + intel_de_write_dsb(display, dsb, + PLANE_CSC_POSTOFF(pipe, plane, 2), + ctm_to_twos_complement(input[11], 0, 12)); +} + +static void +xelpd_program_plane_pre_csc_lut(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state) +{ + struct intel_display *display = to_intel_display(plane_state); + const struct drm_plane_state *state = &plane_state->uapi; + enum pipe pipe = to_intel_plane(state->plane)->pipe; + enum plane_id plane = to_intel_plane(state->plane)->id; + const struct drm_color_lut32 *pre_csc_lut = plane_state->hw.degamma_lut->data; + u32 i, lut_size; + + if (icl_is_hdr_plane(display, plane)) { + lut_size = 128; + + intel_de_write_dsb(display, dsb, + PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, 0), + PLANE_PAL_PREC_AUTO_INCREMENT); + + if (pre_csc_lut) { + for (i = 0; i < lut_size; i++) { + u32 lut_val = drm_color_lut32_extract(pre_csc_lut[i].green, 24); + + intel_de_write_dsb(display, dsb, + PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0), + lut_val); + } + + /* Program the max register to clamp values > 1.0. */ + /* TODO: Restrict to 0x7ffffff */ + do { + intel_de_write_dsb(display, dsb, + PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0), + (1 << 24)); + } while (i++ > 130); + } else { + for (i = 0; i < lut_size; i++) { + u32 v = (i * ((1 << 24) - 1)) / (lut_size - 1); + + intel_de_write_dsb(display, dsb, + PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0), v); + } + + do { + intel_de_write_dsb(display, dsb, + PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, 0), + 1 << 24); + } while (i++ < 130); + } + + intel_de_write_dsb(display, dsb, PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 0); + } +} + +static void +xelpd_program_plane_post_csc_lut(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state) +{ + struct intel_display *display = to_intel_display(plane_state); + const struct drm_plane_state *state = &plane_state->uapi; + enum pipe pipe = to_intel_plane(state->plane)->pipe; + enum plane_id plane = to_intel_plane(state->plane)->id; + const struct drm_color_lut32 *post_csc_lut = plane_state->hw.gamma_lut->data; + u32 i, lut_size, lut_val; + + if (icl_is_hdr_plane(display, plane)) { + intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0), + PLANE_PAL_PREC_AUTO_INCREMENT); + /* TODO: Add macro */ + intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0), + PLANE_PAL_PREC_AUTO_INCREMENT); + if (post_csc_lut) { + lut_size = 32; + for (i = 0; i < lut_size; i++) { + lut_val = drm_color_lut32_extract(post_csc_lut[i].green, 24); + + intel_de_write_dsb(display, dsb, + PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0), + lut_val); + } + + /* Segment 2 */ + do { + intel_de_write_dsb(display, dsb, + PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0), + (1 << 24)); + } while (i++ < 34); + } else { + /*TODO: Add for segment 0 */ + lut_size = 32; + for (i = 0; i < lut_size; i++) { + u32 v = (i * ((1 << 24) - 1)) / (lut_size - 1); + + intel_de_write_dsb(display, dsb, + PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0), v); + } + + do { + intel_de_write_dsb(display, dsb, + PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, 0), + 1 << 24); + } while (i++ < 34); + } + + intel_de_write_dsb(display, dsb, PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, 0), 0); + intel_de_write_dsb(display, dsb, + PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, 0), 0); + } +} + +static void +xelpd_plane_load_luts(struct intel_dsb *dsb, const struct intel_plane_state *plane_state) +{ + if (plane_state->hw.degamma_lut) + xelpd_program_plane_pre_csc_lut(dsb, plane_state); + + if (plane_state->hw.gamma_lut) + xelpd_program_plane_post_csc_lut(dsb, plane_state); +} + +static u32 glk_3dlut_10(const struct drm_color_lut32 *color) +{ + return REG_FIELD_PREP(LUT_3D_DATA_RED_MASK, drm_color_lut32_extract(color->red, 10)) | + REG_FIELD_PREP(LUT_3D_DATA_GREEN_MASK, drm_color_lut32_extract(color->green, 10)) | + REG_FIELD_PREP(LUT_3D_DATA_BLUE_MASK, drm_color_lut32_extract(color->blue, 10)); +} + +static void glk_load_lut_3d(struct intel_dsb *dsb, + struct intel_crtc *crtc, + const struct drm_property_blob *blob) +{ + struct intel_display *display = to_intel_display(crtc->base.dev); + const struct drm_color_lut32 *lut = blob->data; + int i, lut_size = drm_color_lut32_size(blob); + enum pipe pipe = crtc->pipe; + + if (!dsb && intel_de_read(display, LUT_3D_CTL(pipe)) & LUT_3D_READY) { + drm_err(display->drm, "[CRTC:%d:%s] 3D LUT not ready, not loading LUTs\n", + crtc->base.base.id, crtc->base.name); + return; + } + + intel_de_write_dsb(display, dsb, LUT_3D_INDEX(pipe), LUT_3D_AUTO_INCREMENT); + for (i = 0; i < lut_size; i++) + intel_de_write_dsb(display, dsb, LUT_3D_DATA(pipe), glk_3dlut_10(&lut[i])); + intel_de_write_dsb(display, dsb, LUT_3D_INDEX(pipe), 0); +} + +static void glk_lut_3d_commit(struct intel_dsb *dsb, struct intel_crtc *crtc, bool enable) +{ + struct intel_display *display = to_intel_display(crtc); + enum pipe pipe = crtc->pipe; + u32 val = 0; + + if (!dsb && intel_de_read(display, LUT_3D_CTL(pipe)) & LUT_3D_READY) { + drm_err(display->drm, "[CRTC:%d:%s] 3D LUT not ready, not committing change\n", + crtc->base.base.id, crtc->base.name); + return; + } + + if (enable) + val = LUT_3D_ENABLE | LUT_3D_READY | LUT_3D_BIND_PLANE_1; + + intel_de_write_dsb(display, dsb, LUT_3D_CTL(pipe), val); +} + static const struct intel_color_funcs chv_color_funcs = { .color_check = chv_color_check, .color_commit_arm = i9xx_color_commit_arm, @@ -3883,6 +4155,8 @@ static const struct intel_color_funcs tgl_color_funcs = { .lut_equal = icl_lut_equal, .read_csc = icl_read_csc, .get_config = skl_get_config, + .load_plane_csc_matrix = xelpd_load_plane_csc_matrix, + .load_plane_luts = xelpd_plane_load_luts, }; static const struct intel_color_funcs icl_color_funcs = { @@ -3963,6 +4237,67 @@ static const struct intel_color_funcs ilk_color_funcs = { .get_config = ilk_get_config, }; +void intel_color_plane_commit_arm(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state) +{ + struct intel_display *display = to_intel_display(plane_state); + struct intel_crtc *crtc = to_intel_crtc(plane_state->uapi.crtc); + + if (crtc && intel_color_crtc_has_3dlut(display, crtc->pipe)) + glk_lut_3d_commit(dsb, crtc, !!plane_state->hw.lut_3d); +} + +static void +intel_color_load_plane_csc_matrix(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state) +{ + struct intel_display *display = to_intel_display(plane_state); + + if (display->funcs.color->load_plane_csc_matrix) + display->funcs.color->load_plane_csc_matrix(dsb, plane_state); +} + +static void +intel_color_load_plane_luts(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state) +{ + struct intel_display *display = to_intel_display(plane_state); + + if (display->funcs.color->load_plane_luts) + display->funcs.color->load_plane_luts(dsb, plane_state); +} + +bool +intel_color_crtc_has_3dlut(struct intel_display *display, enum pipe pipe) +{ + if (DISPLAY_VER(display) >= 12) + return pipe == PIPE_A || pipe == PIPE_B; + else + return false; +} + +static void +intel_color_load_3dlut(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state) +{ + struct intel_display *display = to_intel_display(plane_state); + struct intel_crtc *crtc = to_intel_crtc(plane_state->uapi.crtc); + + if (crtc && intel_color_crtc_has_3dlut(display, crtc->pipe)) + glk_load_lut_3d(dsb, crtc, plane_state->hw.lut_3d); +} + +void intel_color_plane_program_pipeline(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state) +{ + if (plane_state->hw.ctm) + intel_color_load_plane_csc_matrix(dsb, plane_state); + if (plane_state->hw.degamma_lut || plane_state->hw.gamma_lut) + intel_color_load_plane_luts(dsb, plane_state); + if (plane_state->hw.lut_3d) + intel_color_load_3dlut(dsb, plane_state); +} + void intel_color_crtc_init(struct intel_crtc *crtc) { struct intel_display *display = to_intel_display(crtc); diff --git a/drivers/gpu/drm/i915/display/intel_color.h b/drivers/gpu/drm/i915/display/intel_color.h index bf7a12ce9df0..c21b9bdf7bb8 100644 --- a/drivers/gpu/drm/i915/display/intel_color.h +++ b/drivers/gpu/drm/i915/display/intel_color.h @@ -13,7 +13,9 @@ struct intel_crtc_state; struct intel_crtc; struct intel_display; struct intel_dsb; +struct intel_plane_state; struct drm_property_blob; +enum pipe; void intel_color_init_hooks(struct intel_display *display); int intel_color_init(struct intel_display *display); @@ -40,5 +42,9 @@ bool intel_color_lut_equal(const struct intel_crtc_state *crtc_state, const struct drm_property_blob *blob2, bool is_pre_csc_lut); void intel_color_assert_luts(const struct intel_crtc_state *crtc_state); - +void intel_color_plane_program_pipeline(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state); +void intel_color_plane_commit_arm(struct intel_dsb *dsb, + const struct intel_plane_state *plane_state); +bool intel_color_crtc_has_3dlut(struct intel_display *display, enum pipe pipe); #endif /* __INTEL_COLOR_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_color_pipeline.c b/drivers/gpu/drm/i915/display/intel_color_pipeline.c new file mode 100644 index 000000000000..942d9b9c93ce --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_color_pipeline.c @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ +#include "intel_color.h" +#include "intel_colorop.h" +#include "intel_color_pipeline.h" +#include "intel_de.h" +#include "intel_display_types.h" +#include "skl_universal_plane.h" + +#define MAX_COLOR_PIPELINES 1 +#define PLANE_DEGAMMA_SIZE 128 +#define PLANE_GAMMA_SIZE 32 + +static +int _intel_color_pipeline_plane_init(struct drm_plane *plane, struct drm_prop_enum_list *list, + enum pipe pipe) +{ + struct drm_device *dev = plane->dev; + struct intel_display *display = to_intel_display(dev); + struct drm_colorop *prev_op; + struct intel_colorop *colorop; + int ret; + + colorop = intel_colorop_create(INTEL_PLANE_CB_PRE_CSC_LUT); + + ret = drm_plane_colorop_curve_1d_lut_init(dev, &colorop->base, plane, + PLANE_DEGAMMA_SIZE, + DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR, + DRM_COLOROP_FLAG_ALLOW_BYPASS); + + if (ret) + return ret; + + list->type = colorop->base.base.id; + list->name = kasprintf(GFP_KERNEL, "Color Pipeline %d", colorop->base.base.id); + + /* TODO: handle failures and clean up */ + prev_op = &colorop->base; + + if (DISPLAY_VER(display) >= 35 && + intel_color_crtc_has_3dlut(display, pipe) && + plane->type == DRM_PLANE_TYPE_PRIMARY) { + colorop = intel_colorop_create(INTEL_PLANE_CB_3DLUT); + + ret = drm_plane_colorop_3dlut_init(dev, &colorop->base, plane, 17, + DRM_COLOROP_LUT3D_INTERPOLATION_TETRAHEDRAL, + true); + if (ret) + return ret; + + drm_colorop_set_next_property(prev_op, &colorop->base); + + prev_op = &colorop->base; + } + + colorop = intel_colorop_create(INTEL_PLANE_CB_CSC); + ret = drm_plane_colorop_ctm_3x4_init(dev, &colorop->base, plane, + DRM_COLOROP_FLAG_ALLOW_BYPASS); + if (ret) + return ret; + + drm_colorop_set_next_property(prev_op, &colorop->base); + prev_op = &colorop->base; + + colorop = intel_colorop_create(INTEL_PLANE_CB_POST_CSC_LUT); + ret = drm_plane_colorop_curve_1d_lut_init(dev, &colorop->base, plane, + PLANE_GAMMA_SIZE, + DRM_COLOROP_LUT1D_INTERPOLATION_LINEAR, + DRM_COLOROP_FLAG_ALLOW_BYPASS); + if (ret) + return ret; + + drm_colorop_set_next_property(prev_op, &colorop->base); + + return 0; +} + +int intel_color_pipeline_plane_init(struct drm_plane *plane, enum pipe pipe) +{ + struct drm_device *dev = plane->dev; + struct intel_display *display = to_intel_display(dev); + struct drm_prop_enum_list pipelines[MAX_COLOR_PIPELINES]; + int len = 0; + int ret; + + /* Currently expose pipeline only for HDR planes */ + if (!icl_is_hdr_plane(display, to_intel_plane(plane)->id)) + return 0; + + /* Add pipeline consisting of transfer functions */ + ret = _intel_color_pipeline_plane_init(plane, &pipelines[len], pipe); + if (ret) + return ret; + len++; + + return drm_plane_create_color_pipeline_property(plane, pipelines, len); +} diff --git a/drivers/gpu/drm/i915/display/intel_color_pipeline.h b/drivers/gpu/drm/i915/display/intel_color_pipeline.h new file mode 100644 index 000000000000..a457d306da7f --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_color_pipeline.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef __INTEL_COLOR_PIPELINE_H__ +#define __INTEL_COLOR_PIPELINE_H__ + +struct drm_plane; +enum pipe; + +int intel_color_pipeline_plane_init(struct drm_plane *plane, enum pipe pipe); + +#endif /* __INTEL_COLOR_PIPELINE_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_color_regs.h b/drivers/gpu/drm/i915/display/intel_color_regs.h index 8eb643cfead7..c370b6029369 100644 --- a/drivers/gpu/drm/i915/display/intel_color_regs.h +++ b/drivers/gpu/drm/i915/display/intel_color_regs.h @@ -316,4 +316,33 @@ #define SKL_BOTTOM_COLOR_CSC_ENABLE REG_BIT(30) #define SKL_BOTTOM_COLOR(pipe) _MMIO_PIPE(pipe, _SKL_BOTTOM_COLOR_A, _SKL_BOTTOM_COLOR_B) +/* 3D LUT */ +#define _LUT_3D_CTL_A 0x490A4 +#define _LUT_3D_CTL_B 0x491A4 +#define LUT_3D_CTL(pipe) _MMIO_PIPE(pipe, _LUT_3D_CTL_A, _LUT_3D_CTL_B) +#define LUT_3D_ENABLE REG_BIT(31) +#define LUT_3D_READY REG_BIT(30) +#define LUT_3D_BINDING_MASK REG_GENMASK(23, 22) +#define LUT_3D_BIND_PIPE REG_FIELD_PREP(LUT_3D_BINDING_MASK, 0) +#define LUT_3D_BIND_PLANE_1 REG_FIELD_PREP(LUT_3D_BINDING_MASK, 1) +#define LUT_3D_BIND_PLANE_2 REG_FIELD_PREP(LUT_3D_BINDING_MASK, 2) +#define LUT_3D_BIND_PLANE_3 REG_FIELD_PREP(LUT_3D_BINDING_MASK, 3) + +#define _LUT_3D_INDEX_A 0x490A8 +#define _LUT_3D_INDEX_B 0x491A8 +#define LUT_3D_INDEX(pipe) _MMIO_PIPE(pipe, _LUT_3D_INDEX_A, _LUT_3D_INDEX_B) +#define LUT_3D_AUTO_INCREMENT REG_BIT(13) +#define LUT_3D_INDEX_VALUE_MASK REG_GENMASK(12, 0) +#define LUT_3D_INDEX_VALUE(x) REG_FIELD_PREP(LUT_3D_INDEX_VALUE_MASK, (x)) + +#define _LUT_3D_DATA_A 0x490AC +#define _LUT_3D_DATA_B 0x491AC +#define LUT_3D_DATA(pipe) _MMIO_PIPE(pipe, _LUT_3D_DATA_A, _LUT_3D_DATA_B) +#define LUT_3D_DATA_RED_MASK REG_GENMASK(29, 20) +#define LUT_3D_DATA_GREEN_MASK REG_GENMASK(19, 10) +#define LUT_3D_DATA_BLUE_MASK REG_GENMASK(9, 0) +#define LUT_3D_DATA_RED(x) REG_FIELD_PREP(LUT_3D_DATA_RED_MASK, (x)) +#define LUT_3D_DATA_GREEN(x) REG_FIELD_PREP(LUT_3D_DATA_GREEN_MASK, (x)) +#define LUT_3D_DATA_BLUE(x) REG_FIELD_PREP(LUT_3D_DATA_BLUE_MASK, (x)) + #endif /* __INTEL_COLOR_REGS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_colorop.c b/drivers/gpu/drm/i915/display/intel_colorop.c new file mode 100644 index 000000000000..f2fc0d8780ce --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_colorop.c @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ +#include "intel_colorop.h" + +struct intel_colorop *to_intel_colorop(struct drm_colorop *colorop) +{ + return container_of(colorop, struct intel_colorop, base); +} + +struct intel_colorop *intel_colorop_alloc(void) +{ + struct intel_colorop *colorop; + + colorop = kzalloc(sizeof(*colorop), GFP_KERNEL); + if (!colorop) + return ERR_PTR(-ENOMEM); + + return colorop; +} + +struct intel_colorop *intel_colorop_create(enum intel_color_block id) +{ + struct intel_colorop *colorop; + + colorop = intel_colorop_alloc(); + + if (IS_ERR(colorop)) + return colorop; + + colorop->id = id; + + return colorop; +} diff --git a/drivers/gpu/drm/i915/display/intel_colorop.h b/drivers/gpu/drm/i915/display/intel_colorop.h new file mode 100644 index 000000000000..21d58eb9f3d0 --- /dev/null +++ b/drivers/gpu/drm/i915/display/intel_colorop.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2025 Intel Corporation + */ + +#ifndef __INTEL_COLOROP_H__ +#define __INTEL_COLOROP_H__ + +#include "intel_display_types.h" + +struct intel_colorop *to_intel_colorop(struct drm_colorop *colorop); +struct intel_colorop *intel_colorop_alloc(void); +struct intel_colorop *intel_colorop_create(enum intel_color_block id); + +#endif /* __INTEL_COLOROP_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 7b4fd18c60e2..095a319f8bc9 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -7304,6 +7304,7 @@ static void intel_atomic_dsb_finish(struct intel_atomic_state *state, struct intel_display *display = to_intel_display(state); struct intel_crtc_state *new_crtc_state = intel_atomic_get_new_crtc_state(state, crtc); + unsigned int size = new_crtc_state->plane_color_changed ? 8192 : 1024; if (!new_crtc_state->use_flipq && !new_crtc_state->use_dsb && @@ -7314,10 +7315,12 @@ static void intel_atomic_dsb_finish(struct intel_atomic_state *state, * Rough estimate: * ~64 registers per each plane * 8 planes = 512 * Double that for pipe stuff and other overhead. + * ~4913 registers for 3DLUT + * ~200 color registers * 3 HDR planes */ new_crtc_state->dsb_commit = intel_dsb_prepare(state, crtc, INTEL_DSB_0, new_crtc_state->use_dsb || - new_crtc_state->use_flipq ? 1024 : 16); + new_crtc_state->use_flipq ? size : 16); if (!new_crtc_state->dsb_commit) { new_crtc_state->use_flipq = false; new_crtc_state->use_dsb = false; diff --git a/drivers/gpu/drm/i915/display/intel_display_limits.h b/drivers/gpu/drm/i915/display/intel_display_limits.h index f0fa27e365ab..cb3c9c665c44 100644 --- a/drivers/gpu/drm/i915/display/intel_display_limits.h +++ b/drivers/gpu/drm/i915/display/intel_display_limits.h @@ -138,4 +138,13 @@ enum hpd_pin { HPD_NUM_PINS }; +enum intel_color_block { + INTEL_PLANE_CB_PRE_CSC_LUT, + INTEL_PLANE_CB_CSC, + INTEL_PLANE_CB_POST_CSC_LUT, + INTEL_PLANE_CB_3DLUT, + + INTEL_CB_MAX +}; + #endif /* __INTEL_DISPLAY_LIMITS_H__ */ diff --git a/drivers/gpu/drm/i915/display/intel_display_types.h b/drivers/gpu/drm/i915/display/intel_display_types.h index 38702a9e0f50..06bf8f7c0989 100644 --- a/drivers/gpu/drm/i915/display/intel_display_types.h +++ b/drivers/gpu/drm/i915/display/intel_display_types.h @@ -646,6 +646,7 @@ struct intel_plane_state { enum drm_color_encoding color_encoding; enum drm_color_range color_range; enum drm_scaling_filter scaling_filter; + struct drm_property_blob *ctm, *degamma_lut, *gamma_lut, *lut_3d; } hw; struct i915_vma *ggtt_vma; @@ -1391,6 +1392,9 @@ struct intel_crtc_state { u8 silence_period_sym_clocks; u8 lfps_half_cycle_num_of_syms; } alpm_state; + + /* to track changes in plane color blocks */ + bool plane_color_changed; }; enum intel_pipe_crc_source { @@ -1985,6 +1989,11 @@ struct intel_dp_mst_encoder { struct intel_connector *connector; }; +struct intel_colorop { + struct drm_colorop base; + enum intel_color_block id; +}; + static inline struct intel_encoder * intel_attached_encoder(struct intel_connector *connector) { diff --git a/drivers/gpu/drm/i915/display/intel_plane.c b/drivers/gpu/drm/i915/display/intel_plane.c index 5105e3278bc4..ab6a58530b39 100644 --- a/drivers/gpu/drm/i915/display/intel_plane.c +++ b/drivers/gpu/drm/i915/display/intel_plane.c @@ -49,6 +49,7 @@ #include "i9xx_plane_regs.h" #include "intel_cdclk.h" #include "intel_cursor.h" +#include "intel_colorop.h" #include "intel_display_rps.h" #include "intel_display_trace.h" #include "intel_display_types.h" @@ -336,6 +337,58 @@ intel_plane_copy_uapi_plane_damage(struct intel_plane_state *new_plane_state, *damage = drm_plane_state_src(&new_uapi_plane_state->uapi); } +static bool +intel_plane_colorop_replace_blob(struct intel_plane_state *plane_state, + struct intel_colorop *intel_colorop, + struct drm_property_blob *blob) +{ + if (intel_colorop->id == INTEL_PLANE_CB_CSC) + return drm_property_replace_blob(&plane_state->hw.ctm, blob); + else if (intel_colorop->id == INTEL_PLANE_CB_PRE_CSC_LUT) + return drm_property_replace_blob(&plane_state->hw.degamma_lut, blob); + else if (intel_colorop->id == INTEL_PLANE_CB_POST_CSC_LUT) + return drm_property_replace_blob(&plane_state->hw.gamma_lut, blob); + else if (intel_colorop->id == INTEL_PLANE_CB_3DLUT) + return drm_property_replace_blob(&plane_state->hw.lut_3d, blob); + + return false; +} + +static void +intel_plane_color_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, + const struct intel_plane_state *from_plane_state, + struct intel_crtc *crtc) +{ + struct drm_colorop *iter_colorop, *colorop; + struct drm_colorop_state *new_colorop_state; + struct drm_atomic_state *state = plane_state->uapi.state; + struct intel_colorop *intel_colorop; + struct drm_property_blob *blob; + struct intel_atomic_state *intel_atomic_state = to_intel_atomic_state(state); + struct intel_crtc_state *new_crtc_state = intel_atomic_state ? + intel_atomic_get_new_crtc_state(intel_atomic_state, crtc) : NULL; + bool changed = false; + int i = 0; + + iter_colorop = plane_state->uapi.color_pipeline; + + while (iter_colorop) { + for_each_new_colorop_in_state(state, colorop, new_colorop_state, i) { + if (new_colorop_state->colorop == iter_colorop) { + blob = new_colorop_state->bypass ? NULL : new_colorop_state->data; + intel_colorop = to_intel_colorop(colorop); + changed |= intel_plane_colorop_replace_blob(plane_state, + intel_colorop, + blob); + } + } + iter_colorop = iter_colorop->next; + } + + if (new_crtc_state && changed) + new_crtc_state->plane_color_changed = true; +} + void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, const struct intel_plane_state *from_plane_state, struct intel_crtc *crtc) @@ -364,6 +417,8 @@ void intel_plane_copy_uapi_to_hw_state(struct intel_plane_state *plane_state, plane_state->uapi.src = drm_plane_state_src(&from_plane_state->uapi); plane_state->uapi.dst = drm_plane_state_dest(&from_plane_state->uapi); + + intel_plane_color_copy_uapi_to_hw_state(plane_state, from_plane_state, crtc); } void intel_plane_copy_hw_state(struct intel_plane_state *plane_state, diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane.c b/drivers/gpu/drm/i915/display/skl_universal_plane.c index 89c8003ccfe7..ee8e24497d2c 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane.c +++ b/drivers/gpu/drm/i915/display/skl_universal_plane.c @@ -11,6 +11,8 @@ #include "pxp/intel_pxp.h" #include "intel_bo.h" +#include "intel_color.h" +#include "intel_color_pipeline.h" #include "intel_de.h" #include "intel_display_irq.h" #include "intel_display_regs.h" @@ -1275,6 +1277,18 @@ static u32 glk_plane_color_ctl(const struct intel_plane_state *plane_state) if (plane_state->force_black) plane_color_ctl |= PLANE_COLOR_PLANE_CSC_ENABLE; + if (plane_state->hw.degamma_lut) + plane_color_ctl |= PLANE_COLOR_PRE_CSC_GAMMA_ENABLE; + + if (plane_state->hw.ctm) + plane_color_ctl |= PLANE_COLOR_PLANE_CSC_ENABLE; + + if (plane_state->hw.gamma_lut) { + plane_color_ctl &= ~PLANE_COLOR_PLANE_GAMMA_DISABLE; + if (drm_color_lut32_size(plane_state->hw.gamma_lut) != 32) + plane_color_ctl |= PLANE_COLOR_POST_CSC_GAMMA_MULTSEG_ENABLE; + } + return plane_color_ctl; } @@ -1556,6 +1570,8 @@ icl_plane_update_noarm(struct intel_dsb *dsb, plane_color_ctl = plane_state->color_ctl | glk_plane_color_ctl_crtc(crtc_state); + intel_color_plane_program_pipeline(dsb, plane_state); + /* The scaler will handle the output position */ if (plane_state->scaler_id >= 0) { crtc_x = 0; @@ -1657,6 +1673,8 @@ icl_plane_update_arm(struct intel_dsb *dsb, icl_plane_update_sel_fetch_arm(dsb, plane, crtc_state, plane_state); + intel_color_plane_commit_arm(dsb, plane_state); + /* * In order to have FBC for fp16 formats pixel normalizer block must be * active. Check if pixel normalizer block need to be enabled for FBC. @@ -3001,6 +3019,9 @@ skl_universal_plane_create(struct intel_display *display, DRM_COLOR_YCBCR_BT709, DRM_COLOR_YCBCR_LIMITED_RANGE); + if (DISPLAY_VER(display) >= 12) + intel_color_pipeline_plane_init(&plane->base, pipe); + drm_plane_create_alpha_property(&plane->base); drm_plane_create_blend_mode_property(&plane->base, BIT(DRM_MODE_BLEND_PIXEL_NONE) | diff --git a/drivers/gpu/drm/i915/display/skl_universal_plane_regs.h b/drivers/gpu/drm/i915/display/skl_universal_plane_regs.h index 6f815b231340..6fd4da9f63cf 100644 --- a/drivers/gpu/drm/i915/display/skl_universal_plane_regs.h +++ b/drivers/gpu/drm/i915/display/skl_universal_plane_regs.h @@ -254,6 +254,8 @@ #define PLANE_COLOR_PIPE_CSC_ENABLE REG_BIT(23) /* Pre-ICL */ #define PLANE_COLOR_PLANE_CSC_ENABLE REG_BIT(21) /* ICL+ */ #define PLANE_COLOR_INPUT_CSC_ENABLE REG_BIT(20) /* ICL+ */ +#define PLANE_COLOR_POST_CSC_GAMMA_MULTSEG_ENABLE REG_BIT(15) /* TGL+ */ +#define PLANE_COLOR_PRE_CSC_GAMMA_ENABLE REG_BIT(14) #define PLANE_COLOR_CSC_MODE_MASK REG_GENMASK(19, 17) #define PLANE_COLOR_CSC_MODE_BYPASS REG_FIELD_PREP(PLANE_COLOR_CSC_MODE_MASK, 0) #define PLANE_COLOR_CSC_MODE_YUV601_TO_RGB601 REG_FIELD_PREP(PLANE_COLOR_CSC_MODE_MASK, 1) @@ -290,6 +292,119 @@ _PLANE_INPUT_CSC_POSTOFF_HI_1_A, _PLANE_INPUT_CSC_POSTOFF_HI_1_B, \ _PLANE_INPUT_CSC_POSTOFF_HI_2_A, _PLANE_INPUT_CSC_POSTOFF_HI_2_B) +#define _MMIO_PLANE_GAMC(plane, i, a, b) _MMIO(_PIPE(plane, a, b) + (i) * 4) + +#define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1_A 0x70160 +#define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1_B 0x71160 +#define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2_A 0x70260 +#define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2_B 0x71260 +#define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1_A, \ + _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1_B) +#define _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2_A, \ + _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2_B) +#define PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_1(pipe), \ + _PLANE_POST_CSC_GAMC_SEG0_INDEX_ENH_2(pipe)) + +#define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1_A 0x70164 +#define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1_B 0x71164 +#define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2_A 0x70264 +#define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2_B 0x71264 +#define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1_A, \ + _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1_B) +#define _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2_A, \ + _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2_B) +#define PLANE_POST_CSC_GAMC_SEG0_DATA_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_1(pipe), \ + _PLANE_POST_CSC_GAMC_SEG0_DATA_ENH_2(pipe)) + +#define _PLANE_POST_CSC_GAMC_INDEX_ENH_1_A 0x701d8 +#define _PLANE_POST_CSC_GAMC_INDEX_ENH_1_B 0x711d8 +#define _PLANE_POST_CSC_GAMC_INDEX_ENH_2_A 0x702d8 +#define _PLANE_POST_CSC_GAMC_INDEX_ENH_2_B 0x712d8 +#define _PLANE_POST_CSC_GAMC_INDEX_ENH_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_INDEX_ENH_1_A, \ + _PLANE_POST_CSC_GAMC_INDEX_ENH_1_B) +#define _PLANE_POST_CSC_GAMC_INDEX_ENH_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_INDEX_ENH_2_A, \ + _PLANE_POST_CSC_GAMC_INDEX_ENH_2_B) +#define PLANE_POST_CSC_GAMC_INDEX_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_INDEX_ENH_1(pipe), \ + _PLANE_POST_CSC_GAMC_INDEX_ENH_2(pipe)) + +#define _PLANE_POST_CSC_GAMC_DATA_ENH_1_A 0x701dc +#define _PLANE_POST_CSC_GAMC_DATA_ENH_1_B 0x711dc +#define _PLANE_POST_CSC_GAMC_DATA_ENH_2_A 0x702dc +#define _PLANE_POST_CSC_GAMC_DATA_ENH_2_B 0x712dc +#define _PLANE_POST_CSC_GAMC_DATA_ENH_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_DATA_ENH_1_A, \ + _PLANE_POST_CSC_GAMC_DATA_ENH_1_B) +#define _PLANE_POST_CSC_GAMC_DATA_ENH_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_DATA_ENH_2_A, \ + _PLANE_POST_CSC_GAMC_DATA_ENH_2_B) +#define PLANE_POST_CSC_GAMC_DATA_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_DATA_ENH_1(pipe), \ + _PLANE_POST_CSC_GAMC_DATA_ENH_2(pipe)) + +#define _PLANE_POST_CSC_GAMC_INDEX_1_A 0x704d8 +#define _PLANE_POST_CSC_GAMC_INDEX_1_B 0x714d8 +#define _PLANE_POST_CSC_GAMC_INDEX_2_A 0x705d8 +#define _PLANE_POST_CSC_GAMC_INDEX_2_B 0x715d8 +#define _PLANE_POST_CSC_GAMC_INDEX_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_INDEX_1_A, \ + _PLANE_POST_CSC_GAMC_INDEX_1_B) +#define _PLANE_POST_CSC_GAMC_INDEX_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_INDEX_2_A, \ + _PLANE_POST_CSC_GAMC_INDEX_2_B) +#define PLANE_POST_CSC_GAMC_INDEX(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_INDEX_1(pipe), \ + _PLANE_POST_CSC_GAMC_INDEX_2(pipe)) + +#define _PLANE_POST_CSC_GAMC_DATA_1_A 0x704dc +#define _PLANE_POST_CSC_GAMC_DATA_1_B 0x714dc +#define _PLANE_POST_CSC_GAMC_DATA_2_A 0x705dc +#define _PLANE_POST_CSC_GAMC_DATA_2_B 0x715dc +#define _PLANE_POST_CSC_GAMC_DATA_1(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_DATA_1_A, \ + _PLANE_POST_CSC_GAMC_DATA_1_B) +#define _PLANE_POST_CSC_GAMC_DATA_2(pipe) _PIPE(pipe, _PLANE_POST_CSC_GAMC_DATA_2_A, \ + _PLANE_POST_CSC_GAMC_DATA_2_B) +#define PLANE_POST_CSC_GAMC_DATA(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_POST_CSC_GAMC_DATA_1(pipe), \ + _PLANE_POST_CSC_GAMC_DATA_2(pipe)) + +#define _PLANE_PRE_CSC_GAMC_INDEX_ENH_1_A 0x701d0 +#define _PLANE_PRE_CSC_GAMC_INDEX_ENH_1_B 0x711d0 +#define _PLANE_PRE_CSC_GAMC_INDEX_ENH_2_A 0x702d0 +#define _PLANE_PRE_CSC_GAMC_INDEX_ENH_2_B 0x712d0 +#define _PLANE_PRE_CSC_GAMC_INDEX_ENH_1(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_INDEX_ENH_1_A, \ + _PLANE_PRE_CSC_GAMC_INDEX_ENH_1_B) +#define _PLANE_PRE_CSC_GAMC_INDEX_ENH_2(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_INDEX_ENH_2_A, \ + _PLANE_PRE_CSC_GAMC_INDEX_ENH_2_B) +#define PLANE_PRE_CSC_GAMC_INDEX_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_PRE_CSC_GAMC_INDEX_ENH_1(pipe), \ + _PLANE_PRE_CSC_GAMC_INDEX_ENH_2(pipe)) +#define PLANE_PAL_PREC_AUTO_INCREMENT REG_BIT(10) + +#define _PLANE_PRE_CSC_GAMC_DATA_ENH_1_A 0x701d4 +#define _PLANE_PRE_CSC_GAMC_DATA_ENH_1_B 0x711d4 +#define _PLANE_PRE_CSC_GAMC_DATA_ENH_2_A 0x702d4 +#define _PLANE_PRE_CSC_GAMC_DATA_ENH_2_B 0x712d4 +#define _PLANE_PRE_CSC_GAMC_DATA_ENH_1(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_DATA_ENH_1_A, \ + _PLANE_PRE_CSC_GAMC_DATA_ENH_1_B) +#define _PLANE_PRE_CSC_GAMC_DATA_ENH_2(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_DATA_ENH_2_A, \ + _PLANE_PRE_CSC_GAMC_DATA_ENH_2_B) +#define PLANE_PRE_CSC_GAMC_DATA_ENH(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_PRE_CSC_GAMC_DATA_ENH_1(pipe), \ + _PLANE_PRE_CSC_GAMC_DATA_ENH_2(pipe)) + +#define _PLANE_PRE_CSC_GAMC_INDEX_1_A 0x704d0 +#define _PLANE_PRE_CSC_GAMC_INDEX_1_B 0x714d0 +#define _PLANE_PRE_CSC_GAMC_INDEX_2_A 0x705d0 +#define _PLANE_PRE_CSC_GAMC_INDEX_2_B 0x715d0 +#define _PLANE_PRE_CSC_GAMC_INDEX_1(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_INDEX_1_A, \ + _PLANE_PRE_CSC_GAMC_INDEX_1_B) +#define _PLANE_PRE_CSC_GAMC_INDEX_2(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_INDEX_2_A, \ + _PLANE_PRE_CSC_GAMC_INDEX_2_B) +#define PLANE_PRE_CSC_GAMC_INDEX(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_PRE_CSC_GAMC_INDEX_1(pipe), \ + _PLANE_PRE_CSC_GAMC_INDEX_2(pipe)) + +#define _PLANE_PRE_CSC_GAMC_DATA_1_A 0x704d4 +#define _PLANE_PRE_CSC_GAMC_DATA_1_B 0x714d4 +#define _PLANE_PRE_CSC_GAMC_DATA_2_A 0x705d4 +#define _PLANE_PRE_CSC_GAMC_DATA_2_B 0x715d4 +#define _PLANE_PRE_CSC_GAMC_DATA_1(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_DATA_1_A, \ + _PLANE_PRE_CSC_GAMC_DATA_1_B) +#define _PLANE_PRE_CSC_GAMC_DATA_2(pipe) _PIPE(pipe, _PLANE_PRE_CSC_GAMC_DATA_2_A, \ + _PLANE_PRE_CSC_GAMC_DATA_2_B) +#define PLANE_PRE_CSC_GAMC_DATA(pipe, plane, i) _MMIO_PLANE_GAMC(plane, i, _PLANE_PRE_CSC_GAMC_DATA_1(pipe), \ + _PLANE_PRE_CSC_GAMC_DATA_2(pipe)) + #define _PLANE_CSC_RY_GY_1_A 0x70210 #define _PLANE_CSC_RY_GY_2_A 0x70310 #define _PLANE_CSC_RY_GY_1_B 0x71210 diff --git a/drivers/gpu/drm/xe/Makefile b/drivers/gpu/drm/xe/Makefile index e4b273b025d2..62be4a5227e4 100644 --- a/drivers/gpu/drm/xe/Makefile +++ b/drivers/gpu/drm/xe/Makefile @@ -184,6 +184,10 @@ xe-$(CONFIG_PCI_IOV) += \ xe_sriov_pf_sysfs.o \ xe_tile_sriov_pf_debugfs.o +ifdef CONFIG_XE_VFIO_PCI + xe-$(CONFIG_PCI_IOV) += xe_sriov_vfio.o +endif + # include helpers for tests even when XE is built-in ifdef CONFIG_DRM_XE_KUNIT_TEST xe-y += tests/xe_kunit_helpers.o @@ -242,6 +246,8 @@ xe-$(CONFIG_DRM_XE_DISPLAY) += \ i915-display/intel_cdclk.o \ i915-display/intel_cmtg.o \ i915-display/intel_color.o \ + i915-display/intel_colorop.o \ + i915-display/intel_color_pipeline.o \ i915-display/intel_combo_phy.o \ i915-display/intel_connector.o \ i915-display/intel_crtc.o \ diff --git a/drivers/gpu/drm/xe/xe_gpu_scheduler.h b/drivers/gpu/drm/xe/xe_gpu_scheduler.h index 9955397aaaa9..c7a77a3a9681 100644 --- a/drivers/gpu/drm/xe/xe_gpu_scheduler.h +++ b/drivers/gpu/drm/xe/xe_gpu_scheduler.h @@ -54,13 +54,14 @@ static inline void xe_sched_tdr_queue_imm(struct xe_gpu_scheduler *sched) static inline void xe_sched_resubmit_jobs(struct xe_gpu_scheduler *sched) { struct drm_sched_job *s_job; + bool restore_replay = false; list_for_each_entry(s_job, &sched->base.pending_list, list) { struct drm_sched_fence *s_fence = s_job->s_fence; struct dma_fence *hw_fence = s_fence->parent; - if (to_xe_sched_job(s_job)->skip_emit || - (hw_fence && !dma_fence_is_signaled(hw_fence))) + restore_replay |= to_xe_sched_job(s_job)->restore_replay; + if (restore_replay || (hw_fence && !dma_fence_is_signaled(hw_fence))) sched->base.ops->run_job(s_job); } } diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c index 62f6cc45a764..59c5c6b4d994 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_config.c @@ -711,7 +711,7 @@ static u64 pf_profile_fair_ggtt(struct xe_gt *gt, unsigned int num_vfs) if (num_vfs > 56) return SZ_64M - SZ_8M; - return rounddown_pow_of_two(shareable / num_vfs); + return rounddown_pow_of_two(div_u64(shareable, num_vfs)); } /** diff --git a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c index d5d918ddce4f..3174a8dee779 100644 --- a/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_gt_sriov_pf_migration.c @@ -17,6 +17,7 @@ #include "xe_gt_sriov_pf_helpers.h" #include "xe_gt_sriov_pf_migration.h" #include "xe_gt_sriov_printk.h" +#include "xe_guc.h" #include "xe_guc_buf.h" #include "xe_guc_ct.h" #include "xe_migrate.h" @@ -1023,6 +1024,12 @@ static void action_ring_cleanup(void *arg) ptr_ring_cleanup(r, destroy_pf_packet); } +static void pf_gt_migration_check_support(struct xe_gt *gt) +{ + if (GUC_FIRMWARE_VER(>->uc.guc) < MAKE_GUC_VER(70, 54, 0)) + xe_sriov_pf_migration_disable(gt_to_xe(gt), "requires GuC version >= 70.54.0"); +} + /** * xe_gt_sriov_pf_migration_init() - Initialize support for VF migration. * @gt: the &xe_gt @@ -1039,6 +1046,8 @@ int xe_gt_sriov_pf_migration_init(struct xe_gt *gt) xe_gt_assert(gt, IS_SRIOV_PF(xe)); + pf_gt_migration_check_support(gt); + if (!pf_migration_supported(gt)) return 0; diff --git a/drivers/gpu/drm/xe/xe_guc_submit.c b/drivers/gpu/drm/xe/xe_guc_submit.c index d4ffdb71ef3d..ed7be50b2f72 100644 --- a/drivers/gpu/drm/xe/xe_guc_submit.c +++ b/drivers/gpu/drm/xe/xe_guc_submit.c @@ -822,7 +822,7 @@ static void submit_exec_queue(struct xe_exec_queue *q, struct xe_sched_job *job) xe_gt_assert(guc_to_gt(guc), exec_queue_registered(q)); - if (!job->skip_emit || job->last_replay) { + if (!job->restore_replay || job->last_replay) { if (xe_exec_queue_is_parallel(q)) wq_item_append(q); else @@ -881,10 +881,10 @@ guc_exec_queue_run_job(struct drm_sched_job *drm_job) if (!killed_or_banned_or_wedged && !xe_sched_job_is_error(job)) { if (!exec_queue_registered(q)) register_exec_queue(q, GUC_CONTEXT_NORMAL); - if (!job->skip_emit) + if (!job->restore_replay) q->ring_ops->emit_job(job); submit_exec_queue(q, job); - job->skip_emit = false; + job->restore_replay = false; } /* @@ -2112,6 +2112,18 @@ static void guc_exec_queue_revert_pending_state_change(struct xe_guc *guc, q->guc->resume_time = 0; } +static void lrc_parallel_clear(struct xe_lrc *lrc) +{ + struct xe_device *xe = gt_to_xe(lrc->gt); + struct iosys_map map = xe_lrc_parallel_map(lrc); + int i; + + for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) + parallel_write(xe, map, wq[i], + FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | + FIELD_PREP(WQ_LEN_MASK, 0)); +} + /* * This function is quite complex but only real way to ensure no state is lost * during VF resume flows. The function scans the queue state, make adjustments @@ -2135,8 +2147,8 @@ static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) guc_exec_queue_revert_pending_state_change(guc, q); if (xe_exec_queue_is_parallel(q)) { - struct xe_device *xe = guc_to_xe(guc); - struct iosys_map map = xe_lrc_parallel_map(q->lrc[0]); + /* Pairs with WRITE_ONCE in __xe_exec_queue_init */ + struct xe_lrc *lrc = READ_ONCE(q->lrc[0]); /* * NOP existing WQ commands that may contain stale GGTT @@ -2144,14 +2156,14 @@ static void guc_exec_queue_pause(struct xe_guc *guc, struct xe_exec_queue *q) * seems to get confused if the WQ head/tail pointers are * adjusted. */ - for (i = 0; i < WQ_SIZE / sizeof(u32); ++i) - parallel_write(xe, map, wq[i], - FIELD_PREP(WQ_TYPE_MASK, WQ_TYPE_NOOP) | - FIELD_PREP(WQ_LEN_MASK, 0)); + if (lrc) + lrc_parallel_clear(lrc); } job = xe_sched_first_pending_job(sched); if (job) { + job->restore_replay = true; + /* * Adjust software tail so jobs submitted overwrite previous * position in ring buffer with new GGTT addresses. @@ -2241,17 +2253,18 @@ static void guc_exec_queue_unpause_prepare(struct xe_guc *guc, struct xe_exec_queue *q) { struct xe_gpu_scheduler *sched = &q->guc->sched; - struct drm_sched_job *s_job; struct xe_sched_job *job = NULL; + bool restore_replay = false; - list_for_each_entry(s_job, &sched->base.pending_list, list) { - job = to_xe_sched_job(s_job); - - xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", - q->guc->id, xe_sched_job_seqno(job)); + list_for_each_entry(job, &sched->base.pending_list, drm.list) { + restore_replay |= job->restore_replay; + if (restore_replay) { + xe_gt_dbg(guc_to_gt(guc), "Replay JOB - guc_id=%d, seqno=%d", + q->guc->id, xe_sched_job_seqno(job)); - q->ring_ops->emit_job(job); - job->skip_emit = true; + q->ring_ops->emit_job(job); + job->restore_replay = true; + } } if (job) diff --git a/drivers/gpu/drm/xe/xe_pagefault.c b/drivers/gpu/drm/xe/xe_pagefault.c index fe3e40145012..afb06598b6e1 100644 --- a/drivers/gpu/drm/xe/xe_pagefault.c +++ b/drivers/gpu/drm/xe/xe_pagefault.c @@ -102,7 +102,6 @@ retry_userptr: /* Lock VM and BOs dma-resv */ xe_validation_ctx_init(&ctx, &vm->xe->val, &exec, (struct xe_val_flags) {}); - drm_exec_init(&exec, 0, 0); drm_exec_until_all_locked(&exec) { err = xe_pagefault_begin(&exec, vma, tile->mem.vram, needs_vram == 1); diff --git a/drivers/gpu/drm/xe/xe_pci.c b/drivers/gpu/drm/xe/xe_pci.c index 4636e4ef9baa..9c9ea10d994c 100644 --- a/drivers/gpu/drm/xe/xe_pci.c +++ b/drivers/gpu/drm/xe/xe_pci.c @@ -1223,6 +1223,23 @@ static struct pci_driver xe_pci_driver = { #endif }; +/** + * xe_pci_to_pf_device() - Get PF &xe_device. + * @pdev: the VF &pci_dev device + * + * Return: pointer to PF &xe_device, NULL otherwise. + */ +struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev) +{ + struct drm_device *drm; + + drm = pci_iov_get_pf_drvdata(pdev, &xe_pci_driver); + if (IS_ERR(drm)) + return NULL; + + return to_xe_device(drm); +} + int xe_register_pci_driver(void) { return pci_register_driver(&xe_pci_driver); diff --git a/drivers/gpu/drm/xe/xe_pci.h b/drivers/gpu/drm/xe/xe_pci.h index 611c1209b14c..11bcc5fe2c5b 100644 --- a/drivers/gpu/drm/xe/xe_pci.h +++ b/drivers/gpu/drm/xe/xe_pci.h @@ -6,7 +6,10 @@ #ifndef _XE_PCI_H_ #define _XE_PCI_H_ +struct pci_dev; + int xe_register_pci_driver(void); void xe_unregister_pci_driver(void); +struct xe_device *xe_pci_to_pf_device(struct pci_dev *pdev); #endif diff --git a/drivers/gpu/drm/xe/xe_pm.c b/drivers/gpu/drm/xe/xe_pm.c index 44924512830f..766922530265 100644 --- a/drivers/gpu/drm/xe/xe_pm.c +++ b/drivers/gpu/drm/xe/xe_pm.c @@ -726,6 +726,13 @@ static void xe_pm_runtime_lockdep_prime(void) /** * xe_pm_runtime_get - Get a runtime_pm reference and resume synchronously * @xe: xe device instance + * + * When possible, scope-based runtime PM (through guard(xe_pm_runtime)) is + * be preferred over direct usage of this function. Manual get/put handling + * should only be used when the function contains goto-based logic which + * can break scope-based handling, or when the lifetime of the runtime PM + * reference does not match a specific scope (e.g., runtime PM obtained in one + * function and released in a different one). */ void xe_pm_runtime_get(struct xe_device *xe) { @@ -758,6 +765,13 @@ void xe_pm_runtime_put(struct xe_device *xe) * xe_pm_runtime_get_ioctl - Get a runtime_pm reference before ioctl * @xe: xe device instance * + * When possible, scope-based runtime PM (through + * ACQUIRE(xe_pm_runtime_ioctl, ...)) is be preferred over direct usage of this + * function. Manual get/put handling should only be used when the function + * contains goto-based logic which can break scope-based handling, or when the + * lifetime of the runtime PM reference does not match a specific scope (e.g., + * runtime PM obtained in one function and released in a different one). + * * Returns: Any number greater than or equal to 0 for success, negative error * code otherwise. */ @@ -827,6 +841,13 @@ static bool xe_pm_suspending_or_resuming(struct xe_device *xe) * It will warn if not protected. * The reference should be put back after this function regardless, since it * will always bump the usage counter, regardless. + * + * When possible, scope-based runtime PM (through guard(xe_pm_runtime_noresume)) + * is be preferred over direct usage of this function. Manual get/put handling + * should only be used when the function contains goto-based logic which can + * break scope-based handling, or when the lifetime of the runtime PM reference + * does not match a specific scope (e.g., runtime PM obtained in one function + * and released in a different one). */ void xe_pm_runtime_get_noresume(struct xe_device *xe) { diff --git a/drivers/gpu/drm/xe/xe_pm.h b/drivers/gpu/drm/xe/xe_pm.h index f7f89a18b6fc..6b27039e7b2d 100644 --- a/drivers/gpu/drm/xe/xe_pm.h +++ b/drivers/gpu/drm/xe/xe_pm.h @@ -6,6 +6,7 @@ #ifndef _XE_PM_H_ #define _XE_PM_H_ +#include <linux/cleanup.h> #include <linux/pm_runtime.h> #define DEFAULT_VRAM_THRESHOLD 300 /* in MB */ @@ -37,4 +38,20 @@ int xe_pm_block_on_suspend(struct xe_device *xe); void xe_pm_might_block_on_suspend(void); int xe_pm_module_init(void); +static inline void __xe_pm_runtime_noop(struct xe_device *xe) {} + +DEFINE_GUARD(xe_pm_runtime, struct xe_device *, + xe_pm_runtime_get(_T), xe_pm_runtime_put(_T)) +DEFINE_GUARD(xe_pm_runtime_noresume, struct xe_device *, + xe_pm_runtime_get_noresume(_T), xe_pm_runtime_put(_T)) +DEFINE_GUARD_COND(xe_pm_runtime, _ioctl, xe_pm_runtime_get_ioctl(_T), _RET >= 0) + +/* + * Used when a function needs to release runtime PM in all possible cases + * and error paths, but the wakeref was already acquired by a different + * function (i.e., get() has already happened so only a put() is needed). + */ +DEFINE_GUARD(xe_pm_runtime_release_only, struct xe_device *, + __xe_pm_runtime_noop(_T), xe_pm_runtime_put(_T)); + #endif diff --git a/drivers/gpu/drm/xe/xe_sched_job_types.h b/drivers/gpu/drm/xe/xe_sched_job_types.h index d26612abb4ca..7c4c54fe920a 100644 --- a/drivers/gpu/drm/xe/xe_sched_job_types.h +++ b/drivers/gpu/drm/xe/xe_sched_job_types.h @@ -63,8 +63,8 @@ struct xe_sched_job { bool ring_ops_flush_tlb; /** @ggtt: mapped in ggtt. */ bool ggtt; - /** @skip_emit: skip emitting the job */ - bool skip_emit; + /** @restore_replay: job being replayed for restore */ + bool restore_replay; /** @last_replay: last job being replayed */ bool last_replay; /** @ptrs: per instance pointers. */ diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c index de06cc690fc8..6c4b16409cc9 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_migration.c +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.c @@ -46,13 +46,37 @@ bool xe_sriov_pf_migration_supported(struct xe_device *xe) { xe_assert(xe, IS_SRIOV_PF(xe)); - return xe->sriov.pf.migration.supported; + return IS_ENABLED(CONFIG_DRM_XE_DEBUG) || !xe->sriov.pf.migration.disabled; } -static bool pf_check_migration_support(struct xe_device *xe) +/** + * xe_sriov_pf_migration_disable() - Turn off SR-IOV VF migration support on PF. + * @xe: the &xe_device instance. + * @fmt: format string for the log message, to be combined with following VAs. + */ +void xe_sriov_pf_migration_disable(struct xe_device *xe, const char *fmt, ...) +{ + struct va_format vaf; + va_list va_args; + + xe_assert(xe, IS_SRIOV_PF(xe)); + + va_start(va_args, fmt); + vaf.fmt = fmt; + vaf.va = &va_args; + xe_sriov_notice(xe, "migration %s: %pV\n", + IS_ENABLED(CONFIG_DRM_XE_DEBUG) ? + "missing prerequisite" : "disabled", + &vaf); + va_end(va_args); + + xe->sriov.pf.migration.disabled = true; +} + +static void pf_migration_check_support(struct xe_device *xe) { - /* XXX: for now this is for feature enabling only */ - return IS_ENABLED(CONFIG_DRM_XE_DEBUG); + if (!xe_device_has_memirq(xe)) + xe_sriov_pf_migration_disable(xe, "requires memory-based IRQ support"); } static void pf_migration_cleanup(void *arg) @@ -77,7 +101,8 @@ int xe_sriov_pf_migration_init(struct xe_device *xe) xe_assert(xe, IS_SRIOV_PF(xe)); - xe->sriov.pf.migration.supported = pf_check_migration_support(xe); + pf_migration_check_support(xe); + if (!xe_sriov_pf_migration_supported(xe)) return 0; diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration.h b/drivers/gpu/drm/xe/xe_sriov_pf_migration.h index b806298a0bb6..f8f408df8481 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_migration.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration.h @@ -14,6 +14,7 @@ struct xe_sriov_packet; int xe_sriov_pf_migration_init(struct xe_device *xe); bool xe_sriov_pf_migration_supported(struct xe_device *xe); +void xe_sriov_pf_migration_disable(struct xe_device *xe, const char *fmt, ...); int xe_sriov_pf_migration_restore_produce(struct xe_device *xe, unsigned int vfid, struct xe_sriov_packet *data); struct xe_sriov_packet * diff --git a/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h b/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h index 363d673ee1dd..7d9a8a278d91 100644 --- a/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h +++ b/drivers/gpu/drm/xe/xe_sriov_pf_migration_types.h @@ -14,8 +14,8 @@ * struct xe_sriov_pf_migration - Xe device level VF migration data */ struct xe_sriov_pf_migration { - /** @supported: indicates whether VF migration feature is supported */ - bool supported; + /** @disabled: indicates whether VF migration feature is disabled */ + bool disabled; }; /** diff --git a/drivers/gpu/drm/xe/xe_sriov_vfio.c b/drivers/gpu/drm/xe/xe_sriov_vfio.c new file mode 100644 index 000000000000..e9a7615bb5c5 --- /dev/null +++ b/drivers/gpu/drm/xe/xe_sriov_vfio.c @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT +/* + * Copyright © 2025 Intel Corporation + */ + +#include <drm/intel/xe_sriov_vfio.h> +#include <linux/cleanup.h> + +#include "xe_pci.h" +#include "xe_pm.h" +#include "xe_sriov_pf_control.h" +#include "xe_sriov_pf_helpers.h" +#include "xe_sriov_pf_migration.h" + +struct xe_device *xe_sriov_vfio_get_pf(struct pci_dev *pdev) +{ + return xe_pci_to_pf_device(pdev); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_get_pf, "xe-vfio-pci"); + +bool xe_sriov_vfio_migration_supported(struct xe_device *xe) +{ + if (!IS_SRIOV_PF(xe)) + return -EPERM; + + return xe_sriov_pf_migration_supported(xe); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_migration_supported, "xe-vfio-pci"); + +#define DEFINE_XE_SRIOV_VFIO_FUNCTION(_type, _func, _impl) \ +_type xe_sriov_vfio_##_func(struct xe_device *xe, unsigned int vfid) \ +{ \ + if (!IS_SRIOV_PF(xe)) \ + return -EPERM; \ + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) \ + return -EINVAL; \ + \ + guard(xe_pm_runtime_noresume)(xe); \ + \ + return xe_sriov_pf_##_impl(xe, vfid); \ +} \ +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_##_func, "xe-vfio-pci") + +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, wait_flr_done, control_wait_flr); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, suspend_device, control_pause_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_device, control_resume_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_enter, control_trigger_save_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, stop_copy_exit, control_finish_save_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_data_enter, control_trigger_restore_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, resume_data_exit, control_finish_restore_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(int, error, control_stop_vf); +DEFINE_XE_SRIOV_VFIO_FUNCTION(ssize_t, stop_copy_size, migration_size); + +ssize_t xe_sriov_vfio_data_read(struct xe_device *xe, unsigned int vfid, + char __user *buf, size_t len) +{ + if (!IS_SRIOV_PF(xe)) + return -EPERM; + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) + return -EINVAL; + + guard(xe_pm_runtime_noresume)(xe); + + return xe_sriov_pf_migration_read(xe, vfid, buf, len); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_data_read, "xe-vfio-pci"); + +ssize_t xe_sriov_vfio_data_write(struct xe_device *xe, unsigned int vfid, + const char __user *buf, size_t len) +{ + if (!IS_SRIOV_PF(xe)) + return -EPERM; + if (vfid == PFID || vfid > xe_sriov_pf_num_vfs(xe)) + return -EINVAL; + + guard(xe_pm_runtime_noresume)(xe); + + return xe_sriov_pf_migration_write(xe, vfid, buf, len); +} +EXPORT_SYMBOL_FOR_MODULES(xe_sriov_vfio_data_write, "xe-vfio-pci"); diff --git a/drivers/vdpa/mlx5/net/mlx5_vnet.c b/drivers/vdpa/mlx5/net/mlx5_vnet.c index a7936bd1aabe..ddaa1366704b 100644 --- a/drivers/vdpa/mlx5/net/mlx5_vnet.c +++ b/drivers/vdpa/mlx5/net/mlx5_vnet.c @@ -1256,7 +1256,7 @@ static int query_virtqueues(struct mlx5_vdpa_net *ndev, int vq_idx = start_vq + i; if (cmd->err) { - mlx5_vdpa_err(mvdev, "query vq %d failed, err: %d\n", vq_idx, err); + mlx5_vdpa_err(mvdev, "query vq %d failed, err: %d\n", vq_idx, cmd->err); if (!err) err = cmd->err; continue; diff --git a/drivers/vdpa/octeon_ep/octep_vdpa_main.c b/drivers/vdpa/octeon_ep/octep_vdpa_main.c index 9e8d07078606..31a02e7fd7f2 100644 --- a/drivers/vdpa/octeon_ep/octep_vdpa_main.c +++ b/drivers/vdpa/octeon_ep/octep_vdpa_main.c @@ -736,6 +736,7 @@ static int octep_sriov_enable(struct pci_dev *pdev, int num_vfs) octep_vdpa_assign_barspace(vf_pdev, pdev, index); if (++index == num_vfs) { done = true; + pci_dev_put(vf_pdev); break; } } diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c index 36f61cc96e21..43426bd971ac 100644 --- a/drivers/vdpa/pds/vdpa_dev.c +++ b/drivers/vdpa/pds/vdpa_dev.c @@ -51,7 +51,7 @@ static int pds_vdpa_register_event_handler(struct pds_vdpa_device *pdsv) err = pdsc_register_notify(nb); if (err) { nb->notifier_call = NULL; - dev_err(dev, "failed to register pds event handler: %ps\n", + dev_err(dev, "failed to register pds event handler: %pe\n", ERR_PTR(err)); return -EINVAL; } diff --git a/drivers/vdpa/vdpa_user/vduse_dev.c b/drivers/vdpa/vdpa_user/vduse_dev.c index e7bced0b5542..ae357d014564 100644 --- a/drivers/vdpa/vdpa_user/vduse_dev.c +++ b/drivers/vdpa/vdpa_user/vduse_dev.c @@ -2173,7 +2173,8 @@ static int vduse_init(void) if (!vduse_irq_wq) goto err_wq; - vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", WQ_HIGHPRI, 0); + vduse_irq_bound_wq = alloc_workqueue("vduse-irq-bound", + WQ_HIGHPRI | WQ_PERCPU, 0); if (!vduse_irq_bound_wq) goto err_bound_wq; diff --git a/drivers/vfio/pci/Kconfig b/drivers/vfio/pci/Kconfig index 2b9fca00e9e8..1e82b44bda1a 100644 --- a/drivers/vfio/pci/Kconfig +++ b/drivers/vfio/pci/Kconfig @@ -70,4 +70,6 @@ source "drivers/vfio/pci/nvgrace-gpu/Kconfig" source "drivers/vfio/pci/qat/Kconfig" +source "drivers/vfio/pci/xe/Kconfig" + endmenu diff --git a/drivers/vfio/pci/Makefile b/drivers/vfio/pci/Makefile index 53f59226ae01..e0a0757dd1d2 100644 --- a/drivers/vfio/pci/Makefile +++ b/drivers/vfio/pci/Makefile @@ -20,3 +20,5 @@ obj-$(CONFIG_VIRTIO_VFIO_PCI) += virtio/ obj-$(CONFIG_NVGRACE_GPU_VFIO_PCI) += nvgrace-gpu/ obj-$(CONFIG_QAT_VFIO_PCI) += qat/ + +obj-$(CONFIG_XE_VFIO_PCI) += xe/ diff --git a/drivers/vfio/pci/xe/Kconfig b/drivers/vfio/pci/xe/Kconfig new file mode 100644 index 000000000000..cc9b6dac6ed3 --- /dev/null +++ b/drivers/vfio/pci/xe/Kconfig @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0-only +config XE_VFIO_PCI + tristate "VFIO support for Intel Graphics" + depends on DRM_XE && PCI_IOV + select VFIO_PCI_CORE + help + This option enables device specific VFIO driver variant for Intel Graphics. + In addition to generic VFIO PCI functionality, it implements VFIO + migration uAPI allowing userspace to enable migration for + Intel Graphics SR-IOV Virtual Functions supported by the Xe driver. + + If you don't know what to do here, say N. diff --git a/drivers/vfio/pci/xe/Makefile b/drivers/vfio/pci/xe/Makefile new file mode 100644 index 000000000000..13aa0fd192cd --- /dev/null +++ b/drivers/vfio/pci/xe/Makefile @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_XE_VFIO_PCI) += xe-vfio-pci.o +xe-vfio-pci-y := main.o diff --git a/drivers/vfio/pci/xe/main.c b/drivers/vfio/pci/xe/main.c new file mode 100644 index 000000000000..0156b53c678b --- /dev/null +++ b/drivers/vfio/pci/xe/main.c @@ -0,0 +1,573 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright © 2025 Intel Corporation + */ + +#include <linux/anon_inodes.h> +#include <linux/delay.h> +#include <linux/file.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/sizes.h> +#include <linux/types.h> +#include <linux/vfio.h> +#include <linux/vfio_pci_core.h> + +#include <drm/intel/xe_sriov_vfio.h> +#include <drm/intel/pciids.h> + +struct xe_vfio_pci_migration_file { + struct file *filp; + /* serializes accesses to migration data */ + struct mutex lock; + struct xe_vfio_pci_core_device *xe_vdev; + u8 disabled:1; +}; + +struct xe_vfio_pci_core_device { + struct vfio_pci_core_device core_device; + struct xe_device *xe; + /* PF internal control uses vfid index starting from 1 */ + unsigned int vfid; + u8 deferred_reset:1; + /* protects migration state */ + struct mutex state_mutex; + enum vfio_device_mig_state mig_state; + /* protects the reset_done flow */ + spinlock_t reset_lock; + struct xe_vfio_pci_migration_file *migf; +}; + +#define xe_vdev_to_dev(xe_vdev) (&(xe_vdev)->core_device.pdev->dev) + +static void xe_vfio_pci_disable_file(struct xe_vfio_pci_migration_file *migf) +{ + mutex_lock(&migf->lock); + migf->disabled = true; + mutex_unlock(&migf->lock); +} + +static void xe_vfio_pci_put_file(struct xe_vfio_pci_core_device *xe_vdev) +{ + xe_vfio_pci_disable_file(xe_vdev->migf); + fput(xe_vdev->migf->filp); + xe_vdev->migf = NULL; +} + +static void xe_vfio_pci_reset(struct xe_vfio_pci_core_device *xe_vdev) +{ + if (xe_vdev->migf) + xe_vfio_pci_put_file(xe_vdev); + + xe_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; +} + +static void xe_vfio_pci_state_mutex_lock(struct xe_vfio_pci_core_device *xe_vdev) +{ + mutex_lock(&xe_vdev->state_mutex); +} + +/* + * This function is called in all state_mutex unlock cases to + * handle a 'deferred_reset' if exists. + */ +static void xe_vfio_pci_state_mutex_unlock(struct xe_vfio_pci_core_device *xe_vdev) +{ +again: + spin_lock(&xe_vdev->reset_lock); + if (xe_vdev->deferred_reset) { + xe_vdev->deferred_reset = false; + spin_unlock(&xe_vdev->reset_lock); + xe_vfio_pci_reset(xe_vdev); + goto again; + } + mutex_unlock(&xe_vdev->state_mutex); + spin_unlock(&xe_vdev->reset_lock); +} + +static void xe_vfio_pci_reset_done(struct pci_dev *pdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = pci_get_drvdata(pdev); + int ret; + + if (!pdev->is_virtfn) + return; + + /* + * VF FLR requires additional processing done by PF driver. + * The processing is done after FLR is already finished from PCIe + * perspective. + * In order to avoid a scenario where VF is used while PF processing + * is still in progress, additional synchronization point is needed. + */ + ret = xe_sriov_vfio_wait_flr_done(xe_vdev->xe, xe_vdev->vfid); + if (ret) + dev_err(&pdev->dev, "Failed to wait for FLR: %d\n", ret); + + if (!xe_vdev->vfid) + return; + + /* + * As the higher VFIO layers are holding locks across reset and using + * those same locks with the mm_lock we need to prevent ABBA deadlock + * with the state_mutex and mm_lock. + * In case the state_mutex was taken already we defer the cleanup work + * to the unlock flow of the other running context. + */ + spin_lock(&xe_vdev->reset_lock); + xe_vdev->deferred_reset = true; + if (!mutex_trylock(&xe_vdev->state_mutex)) { + spin_unlock(&xe_vdev->reset_lock); + return; + } + spin_unlock(&xe_vdev->reset_lock); + xe_vfio_pci_state_mutex_unlock(xe_vdev); + + xe_vfio_pci_reset(xe_vdev); +} + +static const struct pci_error_handlers xe_vfio_pci_err_handlers = { + .reset_done = xe_vfio_pci_reset_done, + .error_detected = vfio_pci_core_aer_err_detected, +}; + +static int xe_vfio_pci_open_device(struct vfio_device *core_vdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + struct vfio_pci_core_device *vdev = &xe_vdev->core_device; + int ret; + + ret = vfio_pci_core_enable(vdev); + if (ret) + return ret; + + xe_vdev->mig_state = VFIO_DEVICE_STATE_RUNNING; + + vfio_pci_core_finish_enable(vdev); + + return 0; +} + +static void xe_vfio_pci_close_device(struct vfio_device *core_vdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_state_mutex_lock(xe_vdev); + xe_vfio_pci_reset(xe_vdev); + xe_vfio_pci_state_mutex_unlock(xe_vdev); + vfio_pci_core_close_device(core_vdev); +} + +static int xe_vfio_pci_release_file(struct inode *inode, struct file *filp) +{ + struct xe_vfio_pci_migration_file *migf = filp->private_data; + + mutex_destroy(&migf->lock); + kfree(migf); + + return 0; +} + +static ssize_t xe_vfio_pci_save_read(struct file *filp, char __user *buf, size_t len, loff_t *pos) +{ + struct xe_vfio_pci_migration_file *migf = filp->private_data; + ssize_t ret; + + if (pos) + return -ESPIPE; + + mutex_lock(&migf->lock); + if (migf->disabled) { + mutex_unlock(&migf->lock); + return -ENODEV; + } + + ret = xe_sriov_vfio_data_read(migf->xe_vdev->xe, migf->xe_vdev->vfid, buf, len); + mutex_unlock(&migf->lock); + + return ret; +} + +static const struct file_operations xe_vfio_pci_save_fops = { + .owner = THIS_MODULE, + .read = xe_vfio_pci_save_read, + .release = xe_vfio_pci_release_file, + .llseek = noop_llseek, +}; + +static ssize_t xe_vfio_pci_resume_write(struct file *filp, const char __user *buf, + size_t len, loff_t *pos) +{ + struct xe_vfio_pci_migration_file *migf = filp->private_data; + ssize_t ret; + + if (pos) + return -ESPIPE; + + mutex_lock(&migf->lock); + if (migf->disabled) { + mutex_unlock(&migf->lock); + return -ENODEV; + } + + ret = xe_sriov_vfio_data_write(migf->xe_vdev->xe, migf->xe_vdev->vfid, buf, len); + mutex_unlock(&migf->lock); + + return ret; +} + +static const struct file_operations xe_vfio_pci_resume_fops = { + .owner = THIS_MODULE, + .write = xe_vfio_pci_resume_write, + .release = xe_vfio_pci_release_file, + .llseek = noop_llseek, +}; + +static const char *vfio_dev_state_str(u32 state) +{ + switch (state) { + case VFIO_DEVICE_STATE_RUNNING: return "running"; + case VFIO_DEVICE_STATE_RUNNING_P2P: return "running_p2p"; + case VFIO_DEVICE_STATE_STOP_COPY: return "stopcopy"; + case VFIO_DEVICE_STATE_STOP: return "stop"; + case VFIO_DEVICE_STATE_RESUMING: return "resuming"; + case VFIO_DEVICE_STATE_ERROR: return "error"; + default: return ""; + } +} + +enum xe_vfio_pci_file_type { + XE_VFIO_FILE_SAVE = 0, + XE_VFIO_FILE_RESUME, +}; + +static struct xe_vfio_pci_migration_file * +xe_vfio_pci_alloc_file(struct xe_vfio_pci_core_device *xe_vdev, + enum xe_vfio_pci_file_type type) +{ + struct xe_vfio_pci_migration_file *migf; + const struct file_operations *fops; + int flags; + + migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT); + if (!migf) + return ERR_PTR(-ENOMEM); + + fops = type == XE_VFIO_FILE_SAVE ? &xe_vfio_pci_save_fops : &xe_vfio_pci_resume_fops; + flags = type == XE_VFIO_FILE_SAVE ? O_RDONLY : O_WRONLY; + migf->filp = anon_inode_getfile("xe_vfio_mig", fops, migf, flags); + if (IS_ERR(migf->filp)) { + kfree(migf); + return ERR_CAST(migf->filp); + } + + mutex_init(&migf->lock); + migf->xe_vdev = xe_vdev; + xe_vdev->migf = migf; + + stream_open(migf->filp->f_inode, migf->filp); + + return migf; +} + +static struct file * +xe_vfio_set_state(struct xe_vfio_pci_core_device *xe_vdev, u32 new) +{ + u32 cur = xe_vdev->mig_state; + int ret; + + dev_dbg(xe_vdev_to_dev(xe_vdev), + "state: %s->%s\n", vfio_dev_state_str(cur), vfio_dev_state_str(new)); + + /* + * "STOP" handling is reused for "RUNNING_P2P", as the device doesn't + * have the capability to selectively block outgoing p2p DMA transfers. + * While the device is allowing BAR accesses when the VF is stopped, it + * is not processing any new workload requests, effectively stopping + * any outgoing DMA transfers (not just p2p). + * Any VRAM / MMIO accesses occurring during "RUNNING_P2P" are kept and + * will be migrated to target VF during stop-copy. + */ + if (cur == VFIO_DEVICE_STATE_RUNNING && new == VFIO_DEVICE_STATE_RUNNING_P2P) { + ret = xe_sriov_vfio_suspend_device(xe_vdev->xe, xe_vdev->vfid); + if (ret) + goto err; + + return NULL; + } + + if ((cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_STOP) || + (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RUNNING_P2P)) + return NULL; + + if (cur == VFIO_DEVICE_STATE_RUNNING_P2P && new == VFIO_DEVICE_STATE_RUNNING) { + ret = xe_sriov_vfio_resume_device(xe_vdev->xe, xe_vdev->vfid); + if (ret) + goto err; + + return NULL; + } + + if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_STOP_COPY) { + struct xe_vfio_pci_migration_file *migf; + + migf = xe_vfio_pci_alloc_file(xe_vdev, XE_VFIO_FILE_SAVE); + if (IS_ERR(migf)) { + ret = PTR_ERR(migf); + goto err; + } + get_file(migf->filp); + + ret = xe_sriov_vfio_stop_copy_enter(xe_vdev->xe, xe_vdev->vfid); + if (ret) { + fput(migf->filp); + goto err; + } + + return migf->filp; + } + + if (cur == VFIO_DEVICE_STATE_STOP_COPY && new == VFIO_DEVICE_STATE_STOP) { + if (xe_vdev->migf) + xe_vfio_pci_put_file(xe_vdev); + + ret = xe_sriov_vfio_stop_copy_exit(xe_vdev->xe, xe_vdev->vfid); + if (ret) + goto err; + + return NULL; + } + + if (cur == VFIO_DEVICE_STATE_STOP && new == VFIO_DEVICE_STATE_RESUMING) { + struct xe_vfio_pci_migration_file *migf; + + migf = xe_vfio_pci_alloc_file(xe_vdev, XE_VFIO_FILE_RESUME); + if (IS_ERR(migf)) { + ret = PTR_ERR(migf); + goto err; + } + get_file(migf->filp); + + ret = xe_sriov_vfio_resume_data_enter(xe_vdev->xe, xe_vdev->vfid); + if (ret) { + fput(migf->filp); + goto err; + } + + return migf->filp; + } + + if (cur == VFIO_DEVICE_STATE_RESUMING && new == VFIO_DEVICE_STATE_STOP) { + if (xe_vdev->migf) + xe_vfio_pci_put_file(xe_vdev); + + ret = xe_sriov_vfio_resume_data_exit(xe_vdev->xe, xe_vdev->vfid); + if (ret) + goto err; + + return NULL; + } + + WARN(true, "Unknown state transition %d->%d", cur, new); + return ERR_PTR(-EINVAL); + +err: + dev_dbg(xe_vdev_to_dev(xe_vdev), + "Failed to transition state: %s->%s err=%d\n", + vfio_dev_state_str(cur), vfio_dev_state_str(new), ret); + return ERR_PTR(ret); +} + +static struct file * +xe_vfio_pci_set_device_state(struct vfio_device *core_vdev, + enum vfio_device_mig_state new_state) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + enum vfio_device_mig_state next_state; + struct file *f = NULL; + int ret; + + xe_vfio_pci_state_mutex_lock(xe_vdev); + while (new_state != xe_vdev->mig_state) { + ret = vfio_mig_get_next_state(core_vdev, xe_vdev->mig_state, + new_state, &next_state); + if (ret) { + xe_sriov_vfio_error(xe_vdev->xe, xe_vdev->vfid); + f = ERR_PTR(ret); + break; + } + f = xe_vfio_set_state(xe_vdev, next_state); + if (IS_ERR(f)) + break; + + xe_vdev->mig_state = next_state; + + /* Multiple state transitions with non-NULL file in the middle */ + if (f && new_state != xe_vdev->mig_state) { + fput(f); + f = ERR_PTR(-EINVAL); + break; + } + } + xe_vfio_pci_state_mutex_unlock(xe_vdev); + + return f; +} + +static int xe_vfio_pci_get_device_state(struct vfio_device *core_vdev, + enum vfio_device_mig_state *curr_state) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_state_mutex_lock(xe_vdev); + *curr_state = xe_vdev->mig_state; + xe_vfio_pci_state_mutex_unlock(xe_vdev); + + return 0; +} + +static int xe_vfio_pci_get_data_size(struct vfio_device *vdev, + unsigned long *stop_copy_length) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_state_mutex_lock(xe_vdev); + *stop_copy_length = xe_sriov_vfio_stop_copy_size(xe_vdev->xe, xe_vdev->vfid); + xe_vfio_pci_state_mutex_unlock(xe_vdev); + + return 0; +} + +static const struct vfio_migration_ops xe_vfio_pci_migration_ops = { + .migration_set_state = xe_vfio_pci_set_device_state, + .migration_get_state = xe_vfio_pci_get_device_state, + .migration_get_data_size = xe_vfio_pci_get_data_size, +}; + +static void xe_vfio_pci_migration_init(struct xe_vfio_pci_core_device *xe_vdev) +{ + struct vfio_device *core_vdev = &xe_vdev->core_device.vdev; + struct pci_dev *pdev = to_pci_dev(core_vdev->dev); + struct xe_device *xe = xe_sriov_vfio_get_pf(pdev); + + if (!xe) + return; + if (!xe_sriov_vfio_migration_supported(xe)) + return; + + mutex_init(&xe_vdev->state_mutex); + spin_lock_init(&xe_vdev->reset_lock); + + /* PF internal control uses vfid index starting from 1 */ + xe_vdev->vfid = pci_iov_vf_id(pdev) + 1; + xe_vdev->xe = xe; + + core_vdev->migration_flags = VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P; + core_vdev->mig_ops = &xe_vfio_pci_migration_ops; +} + +static void xe_vfio_pci_migration_fini(struct xe_vfio_pci_core_device *xe_vdev) +{ + if (!xe_vdev->vfid) + return; + + mutex_destroy(&xe_vdev->state_mutex); +} + +static int xe_vfio_pci_init_dev(struct vfio_device *core_vdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_migration_init(xe_vdev); + + return vfio_pci_core_init_dev(core_vdev); +} + +static void xe_vfio_pci_release_dev(struct vfio_device *core_vdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = + container_of(core_vdev, struct xe_vfio_pci_core_device, core_device.vdev); + + xe_vfio_pci_migration_fini(xe_vdev); +} + +static const struct vfio_device_ops xe_vfio_pci_ops = { + .name = "xe-vfio-pci", + .init = xe_vfio_pci_init_dev, + .release = xe_vfio_pci_release_dev, + .open_device = xe_vfio_pci_open_device, + .close_device = xe_vfio_pci_close_device, + .ioctl = vfio_pci_core_ioctl, + .device_feature = vfio_pci_core_ioctl_feature, + .read = vfio_pci_core_read, + .write = vfio_pci_core_write, + .mmap = vfio_pci_core_mmap, + .request = vfio_pci_core_request, + .match = vfio_pci_core_match, + .match_token_uuid = vfio_pci_core_match_token_uuid, + .bind_iommufd = vfio_iommufd_physical_bind, + .unbind_iommufd = vfio_iommufd_physical_unbind, + .attach_ioas = vfio_iommufd_physical_attach_ioas, + .detach_ioas = vfio_iommufd_physical_detach_ioas, +}; + +static int xe_vfio_pci_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct xe_vfio_pci_core_device *xe_vdev; + int ret; + + xe_vdev = vfio_alloc_device(xe_vfio_pci_core_device, core_device.vdev, &pdev->dev, + &xe_vfio_pci_ops); + if (IS_ERR(xe_vdev)) + return PTR_ERR(xe_vdev); + + dev_set_drvdata(&pdev->dev, &xe_vdev->core_device); + + ret = vfio_pci_core_register_device(&xe_vdev->core_device); + if (ret) { + vfio_put_device(&xe_vdev->core_device.vdev); + return ret; + } + + return 0; +} + +static void xe_vfio_pci_remove(struct pci_dev *pdev) +{ + struct xe_vfio_pci_core_device *xe_vdev = pci_get_drvdata(pdev); + + vfio_pci_core_unregister_device(&xe_vdev->core_device); + vfio_put_device(&xe_vdev->core_device.vdev); +} + +#define INTEL_PCI_VFIO_DEVICE(_id) { \ + PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_INTEL, (_id)) \ +} + +static const struct pci_device_id xe_vfio_pci_table[] = { + INTEL_PTL_IDS(INTEL_PCI_VFIO_DEVICE), + INTEL_WCL_IDS(INTEL_PCI_VFIO_DEVICE), + INTEL_BMG_IDS(INTEL_PCI_VFIO_DEVICE), + {} +}; +MODULE_DEVICE_TABLE(pci, xe_vfio_pci_table); + +static struct pci_driver xe_vfio_pci_driver = { + .name = "xe-vfio-pci", + .id_table = xe_vfio_pci_table, + .probe = xe_vfio_pci_probe, + .remove = xe_vfio_pci_remove, + .err_handler = &xe_vfio_pci_err_handlers, + .driver_managed_dma = true, +}; +module_pci_driver(xe_vfio_pci_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Michał Winiarski <michal.winiarski@intel.com>"); +MODULE_DESCRIPTION("VFIO PCI driver with migration support for Intel Graphics"); diff --git a/drivers/vhost/net.c b/drivers/vhost/net.c index 8f7f50acb6d6..7f886d3dba7d 100644 --- a/drivers/vhost/net.c +++ b/drivers/vhost/net.c @@ -69,15 +69,15 @@ MODULE_PARM_DESC(experimental_zcopytx, "Enable Zero Copy TX;" #define VHOST_DMA_IS_DONE(len) ((__force u32)(len) >= (__force u32)VHOST_DMA_DONE_LEN) -static const u64 vhost_net_features[VIRTIO_FEATURES_DWORDS] = { - VHOST_FEATURES | - (1ULL << VHOST_NET_F_VIRTIO_NET_HDR) | - (1ULL << VIRTIO_NET_F_MRG_RXBUF) | - (1ULL << VIRTIO_F_ACCESS_PLATFORM) | - (1ULL << VIRTIO_F_RING_RESET) | - (1ULL << VIRTIO_F_IN_ORDER), - VIRTIO_BIT(VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO) | - VIRTIO_BIT(VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO), +static const int vhost_net_bits[] = { + VHOST_FEATURES, + VHOST_NET_F_VIRTIO_NET_HDR, + VIRTIO_NET_F_MRG_RXBUF, + VIRTIO_F_ACCESS_PLATFORM, + VIRTIO_F_RING_RESET, + VIRTIO_F_IN_ORDER, + VIRTIO_NET_F_GUEST_UDP_TUNNEL_GSO, + VIRTIO_NET_F_HOST_UDP_TUNNEL_GSO }; enum { @@ -1731,7 +1731,8 @@ out: static long vhost_net_ioctl(struct file *f, unsigned int ioctl, unsigned long arg) { - u64 all_features[VIRTIO_FEATURES_DWORDS]; + const DEFINE_VHOST_FEATURES_ARRAY(vhost_net_features, vhost_net_bits); + u64 all_features[VIRTIO_FEATURES_U64S]; struct vhost_net *n = f->private_data; void __user *argp = (void __user *)arg; u64 __user *featurep = argp; @@ -1763,7 +1764,7 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, /* Copy the net features, up to the user-provided buffer size */ argp += sizeof(u64); - copied = min(count, VIRTIO_FEATURES_DWORDS); + copied = min(count, (u64)VIRTIO_FEATURES_U64S); if (copy_to_user(argp, vhost_net_features, copied * sizeof(u64))) return -EFAULT; @@ -1778,13 +1779,13 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, virtio_features_zero(all_features); argp += sizeof(u64); - copied = min(count, VIRTIO_FEATURES_DWORDS); + copied = min(count, (u64)VIRTIO_FEATURES_U64S); if (copy_from_user(all_features, argp, copied * sizeof(u64))) return -EFAULT; /* * Any feature specified by user-space above - * VIRTIO_FEATURES_MAX is not supported by definition. + * VIRTIO_FEATURES_BITS is not supported by definition. */ for (i = copied; i < count; ++i) { if (copy_from_user(&features, featurep + 1 + i, @@ -1794,7 +1795,7 @@ static long vhost_net_ioctl(struct file *f, unsigned int ioctl, return -EOPNOTSUPP; } - for (i = 0; i < VIRTIO_FEATURES_DWORDS; i++) + for (i = 0; i < VIRTIO_FEATURES_U64S; i++) if (all_features[i] & ~vhost_net_features[i]) return -EOPNOTSUPP; diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c index 98e4f68f4e3c..f43c1fe9fad9 100644 --- a/drivers/vhost/scsi.c +++ b/drivers/vhost/scsi.c @@ -197,11 +197,14 @@ enum { }; /* Note: can't set VIRTIO_F_VERSION_1 yet, since that implies ANY_LAYOUT. */ -enum { - VHOST_SCSI_FEATURES = VHOST_FEATURES | (1ULL << VIRTIO_SCSI_F_HOTPLUG) | - (1ULL << VIRTIO_SCSI_F_T10_PI) +static const int vhost_scsi_bits[] = { + VHOST_FEATURES, + VIRTIO_SCSI_F_HOTPLUG, + VIRTIO_SCSI_F_T10_PI }; +#define VHOST_SCSI_FEATURES VHOST_FEATURES_U64(vhost_scsi_bits, 0) + #define VHOST_SCSI_MAX_TARGET 256 #define VHOST_SCSI_MAX_IO_VQ 1024 #define VHOST_SCSI_MAX_EVENT 128 diff --git a/drivers/vhost/test.c b/drivers/vhost/test.c index 42c955a5b211..1e4e36edbcd2 100644 --- a/drivers/vhost/test.c +++ b/drivers/vhost/test.c @@ -28,6 +28,12 @@ */ #define VHOST_TEST_PKT_WEIGHT 256 +static const int vhost_test_bits[] = { + VHOST_FEATURES +}; + +#define VHOST_TEST_FEATURES VHOST_FEATURES_U64(vhost_test_bits, 0) + enum { VHOST_TEST_VQ = 0, VHOST_TEST_VQ_MAX = 1, @@ -328,14 +334,14 @@ static long vhost_test_ioctl(struct file *f, unsigned int ioctl, return -EFAULT; return vhost_test_set_backend(n, backend.index, backend.fd); case VHOST_GET_FEATURES: - features = VHOST_FEATURES; + features = VHOST_TEST_FEATURES; if (copy_to_user(featurep, &features, sizeof features)) return -EFAULT; return 0; case VHOST_SET_FEATURES: if (copy_from_user(&features, featurep, sizeof features)) return -EFAULT; - if (features & ~VHOST_FEATURES) + if (features & ~VHOST_TEST_FEATURES) return -EOPNOTSUPP; return vhost_test_set_features(n, features); case VHOST_RESET_OWNER: diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c index a78226b37739..bccdc9eab267 100644 --- a/drivers/vhost/vhost.c +++ b/drivers/vhost/vhost.c @@ -804,11 +804,13 @@ static int vhost_kthread_worker_create(struct vhost_worker *worker, ret = vhost_attach_task_to_cgroups(worker); if (ret) - goto stop_worker; + goto free_id; worker->id = id; return 0; +free_id: + xa_erase(&dev->worker_xa, id); stop_worker: vhost_kthread_do_stop(worker); return ret; diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h index b49f08e4a1b4..4fe99765c5c7 100644 --- a/drivers/vhost/vhost.h +++ b/drivers/vhost/vhost.h @@ -14,6 +14,7 @@ #include <linux/atomic.h> #include <linux/vhost_iotlb.h> #include <linux/irqbypass.h> +#include <linux/unroll.h> struct vhost_work; struct vhost_task; @@ -287,14 +288,39 @@ void vhost_iotlb_map_free(struct vhost_iotlb *iotlb, eventfd_signal((vq)->error_ctx);\ } while (0) -enum { - VHOST_FEATURES = (1ULL << VIRTIO_F_NOTIFY_ON_EMPTY) | - (1ULL << VIRTIO_RING_F_INDIRECT_DESC) | - (1ULL << VIRTIO_RING_F_EVENT_IDX) | - (1ULL << VHOST_F_LOG_ALL) | - (1ULL << VIRTIO_F_ANY_LAYOUT) | - (1ULL << VIRTIO_F_VERSION_1) -}; +#define VHOST_FEATURES \ + VIRTIO_F_NOTIFY_ON_EMPTY, \ + VIRTIO_RING_F_INDIRECT_DESC, \ + VIRTIO_RING_F_EVENT_IDX, \ + VHOST_F_LOG_ALL, \ + VIRTIO_F_ANY_LAYOUT, \ + VIRTIO_F_VERSION_1 + +static inline u64 vhost_features_u64(const int *features, int size, int idx) +{ + u64 res = 0; + + unrolled_count(VIRTIO_FEATURES_BITS) + for (int i = 0; i < size; ++i) { + int bit = features[i]; + + if (virtio_features_chk_bit(bit) && VIRTIO_U64(bit) == idx) + res |= VIRTIO_BIT(bit); + } + return res; +} + +#define VHOST_FEATURES_U64(features, idx) \ + vhost_features_u64(features, ARRAY_SIZE(features), idx) + +#define DEFINE_VHOST_FEATURES_ARRAY_ENTRY(idx, features) \ + [idx] = VHOST_FEATURES_U64(features, idx), + +#define DEFINE_VHOST_FEATURES_ARRAY(array, features) \ + u64 array[VIRTIO_FEATURES_U64S] = { \ + UNROLL(VIRTIO_FEATURES_U64S, \ + DEFINE_VHOST_FEATURES_ARRAY_ENTRY, features) \ + } /** * vhost_vq_set_backend - Set backend. diff --git a/drivers/vhost/vsock.c b/drivers/vhost/vsock.c index ae01457ea2cd..0298ddc34824 100644 --- a/drivers/vhost/vsock.c +++ b/drivers/vhost/vsock.c @@ -29,12 +29,14 @@ */ #define VHOST_VSOCK_PKT_WEIGHT 256 -enum { - VHOST_VSOCK_FEATURES = VHOST_FEATURES | - (1ULL << VIRTIO_F_ACCESS_PLATFORM) | - (1ULL << VIRTIO_VSOCK_F_SEQPACKET) +static const int vhost_vsock_bits[] = { + VHOST_FEATURES, + VIRTIO_F_ACCESS_PLATFORM, + VIRTIO_VSOCK_F_SEQPACKET }; +#define VHOST_VSOCK_FEATURES VHOST_FEATURES_U64(vhost_vsock_bits, 0) + enum { VHOST_VSOCK_BACKEND_FEATURES = (1ULL << VHOST_BACKEND_F_IOTLB_MSG_V2) }; diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c index a09eb4d62f82..5bdc6b82b30b 100644 --- a/drivers/virtio/virtio.c +++ b/drivers/virtio/virtio.c @@ -53,7 +53,7 @@ static ssize_t features_show(struct device *_d, /* We actually represent this as a bitstring, as it could be * arbitrary length in future. */ - for (i = 0; i < VIRTIO_FEATURES_MAX; i++) + for (i = 0; i < VIRTIO_FEATURES_BITS; i++) len += sysfs_emit_at(buf, len, "%c", __virtio_test_bit(dev, i) ? '1' : '0'); len += sysfs_emit_at(buf, len, "\n"); @@ -272,8 +272,8 @@ static int virtio_dev_probe(struct device *_d) int err, i; struct virtio_device *dev = dev_to_virtio(_d); struct virtio_driver *drv = drv_to_virtio(dev->dev.driver); - u64 device_features[VIRTIO_FEATURES_DWORDS]; - u64 driver_features[VIRTIO_FEATURES_DWORDS]; + u64 device_features[VIRTIO_FEATURES_U64S]; + u64 driver_features[VIRTIO_FEATURES_U64S]; u64 driver_features_legacy; /* We have a driver! */ @@ -286,7 +286,7 @@ static int virtio_dev_probe(struct device *_d) virtio_features_zero(driver_features); for (i = 0; i < drv->feature_table_size; i++) { unsigned int f = drv->feature_table[i]; - if (!WARN_ON_ONCE(f >= VIRTIO_FEATURES_MAX)) + if (!WARN_ON_ONCE(f >= VIRTIO_FEATURES_BITS)) virtio_features_set_bit(driver_features, f); } @@ -303,7 +303,7 @@ static int virtio_dev_probe(struct device *_d) } if (virtio_features_test_bit(device_features, VIRTIO_F_VERSION_1)) { - for (i = 0; i < VIRTIO_FEATURES_DWORDS; ++i) + for (i = 0; i < VIRTIO_FEATURES_U64S; ++i) dev->features_array[i] = driver_features[i] & device_features[i]; } else { @@ -325,7 +325,7 @@ static int virtio_dev_probe(struct device *_d) goto err; if (drv->validate) { - u64 features[VIRTIO_FEATURES_DWORDS]; + u64 features[VIRTIO_FEATURES_U64S]; virtio_features_copy(features, dev->features_array); err = drv->validate(dev); diff --git a/drivers/virtio/virtio_balloon.c b/drivers/virtio/virtio_balloon.c index 1b93d8c64361..74fe59f5a78c 100644 --- a/drivers/virtio/virtio_balloon.c +++ b/drivers/virtio/virtio_balloon.c @@ -983,7 +983,8 @@ static int virtballoon_probe(struct virtio_device *vdev) goto out_del_vqs; } vb->balloon_wq = alloc_workqueue("balloon-wq", - WQ_FREEZABLE | WQ_CPU_INTENSIVE, 0); + WQ_FREEZABLE | WQ_CPU_INTENSIVE | WQ_PERCPU, + 0); if (!vb->balloon_wq) { err = -ENOMEM; goto out_del_vqs; diff --git a/drivers/virtio/virtio_debug.c b/drivers/virtio/virtio_debug.c index d58713ddf2e5..ccf1955a1183 100644 --- a/drivers/virtio/virtio_debug.c +++ b/drivers/virtio/virtio_debug.c @@ -8,12 +8,12 @@ static struct dentry *virtio_debugfs_dir; static int virtio_debug_device_features_show(struct seq_file *s, void *data) { - u64 device_features[VIRTIO_FEATURES_DWORDS]; + u64 device_features[VIRTIO_FEATURES_U64S]; struct virtio_device *dev = s->private; unsigned int i; virtio_get_features(dev, device_features); - for (i = 0; i < VIRTIO_FEATURES_MAX; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS; i++) { if (virtio_features_test_bit(device_features, i)) seq_printf(s, "%u\n", i); } @@ -26,7 +26,7 @@ static int virtio_debug_filter_features_show(struct seq_file *s, void *data) struct virtio_device *dev = s->private; unsigned int i; - for (i = 0; i < VIRTIO_FEATURES_MAX; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS; i++) { if (virtio_features_test_bit(dev->debugfs_filter_features, i)) seq_printf(s, "%u\n", i); } @@ -50,7 +50,7 @@ static int virtio_debug_filter_feature_add(void *data, u64 val) { struct virtio_device *dev = data; - if (val >= VIRTIO_FEATURES_MAX) + if (val >= VIRTIO_FEATURES_BITS) return -EINVAL; virtio_features_set_bit(dev->debugfs_filter_features, val); @@ -64,7 +64,7 @@ static int virtio_debug_filter_feature_del(void *data, u64 val) { struct virtio_device *dev = data; - if (val >= VIRTIO_FEATURES_MAX) + if (val >= VIRTIO_FEATURES_BITS) return -EINVAL; virtio_features_clear_bit(dev->debugfs_filter_features, val); diff --git a/drivers/virtio/virtio_pci_modern_dev.c b/drivers/virtio/virtio_pci_modern_dev.c index 9e503b7a58d8..413a8c353463 100644 --- a/drivers/virtio/virtio_pci_modern_dev.c +++ b/drivers/virtio/virtio_pci_modern_dev.c @@ -401,7 +401,7 @@ void vp_modern_get_extended_features(struct virtio_pci_modern_device *mdev, int i; virtio_features_zero(features); - for (i = 0; i < VIRTIO_FEATURES_WORDS; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS / 32; i++) { u64 cur; vp_iowrite32(i, &cfg->device_feature_select); @@ -427,7 +427,7 @@ vp_modern_get_driver_extended_features(struct virtio_pci_modern_device *mdev, int i; virtio_features_zero(features); - for (i = 0; i < VIRTIO_FEATURES_WORDS; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS / 32; i++) { u64 cur; vp_iowrite32(i, &cfg->guest_feature_select); @@ -448,7 +448,7 @@ void vp_modern_set_extended_features(struct virtio_pci_modern_device *mdev, struct virtio_pci_common_cfg __iomem *cfg = mdev->common; int i; - for (i = 0; i < VIRTIO_FEATURES_WORDS; i++) { + for (i = 0; i < VIRTIO_FEATURES_BITS / 32; i++) { u32 cur = features[i >> 1] >> (32 * (i & 1)); vp_iowrite32(i, &cfg->guest_feature_select); diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c index 7b6205253b46..ddab68959671 100644 --- a/drivers/virtio/virtio_ring.c +++ b/drivers/virtio/virtio_ring.c @@ -3166,6 +3166,7 @@ EXPORT_SYMBOL_GPL(virtqueue_map_alloc_coherent); * @vdev: the virtio device we are talking to * @map: metadata for performing mapping * @size: the size of the buffer + * @vaddr: the virtual address that needs to be freed * @map_handle: the mapped address that needs to be freed * */ @@ -3190,7 +3191,7 @@ EXPORT_SYMBOL_GPL(virtqueue_map_free_coherent); * @dir: mapping direction * @attrs: mapping attributes * - * Returns mapped address. Caller should check that by virtqueue_mapping_error(). + * Returns mapped address. Caller should check that by virtqueue_map_mapping_error(). */ dma_addr_t virtqueue_map_page_attrs(const struct virtqueue *_vq, struct page *page, @@ -3249,7 +3250,7 @@ EXPORT_SYMBOL_GPL(virtqueue_unmap_page_attrs); * The caller calls this to do dma mapping in advance. The DMA address can be * passed to this _vq when it is in pre-mapped mode. * - * return mapped address. Caller should check that by virtqueue_mapping_error(). + * return mapped address. Caller should check that by virtqueue_map_mapping_error(). */ dma_addr_t virtqueue_map_single_attrs(const struct virtqueue *_vq, void *ptr, size_t size, @@ -3299,7 +3300,7 @@ void virtqueue_unmap_single_attrs(const struct virtqueue *_vq, EXPORT_SYMBOL_GPL(virtqueue_unmap_single_attrs); /** - * virtqueue_mapping_error - check dma address + * virtqueue_map_mapping_error - check dma address * @_vq: the struct virtqueue we're talking about. * @addr: DMA address * diff --git a/drivers/virtio/virtio_vdpa.c b/drivers/virtio/virtio_vdpa.c index f9a29045eca0..0a801f67b599 100644 --- a/drivers/virtio/virtio_vdpa.c +++ b/drivers/virtio/virtio_vdpa.c @@ -80,7 +80,7 @@ static void virtio_vdpa_set_status(struct virtio_device *vdev, u8 status) { struct vdpa_device *vdpa = vd_get_vdpa(vdev); - return vdpa_set_status(vdpa, status); + vdpa_set_status(vdpa, status); } static void virtio_vdpa_reset(struct virtio_device *vdev) |