diff options
Diffstat (limited to 'drivers')
505 files changed, 11359 insertions, 5126 deletions
diff --git a/drivers/Kconfig b/drivers/Kconfig index 4915a63866b0..3054b50a2f4c 100644 --- a/drivers/Kconfig +++ b/drivers/Kconfig @@ -251,4 +251,6 @@ source "drivers/hte/Kconfig" source "drivers/cdx/Kconfig" +source "drivers/resctrl/Kconfig" + endmenu diff --git a/drivers/Makefile b/drivers/Makefile index 8e1ffa4358d5..20eb17596b89 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -194,6 +194,7 @@ obj-$(CONFIG_HTE) += hte/ obj-$(CONFIG_DRM_ACCEL) += accel/ obj-$(CONFIG_CDX_BUS) += cdx/ obj-$(CONFIG_DPLL) += dpll/ +obj-y += resctrl/ obj-$(CONFIG_DIBS) += dibs/ obj-$(CONFIG_S390) += s390/ diff --git a/drivers/acpi/acpi_tad.c b/drivers/acpi/acpi_tad.c index 33418dd6768a..6d870d97ada6 100644 --- a/drivers/acpi/acpi_tad.c +++ b/drivers/acpi/acpi_tad.c @@ -90,19 +90,18 @@ static int acpi_tad_set_real_time(struct device *dev, struct acpi_tad_rt *rt) args[0].buffer.pointer = (u8 *)rt; args[0].buffer.length = sizeof(*rt); - pm_runtime_get_sync(dev); + PM_RUNTIME_ACQUIRE(dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return -ENXIO; status = acpi_evaluate_integer(handle, "_SRT", &arg_list, &retval); - - pm_runtime_put_sync(dev); - if (ACPI_FAILURE(status) || retval) return -EIO; return 0; } -static int acpi_tad_get_real_time(struct device *dev, struct acpi_tad_rt *rt) +static int acpi_tad_evaluate_grt(struct device *dev, struct acpi_tad_rt *rt) { acpi_handle handle = ACPI_HANDLE(dev); struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER }; @@ -111,12 +110,7 @@ static int acpi_tad_get_real_time(struct device *dev, struct acpi_tad_rt *rt) acpi_status status; int ret = -EIO; - pm_runtime_get_sync(dev); - status = acpi_evaluate_object(handle, "_GRT", NULL, &output); - - pm_runtime_put_sync(dev); - if (ACPI_FAILURE(status)) goto out_free; @@ -139,6 +133,21 @@ out_free: return ret; } +static int acpi_tad_get_real_time(struct device *dev, struct acpi_tad_rt *rt) +{ + int ret; + + PM_RUNTIME_ACQUIRE(dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return -ENXIO; + + ret = acpi_tad_evaluate_grt(dev, rt); + if (ret) + return ret; + + return 0; +} + static char *acpi_tad_rt_next_field(char *s, int *val) { char *p; @@ -266,12 +275,11 @@ static int acpi_tad_wake_set(struct device *dev, char *method, u32 timer_id, args[0].integer.value = timer_id; args[1].integer.value = value; - pm_runtime_get_sync(dev); + PM_RUNTIME_ACQUIRE(dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return -ENXIO; status = acpi_evaluate_integer(handle, method, &arg_list, &retval); - - pm_runtime_put_sync(dev); - if (ACPI_FAILURE(status) || retval) return -EIO; @@ -314,12 +322,11 @@ static ssize_t acpi_tad_wake_read(struct device *dev, char *buf, char *method, args[0].integer.value = timer_id; - pm_runtime_get_sync(dev); + PM_RUNTIME_ACQUIRE(dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return -ENXIO; status = acpi_evaluate_integer(handle, method, &arg_list, &retval); - - pm_runtime_put_sync(dev); - if (ACPI_FAILURE(status)) return -EIO; @@ -370,12 +377,11 @@ static int acpi_tad_clear_status(struct device *dev, u32 timer_id) args[0].integer.value = timer_id; - pm_runtime_get_sync(dev); + PM_RUNTIME_ACQUIRE(dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return -ENXIO; status = acpi_evaluate_integer(handle, "_CWS", &arg_list, &retval); - - pm_runtime_put_sync(dev); - if (ACPI_FAILURE(status) || retval) return -EIO; @@ -411,12 +417,11 @@ static ssize_t acpi_tad_status_read(struct device *dev, char *buf, u32 timer_id) args[0].integer.value = timer_id; - pm_runtime_get_sync(dev); + PM_RUNTIME_ACQUIRE(dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return -ENXIO; status = acpi_evaluate_integer(handle, "_GWS", &arg_list, &retval); - - pm_runtime_put_sync(dev); - if (ACPI_FAILURE(status)) return -EIO; @@ -563,8 +568,6 @@ static void acpi_tad_remove(struct platform_device *pdev) device_init_wakeup(dev, false); - pm_runtime_get_sync(dev); - if (dd->capabilities & ACPI_TAD_RT) sysfs_remove_group(&dev->kobj, &acpi_tad_time_attr_group); @@ -573,14 +576,16 @@ static void acpi_tad_remove(struct platform_device *pdev) sysfs_remove_group(&dev->kobj, &acpi_tad_attr_group); - acpi_tad_disable_timer(dev, ACPI_TAD_AC_TIMER); - acpi_tad_clear_status(dev, ACPI_TAD_AC_TIMER); - if (dd->capabilities & ACPI_TAD_DC_WAKE) { - acpi_tad_disable_timer(dev, ACPI_TAD_DC_TIMER); - acpi_tad_clear_status(dev, ACPI_TAD_DC_TIMER); + scoped_guard(pm_runtime_noresume, dev) { + acpi_tad_disable_timer(dev, ACPI_TAD_AC_TIMER); + acpi_tad_clear_status(dev, ACPI_TAD_AC_TIMER); + if (dd->capabilities & ACPI_TAD_DC_WAKE) { + acpi_tad_disable_timer(dev, ACPI_TAD_DC_TIMER); + acpi_tad_clear_status(dev, ACPI_TAD_DC_TIMER); + } } - pm_runtime_put_sync(dev); + pm_runtime_suspend(dev); pm_runtime_disable(dev); acpi_remove_cmos_rtc_space_handler(handle); } diff --git a/drivers/acpi/acpica/nswalk.c b/drivers/acpi/acpica/nswalk.c index a2ac06a26e92..5670ff5a43cd 100644 --- a/drivers/acpi/acpica/nswalk.c +++ b/drivers/acpi/acpica/nswalk.c @@ -169,9 +169,12 @@ acpi_ns_walk_namespace(acpi_object_type type, if (start_node == ACPI_ROOT_OBJECT) { start_node = acpi_gbl_root_node; - if (!start_node) { - return_ACPI_STATUS(AE_NO_NAMESPACE); - } + } + + /* Avoid walking the namespace if the StartNode is NULL */ + + if (!start_node) { + return_ACPI_STATUS(AE_NO_NAMESPACE); } /* Null child means "get first node" */ diff --git a/drivers/acpi/arm64/Kconfig b/drivers/acpi/arm64/Kconfig index b3ed6212244c..f2fd79f22e7d 100644 --- a/drivers/acpi/arm64/Kconfig +++ b/drivers/acpi/arm64/Kconfig @@ -21,3 +21,6 @@ config ACPI_AGDI config ACPI_APMT bool + +config ACPI_MPAM + bool diff --git a/drivers/acpi/arm64/Makefile b/drivers/acpi/arm64/Makefile index 05ecde9eaabe..9390b57cb564 100644 --- a/drivers/acpi/arm64/Makefile +++ b/drivers/acpi/arm64/Makefile @@ -4,6 +4,7 @@ obj-$(CONFIG_ACPI_APMT) += apmt.o obj-$(CONFIG_ACPI_FFH) += ffh.o obj-$(CONFIG_ACPI_GTDT) += gtdt.o obj-$(CONFIG_ACPI_IORT) += iort.o +obj-$(CONFIG_ACPI_MPAM) += mpam.o obj-$(CONFIG_ACPI_PROCESSOR_IDLE) += cpuidle.o obj-$(CONFIG_ARM_AMBA) += amba.o obj-y += dma.o init.o diff --git a/drivers/acpi/arm64/gtdt.c b/drivers/acpi/arm64/gtdt.c index 8cc8af8fd408..ffc867bac2d6 100644 --- a/drivers/acpi/arm64/gtdt.c +++ b/drivers/acpi/arm64/gtdt.c @@ -303,40 +303,6 @@ error: return -EINVAL; } -/** - * acpi_arch_timer_mem_init() - Get the info of all GT blocks in GTDT table. - * @timer_mem: The pointer to the array of struct arch_timer_mem for returning - * the result of parsing. The element number of this array should - * be platform_timer_count(the total number of platform timers). - * @timer_count: It points to a integer variable which is used for storing the - * number of GT blocks we have parsed. - * - * Return: 0 if success, -EINVAL/-ENODEV if error. - */ -int __init acpi_arch_timer_mem_init(struct arch_timer_mem *timer_mem, - int *timer_count) -{ - int ret; - void *platform_timer; - - *timer_count = 0; - for_each_platform_timer(platform_timer) { - if (is_timer_block(platform_timer)) { - ret = gtdt_parse_timer_block(platform_timer, timer_mem); - if (ret) - return ret; - timer_mem++; - (*timer_count)++; - } - } - - if (*timer_count) - pr_info("found %d memory-mapped timer block(s).\n", - *timer_count); - - return 0; -} - /* * Initialize a SBSA generic Watchdog platform device info from GTDT */ diff --git a/drivers/acpi/arm64/mpam.c b/drivers/acpi/arm64/mpam.c new file mode 100644 index 000000000000..84963a20c3e7 --- /dev/null +++ b/drivers/acpi/arm64/mpam.c @@ -0,0 +1,411 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Arm Ltd. + +/* Parse the MPAM ACPI table feeding the discovered nodes into the driver */ + +#define pr_fmt(fmt) "ACPI MPAM: " fmt + +#include <linux/acpi.h> +#include <linux/arm_mpam.h> +#include <linux/bits.h> +#include <linux/cpu.h> +#include <linux/cpumask.h> +#include <linux/platform_device.h> + +#include <acpi/processor.h> + +/* + * Flags for acpi_table_mpam_msc.*_interrupt_flags. + * See 2.1.1 Interrupt Flags, Table 5, of DEN0065B_MPAM_ACPI_3.0-bet. + */ +#define ACPI_MPAM_MSC_IRQ_MODE BIT(0) +#define ACPI_MPAM_MSC_IRQ_TYPE_MASK GENMASK(2, 1) +#define ACPI_MPAM_MSC_IRQ_TYPE_WIRED 0 +#define ACPI_MPAM_MSC_IRQ_AFFINITY_TYPE_MASK BIT(3) +#define ACPI_MPAM_MSC_IRQ_AFFINITY_TYPE_PROCESSOR 0 +#define ACPI_MPAM_MSC_IRQ_AFFINITY_TYPE_PROCESSOR_CONTAINER 1 +#define ACPI_MPAM_MSC_IRQ_AFFINITY_VALID BIT(4) + +/* + * Encodings for the MSC node body interface type field. + * See 2.1 MPAM MSC node, Table 4 of DEN0065B_MPAM_ACPI_3.0-bet. + */ +#define ACPI_MPAM_MSC_IFACE_MMIO 0x00 +#define ACPI_MPAM_MSC_IFACE_PCC 0x0a + +static bool _is_ppi_partition(u32 flags) +{ + u32 aff_type, is_ppi; + bool ret; + + is_ppi = FIELD_GET(ACPI_MPAM_MSC_IRQ_AFFINITY_VALID, flags); + if (!is_ppi) + return false; + + aff_type = FIELD_GET(ACPI_MPAM_MSC_IRQ_AFFINITY_TYPE_MASK, flags); + ret = (aff_type == ACPI_MPAM_MSC_IRQ_AFFINITY_TYPE_PROCESSOR_CONTAINER); + if (ret) + pr_err_once("Partitioned interrupts not supported\n"); + + return ret; +} + +static int acpi_mpam_register_irq(struct platform_device *pdev, + u32 intid, u32 flags) +{ + int irq; + u32 int_type; + int trigger; + + if (!intid) + return -EINVAL; + + if (_is_ppi_partition(flags)) + return -EINVAL; + + trigger = FIELD_GET(ACPI_MPAM_MSC_IRQ_MODE, flags); + int_type = FIELD_GET(ACPI_MPAM_MSC_IRQ_TYPE_MASK, flags); + if (int_type != ACPI_MPAM_MSC_IRQ_TYPE_WIRED) + return -EINVAL; + + irq = acpi_register_gsi(&pdev->dev, intid, trigger, ACPI_ACTIVE_HIGH); + if (irq < 0) + pr_err_once("Failed to register interrupt 0x%x with ACPI\n", intid); + + return irq; +} + +static void acpi_mpam_parse_irqs(struct platform_device *pdev, + struct acpi_mpam_msc_node *tbl_msc, + struct resource *res, int *res_idx) +{ + u32 flags, intid; + int irq; + + intid = tbl_msc->overflow_interrupt; + flags = tbl_msc->overflow_interrupt_flags; + irq = acpi_mpam_register_irq(pdev, intid, flags); + if (irq > 0) + res[(*res_idx)++] = DEFINE_RES_IRQ_NAMED(irq, "overflow"); + + intid = tbl_msc->error_interrupt; + flags = tbl_msc->error_interrupt_flags; + irq = acpi_mpam_register_irq(pdev, intid, flags); + if (irq > 0) + res[(*res_idx)++] = DEFINE_RES_IRQ_NAMED(irq, "error"); +} + +static int acpi_mpam_parse_resource(struct mpam_msc *msc, + struct acpi_mpam_resource_node *res) +{ + int level, nid; + u32 cache_id; + + switch (res->locator_type) { + case ACPI_MPAM_LOCATION_TYPE_PROCESSOR_CACHE: + cache_id = res->locator.cache_locator.cache_reference; + level = find_acpi_cache_level_from_id(cache_id); + if (level <= 0) { + pr_err_once("Bad level (%d) for cache with id %u\n", level, cache_id); + return -EINVAL; + } + return mpam_ris_create(msc, res->ris_index, MPAM_CLASS_CACHE, + level, cache_id); + case ACPI_MPAM_LOCATION_TYPE_MEMORY: + nid = pxm_to_node(res->locator.memory_locator.proximity_domain); + if (nid == NUMA_NO_NODE) { + pr_debug("Bad proximity domain %lld, using node 0 instead\n", + res->locator.memory_locator.proximity_domain); + nid = 0; + } + return mpam_ris_create(msc, res->ris_index, MPAM_CLASS_MEMORY, + MPAM_CLASS_ID_DEFAULT, nid); + default: + /* These get discovered later and are treated as unknown */ + return 0; + } +} + +int acpi_mpam_parse_resources(struct mpam_msc *msc, + struct acpi_mpam_msc_node *tbl_msc) +{ + int i, err; + char *ptr, *table_end; + struct acpi_mpam_resource_node *resource; + + table_end = (char *)tbl_msc + tbl_msc->length; + ptr = (char *)(tbl_msc + 1); + for (i = 0; i < tbl_msc->num_resource_nodes; i++) { + u64 max_deps, remaining_table; + + if (ptr + sizeof(*resource) > table_end) + return -EINVAL; + + resource = (struct acpi_mpam_resource_node *)ptr; + + remaining_table = table_end - ptr; + max_deps = remaining_table / sizeof(struct acpi_mpam_func_deps); + if (resource->num_functional_deps > max_deps) { + pr_debug("MSC has impossible number of functional dependencies\n"); + return -EINVAL; + } + + err = acpi_mpam_parse_resource(msc, resource); + if (err) + return err; + + ptr += sizeof(*resource); + ptr += resource->num_functional_deps * sizeof(struct acpi_mpam_func_deps); + } + + return 0; +} + +/* + * Creates the device power management link and returns true if the + * acpi id is valid and usable for cpu affinity. This is the case + * when the linked device is a processor or a processor container. + */ +static bool __init parse_msc_pm_link(struct acpi_mpam_msc_node *tbl_msc, + struct platform_device *pdev, + u32 *acpi_id) +{ + char hid[sizeof(tbl_msc->hardware_id_linked_device) + 1] = { 0 }; + bool acpi_id_valid = false; + struct acpi_device *buddy; + char uid[11]; + int len; + + memcpy(hid, &tbl_msc->hardware_id_linked_device, + sizeof(tbl_msc->hardware_id_linked_device)); + + if (!strcmp(hid, ACPI_PROCESSOR_CONTAINER_HID)) { + *acpi_id = tbl_msc->instance_id_linked_device; + acpi_id_valid = true; + } + + len = snprintf(uid, sizeof(uid), "%u", + tbl_msc->instance_id_linked_device); + if (len >= sizeof(uid)) { + pr_debug("Failed to convert uid of device for power management."); + return acpi_id_valid; + } + + buddy = acpi_dev_get_first_match_dev(hid, uid, -1); + if (buddy) { + device_link_add(&pdev->dev, &buddy->dev, DL_FLAG_STATELESS); + acpi_dev_put(buddy); + } + + return acpi_id_valid; +} + +static int decode_interface_type(struct acpi_mpam_msc_node *tbl_msc, + enum mpam_msc_iface *iface) +{ + switch (tbl_msc->interface_type) { + case ACPI_MPAM_MSC_IFACE_MMIO: + *iface = MPAM_IFACE_MMIO; + return 0; + case ACPI_MPAM_MSC_IFACE_PCC: + *iface = MPAM_IFACE_PCC; + return 0; + default: + return -EINVAL; + } +} + +static struct platform_device * __init acpi_mpam_parse_msc(struct acpi_mpam_msc_node *tbl_msc) +{ + struct platform_device *pdev __free(platform_device_put) = + platform_device_alloc("mpam_msc", tbl_msc->identifier); + int next_res = 0, next_prop = 0, err; + /* pcc, nrdy, affinity and a sentinel */ + struct property_entry props[4] = { 0 }; + /* mmio, 2xirq, no sentinel. */ + struct resource res[3] = { 0 }; + struct acpi_device *companion; + enum mpam_msc_iface iface; + char uid[16]; + u32 acpi_id; + + if (!pdev) + return ERR_PTR(-ENOMEM); + + /* Some power management is described in the namespace: */ + err = snprintf(uid, sizeof(uid), "%u", tbl_msc->identifier); + if (err > 0 && err < sizeof(uid)) { + companion = acpi_dev_get_first_match_dev("ARMHAA5C", uid, -1); + if (companion) { + ACPI_COMPANION_SET(&pdev->dev, companion); + acpi_dev_put(companion); + } else { + pr_debug("MSC.%u: missing namespace entry\n", tbl_msc->identifier); + } + } + + if (decode_interface_type(tbl_msc, &iface)) { + pr_debug("MSC.%u: unknown interface type\n", tbl_msc->identifier); + return ERR_PTR(-EINVAL); + } + + if (iface == MPAM_IFACE_MMIO) { + res[next_res++] = DEFINE_RES_MEM_NAMED(tbl_msc->base_address, + tbl_msc->mmio_size, + "MPAM:MSC"); + } else if (iface == MPAM_IFACE_PCC) { + props[next_prop++] = PROPERTY_ENTRY_U32("pcc-channel", + tbl_msc->base_address); + } + + acpi_mpam_parse_irqs(pdev, tbl_msc, res, &next_res); + + WARN_ON_ONCE(next_res > ARRAY_SIZE(res)); + err = platform_device_add_resources(pdev, res, next_res); + if (err) + return ERR_PTR(err); + + props[next_prop++] = PROPERTY_ENTRY_U32("arm,not-ready-us", + tbl_msc->max_nrdy_usec); + + /* + * The MSC's CPU affinity is described via its linked power + * management device, but only if it points at a Processor or + * Processor Container. + */ + if (parse_msc_pm_link(tbl_msc, pdev, &acpi_id)) + props[next_prop++] = PROPERTY_ENTRY_U32("cpu_affinity", acpi_id); + + WARN_ON_ONCE(next_prop > ARRAY_SIZE(props) - 1); + err = device_create_managed_software_node(&pdev->dev, props, NULL); + if (err) + return ERR_PTR(err); + + /* + * Stash the table entry for acpi_mpam_parse_resources() to discover + * what this MSC controls. + */ + err = platform_device_add_data(pdev, tbl_msc, tbl_msc->length); + if (err) + return ERR_PTR(err); + + err = platform_device_add(pdev); + if (err) + return ERR_PTR(err); + + return_ptr(pdev); +} + +static int __init acpi_mpam_parse(void) +{ + char *table_end, *table_offset; + struct acpi_mpam_msc_node *tbl_msc; + struct platform_device *pdev; + + if (acpi_disabled || !system_supports_mpam()) + return 0; + + struct acpi_table_header *table __free(acpi_put_table) = + acpi_get_table_pointer(ACPI_SIG_MPAM, 0); + + if (IS_ERR(table)) + return 0; + + if (table->revision < 1) { + pr_debug("MPAM ACPI table revision %d not supported\n", table->revision); + return 0; + } + + table_offset = (char *)(table + 1); + table_end = (char *)table + table->length; + + while (table_offset < table_end) { + tbl_msc = (struct acpi_mpam_msc_node *)table_offset; + if (table_offset + sizeof(*tbl_msc) > table_end || + table_offset + tbl_msc->length > table_end) { + pr_err("MSC entry overlaps end of ACPI table\n"); + return -EINVAL; + } + table_offset += tbl_msc->length; + + /* + * If any of the reserved fields are set, make no attempt to + * parse the MSC structure. This MSC will still be counted by + * acpi_mpam_count_msc(), meaning the MPAM driver can't probe + * against all MSC, and will never be enabled. There is no way + * to enable it safely, because we cannot determine safe + * system-wide partid and pmg ranges in this situation. + */ + if (tbl_msc->reserved || tbl_msc->reserved1 || tbl_msc->reserved2) { + pr_err_once("Unrecognised MSC, MPAM not usable\n"); + pr_debug("MSC.%u: reserved field set\n", tbl_msc->identifier); + continue; + } + + if (!tbl_msc->mmio_size) { + pr_debug("MSC.%u: marked as disabled\n", tbl_msc->identifier); + continue; + } + + pdev = acpi_mpam_parse_msc(tbl_msc); + if (IS_ERR(pdev)) + return PTR_ERR(pdev); + } + + return 0; +} + +/** + * acpi_mpam_count_msc() - Count the number of MSC described by firmware. + * + * Returns the number of MSCs, or zero for an error. + * + * This can be called before or in parallel with acpi_mpam_parse(). + */ +int acpi_mpam_count_msc(void) +{ + char *table_end, *table_offset; + struct acpi_mpam_msc_node *tbl_msc; + int count = 0; + + if (acpi_disabled || !system_supports_mpam()) + return 0; + + struct acpi_table_header *table __free(acpi_put_table) = + acpi_get_table_pointer(ACPI_SIG_MPAM, 0); + + if (IS_ERR(table)) + return 0; + + if (table->revision < 1) + return 0; + + table_offset = (char *)(table + 1); + table_end = (char *)table + table->length; + + while (table_offset < table_end) { + tbl_msc = (struct acpi_mpam_msc_node *)table_offset; + + if (table_offset + sizeof(*tbl_msc) > table_end) + return -EINVAL; + if (tbl_msc->length < sizeof(*tbl_msc)) + return -EINVAL; + if (tbl_msc->length > table_end - table_offset) + return -EINVAL; + table_offset += tbl_msc->length; + + if (!tbl_msc->mmio_size) + continue; + + count++; + } + + return count; +} + +/* + * Call after ACPI devices have been created, which happens behind acpi_scan_init() + * called from subsys_initcall(). PCC requires the mailbox driver, which is + * initialised from postcore_initcall(). + */ +subsys_initcall_sync(acpi_mpam_parse); diff --git a/drivers/acpi/battery.c b/drivers/acpi/battery.c index 67b76492c839..34181fa52e93 100644 --- a/drivers/acpi/battery.c +++ b/drivers/acpi/battery.c @@ -91,7 +91,6 @@ enum { }; struct acpi_battery { - struct mutex lock; struct mutex update_lock; struct power_supply *bat; struct power_supply_desc bat_desc; @@ -535,11 +534,9 @@ static int acpi_battery_get_info(struct acpi_battery *battery) struct acpi_buffer buffer = { ACPI_ALLOCATE_BUFFER, NULL }; acpi_status status = AE_ERROR; - mutex_lock(&battery->lock); status = acpi_evaluate_object(battery->device->handle, use_bix ? "_BIX":"_BIF", NULL, &buffer); - mutex_unlock(&battery->lock); if (ACPI_FAILURE(status)) { acpi_handle_info(battery->device->handle, @@ -576,11 +573,8 @@ static int acpi_battery_get_state(struct acpi_battery *battery) msecs_to_jiffies(cache_time))) return 0; - mutex_lock(&battery->lock); status = acpi_evaluate_object(battery->device->handle, "_BST", NULL, &buffer); - mutex_unlock(&battery->lock); - if (ACPI_FAILURE(status)) { acpi_handle_info(battery->device->handle, "_BST evaluation failed: %s", @@ -628,11 +622,8 @@ static int acpi_battery_set_alarm(struct acpi_battery *battery) !test_bit(ACPI_BATTERY_ALARM_PRESENT, &battery->flags)) return -ENODEV; - mutex_lock(&battery->lock); status = acpi_execute_simple_method(battery->device->handle, "_BTP", battery->alarm); - mutex_unlock(&battery->lock); - if (ACPI_FAILURE(status)) return -ENODEV; @@ -1235,9 +1226,6 @@ static int acpi_battery_add(struct acpi_device *device) strscpy(acpi_device_name(device), ACPI_BATTERY_DEVICE_NAME); strscpy(acpi_device_class(device), ACPI_BATTERY_CLASS); device->driver_data = battery; - result = devm_mutex_init(&device->dev, &battery->lock); - if (result) - return result; result = devm_mutex_init(&device->dev, &battery->update_lock); if (result) diff --git a/drivers/acpi/dptf/Makefile b/drivers/acpi/dptf/Makefile index 297340682f66..e912a3be1d28 100644 --- a/drivers/acpi/dptf/Makefile +++ b/drivers/acpi/dptf/Makefile @@ -1,4 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_ACPI) += int340x_thermal.o obj-$(CONFIG_DPTF_POWER) += dptf_power.o obj-$(CONFIG_DPTF_PCH_FIVR) += dptf_pch_fivr.o diff --git a/drivers/acpi/dptf/dptf_pch_fivr.c b/drivers/acpi/dptf/dptf_pch_fivr.c index 952216c67d58..8d7e555929d3 100644 --- a/drivers/acpi/dptf/dptf_pch_fivr.c +++ b/drivers/acpi/dptf/dptf_pch_fivr.c @@ -41,7 +41,7 @@ static int pch_fivr_read(acpi_handle handle, char *method, struct pch_fivr_resp ret = 0; release_buffer: - kfree(buffer.pointer); + ACPI_FREE(buffer.pointer); return ret; } diff --git a/drivers/acpi/dptf/dptf_power.c b/drivers/acpi/dptf/dptf_power.c index 776914f31b9e..55ccbb8ddbe3 100644 --- a/drivers/acpi/dptf/dptf_power.c +++ b/drivers/acpi/dptf/dptf_power.c @@ -240,6 +240,8 @@ static const struct acpi_device_id int3407_device_ids[] = { {"INTC10D9", 0}, {"INTC1100", 0}, {"INTC1101", 0}, + {"INTC10F7", 0}, + {"INTC10F8", 0}, {"", 0}, }; MODULE_DEVICE_TABLE(acpi, int3407_device_ids); diff --git a/drivers/acpi/dptf/int340x_thermal.c b/drivers/acpi/dptf/int340x_thermal.c deleted file mode 100644 index a222df059a16..000000000000 --- a/drivers/acpi/dptf/int340x_thermal.c +++ /dev/null @@ -1,94 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * ACPI support for int340x thermal drivers - * - * Copyright (C) 2014, Intel Corporation - * Authors: Zhang Rui <rui.zhang@intel.com> - */ - -#include <linux/acpi.h> -#include <linux/module.h> - -#include "../internal.h" - -#define INT3401_DEVICE 0X01 -static const struct acpi_device_id int340x_thermal_device_ids[] = { - {"INT3400"}, - {"INT3401", INT3401_DEVICE}, - {"INT3402"}, - {"INT3403"}, - {"INT3404"}, - {"INT3406"}, - {"INT3407"}, - {"INT3408"}, - {"INT3409"}, - {"INT340A"}, - {"INT340B"}, - {"INT3532"}, - {"INTC1040"}, - {"INTC1041"}, - {"INTC1042"}, - {"INTC1043"}, - {"INTC1044"}, - {"INTC1045"}, - {"INTC1046"}, - {"INTC1047"}, - {"INTC1048"}, - {"INTC1049"}, - {"INTC1050"}, - {"INTC1060"}, - {"INTC1061"}, - {"INTC1062"}, - {"INTC1063"}, - {"INTC1064"}, - {"INTC1065"}, - {"INTC1066"}, - {"INTC1068"}, - {"INTC1069"}, - {"INTC106A"}, - {"INTC106B"}, - {"INTC106C"}, - {"INTC106D"}, - {"INTC10A0"}, - {"INTC10A1"}, - {"INTC10A2"}, - {"INTC10A3"}, - {"INTC10A4"}, - {"INTC10A5"}, - {"INTC10D4"}, - {"INTC10D5"}, - {"INTC10D6"}, - {"INTC10D7"}, - {"INTC10D8"}, - {"INTC10D9"}, - {"INTC10FC"}, - {"INTC10FD"}, - {"INTC10FE"}, - {"INTC10FF"}, - {"INTC1100"}, - {"INTC1101"}, - {"INTC1102"}, - {""}, -}; - -static int int340x_thermal_handler_attach(struct acpi_device *adev, - const struct acpi_device_id *id) -{ - if (IS_ENABLED(CONFIG_INT340X_THERMAL)) - acpi_create_platform_device(adev, NULL); - /* Intel SoC DTS thermal driver needs INT3401 to set IRQ descriptor */ - else if (IS_ENABLED(CONFIG_INTEL_SOC_DTS_THERMAL) && - id->driver_data == INT3401_DEVICE) - acpi_create_platform_device(adev, NULL); - return 1; -} - -static struct acpi_scan_handler int340x_thermal_handler = { - .ids = int340x_thermal_device_ids, - .attach = int340x_thermal_handler_attach, -}; - -void __init acpi_int340x_thermal_init(void) -{ - acpi_scan_add_handler(&int340x_thermal_handler); -} diff --git a/drivers/acpi/ec.c b/drivers/acpi/ec.c index 7855bbf752b1..59b3d50ff01e 100644 --- a/drivers/acpi/ec.c +++ b/drivers/acpi/ec.c @@ -2294,7 +2294,8 @@ static int acpi_ec_init_workqueues(void) ec_wq = alloc_ordered_workqueue("kec", 0); if (!ec_query_wq) - ec_query_wq = alloc_workqueue("kec_query", 0, ec_max_queries); + ec_query_wq = alloc_workqueue("kec_query", WQ_PERCPU, + ec_max_queries); if (!ec_wq || !ec_query_wq) { acpi_ec_destroy_workqueues(); diff --git a/drivers/acpi/fan.h b/drivers/acpi/fan.h index bedbab0e8e4e..97ce3212edf3 100644 --- a/drivers/acpi/fan.h +++ b/drivers/acpi/fan.h @@ -11,6 +11,7 @@ #define _ACPI_FAN_H_ #include <linux/kconfig.h> +#include <linux/limits.h> #define ACPI_FAN_DEVICE_IDS \ {"INT3404", }, /* Fan */ \ @@ -21,6 +22,7 @@ {"INTC10A2", }, /* Fan for Raptor Lake generation */ \ {"INTC10D6", }, /* Fan for Panther Lake generation */ \ {"INTC10FE", }, /* Fan for Wildcat Lake generation */ \ + {"INTC10F5", }, /* Fan for Nova Lake generation */ \ {"PNP0C0B", } /* Generic ACPI fan */ #define ACPI_FPS_NAME_LEN 20 @@ -55,19 +57,58 @@ struct acpi_fan { struct acpi_fan_fif fif; struct acpi_fan_fps *fps; int fps_count; + /* A value of 0 means that trippoint-related functions are not supported */ + u32 fan_trip_granularity; +#if IS_REACHABLE(CONFIG_HWMON) + struct device *hdev; +#endif struct thermal_cooling_device *cdev; struct device_attribute fst_speed; struct device_attribute fine_grain_control; }; +/** + * acpi_fan_speed_valid - Check if fan speed value is valid + * @speeed: Speed value returned by the ACPI firmware + * + * Check if the fan speed value returned by the ACPI firmware is valid. This function is + * necessary as ACPI firmware implementations can return 0xFFFFFFFF to signal that the + * ACPI fan does not support speed reporting. Additionally, some buggy ACPI firmware + * implementations return a value larger than the 32-bit integer value defined by + * the ACPI specification when using placeholder values. Such invalid values are also + * detected by this function. + * + * Returns: True if the fan speed value is valid, false otherwise. + */ +static inline bool acpi_fan_speed_valid(u64 speed) +{ + return speed < U32_MAX; +} + +/** + * acpi_fan_power_valid - Check if fan power value is valid + * @power: Power value returned by the ACPI firmware + * + * Check if the fan power value returned by the ACPI firmware is valid. + * See acpi_fan_speed_valid() for details. + * + * Returns: True if the fan power value is valid, false otherwise. + */ +static inline bool acpi_fan_power_valid(u64 power) +{ + return power < U32_MAX; +} + int acpi_fan_get_fst(acpi_handle handle, struct acpi_fan_fst *fst); int acpi_fan_create_attributes(struct acpi_device *device); void acpi_fan_delete_attributes(struct acpi_device *device); #if IS_REACHABLE(CONFIG_HWMON) int devm_acpi_fan_create_hwmon(struct device *dev); +void acpi_fan_notify_hwmon(struct device *dev); #else static inline int devm_acpi_fan_create_hwmon(struct device *dev) { return 0; }; +static inline void acpi_fan_notify_hwmon(struct device *dev) { }; #endif #endif diff --git a/drivers/acpi/fan_core.c b/drivers/acpi/fan_core.c index 46e7fe7a506d..fb08b8549ed7 100644 --- a/drivers/acpi/fan_core.c +++ b/drivers/acpi/fan_core.c @@ -7,11 +7,16 @@ * Copyright (C) 2022 Intel Corporation. All rights reserved. */ +#include <linux/bits.h> #include <linux/kernel.h> +#include <linux/limits.h> +#include <linux/math.h> +#include <linux/math64.h> #include <linux/module.h> #include <linux/init.h> #include <linux/types.h> #include <linux/uaccess.h> +#include <linux/uuid.h> #include <linux/thermal.h> #include <linux/acpi.h> #include <linux/platform_device.h> @@ -19,6 +24,26 @@ #include "fan.h" +#define ACPI_FAN_NOTIFY_STATE_CHANGED 0x80 + +/* + * Defined inside the "Fan Noise Signal" section at + * https://learn.microsoft.com/en-us/windows-hardware/design/device-experiences/design-guide. + */ +static const guid_t acpi_fan_microsoft_guid = GUID_INIT(0xA7611840, 0x99FE, 0x41AE, 0xA4, 0x88, + 0x35, 0xC7, 0x59, 0x26, 0xC8, 0xEB); +#define ACPI_FAN_DSM_GET_TRIP_POINT_GRANULARITY 1 +#define ACPI_FAN_DSM_SET_TRIP_POINTS 2 +#define ACPI_FAN_DSM_GET_OPERATING_RANGES 3 + +/* + * Ensures that fans with a very low trip point granularity + * do not send too many notifications. + */ +static uint min_trip_distance = 100; +module_param(min_trip_distance, uint, 0); +MODULE_PARM_DESC(min_trip_distance, "Minimum distance between fan speed trip points in RPM"); + static const struct acpi_device_id fan_device_ids[] = { ACPI_FAN_DEVICE_IDS, {"", 0}, @@ -308,6 +333,182 @@ err: return status; } +static int acpi_fan_dsm_init(struct device *dev) +{ + union acpi_object dummy = { + .package = { + .type = ACPI_TYPE_PACKAGE, + .count = 0, + .elements = NULL, + }, + }; + struct acpi_fan *fan = dev_get_drvdata(dev); + union acpi_object *obj; + int ret = 0; + + if (!acpi_check_dsm(fan->handle, &acpi_fan_microsoft_guid, 0, + BIT(ACPI_FAN_DSM_GET_TRIP_POINT_GRANULARITY) | + BIT(ACPI_FAN_DSM_SET_TRIP_POINTS))) + return 0; + + dev_info(dev, "Using Microsoft fan extensions\n"); + + obj = acpi_evaluate_dsm_typed(fan->handle, &acpi_fan_microsoft_guid, 0, + ACPI_FAN_DSM_GET_TRIP_POINT_GRANULARITY, &dummy, + ACPI_TYPE_INTEGER); + if (!obj) + return -EIO; + + if (obj->integer.value > U32_MAX) + ret = -EOVERFLOW; + else + fan->fan_trip_granularity = obj->integer.value; + + kfree(obj); + + return ret; +} + +static int acpi_fan_dsm_set_trip_points(struct device *dev, u64 upper, u64 lower) +{ + union acpi_object args[2] = { + { + .integer = { + .type = ACPI_TYPE_INTEGER, + .value = lower, + }, + }, + { + .integer = { + .type = ACPI_TYPE_INTEGER, + .value = upper, + }, + }, + }; + struct acpi_fan *fan = dev_get_drvdata(dev); + union acpi_object in = { + .package = { + .type = ACPI_TYPE_PACKAGE, + .count = ARRAY_SIZE(args), + .elements = args, + }, + }; + union acpi_object *obj; + + obj = acpi_evaluate_dsm(fan->handle, &acpi_fan_microsoft_guid, 0, + ACPI_FAN_DSM_SET_TRIP_POINTS, &in); + kfree(obj); + + return 0; +} + +static int acpi_fan_dsm_start(struct device *dev) +{ + struct acpi_fan *fan = dev_get_drvdata(dev); + int ret; + + if (!fan->fan_trip_granularity) + return 0; + + /* + * Some firmware implementations only update the values returned by the + * _FST control method when a notification is received. This usually + * works with Microsoft Windows as setting up trip points will keep + * triggering said notifications, but will cause issues when using _FST + * without the Microsoft-specific trip point extension. + * + * Because of this, an initial notification needs to be triggered to + * start the cycle of trip points updates. This is achieved by setting + * the trip points sequencially to two separate ranges. As by the + * Microsoft specification the firmware should trigger a notification + * immediately if the fan speed is outside the trip point range. This + * _should_ result in at least one notification as both ranges do not + * overlap, meaning that the current fan speed needs to be outside at + * least one range. + */ + ret = acpi_fan_dsm_set_trip_points(dev, fan->fan_trip_granularity, 0); + if (ret < 0) + return ret; + + return acpi_fan_dsm_set_trip_points(dev, fan->fan_trip_granularity * 3, + fan->fan_trip_granularity * 2); +} + +static int acpi_fan_dsm_update_trips_points(struct device *dev, struct acpi_fan_fst *fst) +{ + struct acpi_fan *fan = dev_get_drvdata(dev); + u64 upper, lower; + + if (!fan->fan_trip_granularity) + return 0; + + if (!acpi_fan_speed_valid(fst->speed)) + return -EINVAL; + + upper = roundup_u64(fst->speed + min_trip_distance, fan->fan_trip_granularity); + if (fst->speed <= min_trip_distance) { + lower = 0; + } else { + /* + * Valid fan speed values cannot be larger than 32 bit, so + * we can safely assume that no overflow will happen here. + */ + lower = rounddown((u32)fst->speed - min_trip_distance, fan->fan_trip_granularity); + } + + return acpi_fan_dsm_set_trip_points(dev, upper, lower); +} + +static void acpi_fan_notify_handler(acpi_handle handle, u32 event, void *context) +{ + struct device *dev = context; + struct acpi_fan_fst fst; + int ret; + + switch (event) { + case ACPI_FAN_NOTIFY_STATE_CHANGED: + /* + * The ACPI specification says that we must evaluate _FST when we + * receive an ACPI event indicating that the fan state has changed. + */ + ret = acpi_fan_get_fst(handle, &fst); + if (ret < 0) { + dev_err(dev, "Error retrieving current fan status: %d\n", ret); + } else { + ret = acpi_fan_dsm_update_trips_points(dev, &fst); + if (ret < 0) + dev_err(dev, "Failed to update trip points: %d\n", ret); + } + + acpi_fan_notify_hwmon(dev); + acpi_bus_generate_netlink_event("fan", dev_name(dev), event, 0); + break; + default: + dev_dbg(dev, "Unsupported ACPI notification 0x%x\n", event); + break; + } +} + +static void acpi_fan_notify_remove(void *data) +{ + struct acpi_fan *fan = data; + + acpi_remove_notify_handler(fan->handle, ACPI_DEVICE_NOTIFY, acpi_fan_notify_handler); +} + +static int devm_acpi_fan_notify_init(struct device *dev) +{ + struct acpi_fan *fan = dev_get_drvdata(dev); + acpi_status status; + + status = acpi_install_notify_handler(fan->handle, ACPI_DEVICE_NOTIFY, + acpi_fan_notify_handler, dev); + if (ACPI_FAILURE(status)) + return -EIO; + + return devm_add_action_or_reset(dev, acpi_fan_notify_remove, fan); +} + static int acpi_fan_probe(struct platform_device *pdev) { int result = 0; @@ -347,10 +548,24 @@ static int acpi_fan_probe(struct platform_device *pdev) } if (fan->has_fst) { + result = acpi_fan_dsm_init(&pdev->dev); + if (result) + return result; + result = devm_acpi_fan_create_hwmon(&pdev->dev); if (result) return result; + result = devm_acpi_fan_notify_init(&pdev->dev); + if (result) + return result; + + result = acpi_fan_dsm_start(&pdev->dev); + if (result) { + dev_err(&pdev->dev, "Failed to start Microsoft fan extensions\n"); + return result; + } + result = acpi_fan_create_attributes(device); if (result) return result; @@ -436,8 +651,14 @@ static int acpi_fan_suspend(struct device *dev) static int acpi_fan_resume(struct device *dev) { - int result; struct acpi_fan *fan = dev_get_drvdata(dev); + int result; + + if (fan->has_fst) { + result = acpi_fan_dsm_start(dev); + if (result) + dev_err(dev, "Failed to start Microsoft fan extensions: %d\n", result); + } if (fan->acpi4) return 0; diff --git a/drivers/acpi/fan_hwmon.c b/drivers/acpi/fan_hwmon.c index 4b2c2007f2d7..d3374f8f524b 100644 --- a/drivers/acpi/fan_hwmon.c +++ b/drivers/acpi/fan_hwmon.c @@ -15,10 +15,6 @@ #include "fan.h" -/* Returned when the ACPI fan does not support speed reporting */ -#define FAN_SPEED_UNAVAILABLE U32_MAX -#define FAN_POWER_UNAVAILABLE U32_MAX - static struct acpi_fan_fps *acpi_fan_get_current_fps(struct acpi_fan *fan, u64 control) { unsigned int i; @@ -77,7 +73,7 @@ static umode_t acpi_fan_hwmon_is_visible(const void *drvdata, enum hwmon_sensor_ * when the associated attribute should not be created. */ for (i = 0; i < fan->fps_count; i++) { - if (fan->fps[i].power != FAN_POWER_UNAVAILABLE) + if (acpi_fan_power_valid(fan->fps[i].power)) return 0444; } @@ -106,7 +102,7 @@ static int acpi_fan_hwmon_read(struct device *dev, enum hwmon_sensor_types type, case hwmon_fan: switch (attr) { case hwmon_fan_input: - if (fst.speed == FAN_SPEED_UNAVAILABLE) + if (!acpi_fan_speed_valid(fst.speed)) return -ENODEV; if (fst.speed > LONG_MAX) @@ -134,7 +130,7 @@ static int acpi_fan_hwmon_read(struct device *dev, enum hwmon_sensor_types type, if (!fps) return -EIO; - if (fps->power == FAN_POWER_UNAVAILABLE) + if (!acpi_fan_power_valid(fps->power)) return -ENODEV; if (fps->power > LONG_MAX / MICROWATT_PER_MILLIWATT) @@ -166,12 +162,19 @@ static const struct hwmon_chip_info acpi_fan_hwmon_chip_info = { .info = acpi_fan_hwmon_info, }; +void acpi_fan_notify_hwmon(struct device *dev) +{ + struct acpi_fan *fan = dev_get_drvdata(dev); + + hwmon_notify_event(fan->hdev, hwmon_fan, hwmon_fan_input, 0); +} + int devm_acpi_fan_create_hwmon(struct device *dev) { struct acpi_fan *fan = dev_get_drvdata(dev); - struct device *hdev; - hdev = devm_hwmon_device_register_with_info(dev, "acpi_fan", fan, &acpi_fan_hwmon_chip_info, - NULL); - return PTR_ERR_OR_ZERO(hdev); + fan->hdev = devm_hwmon_device_register_with_info(dev, "acpi_fan", fan, + &acpi_fan_hwmon_chip_info, NULL); + + return PTR_ERR_OR_ZERO(fan->hdev); } diff --git a/drivers/acpi/internal.h b/drivers/acpi/internal.h index 63354972ab0b..40f875b265a9 100644 --- a/drivers/acpi/internal.h +++ b/drivers/acpi/internal.h @@ -27,7 +27,6 @@ static inline void acpi_pci_link_init(void) {} void acpi_processor_init(void); void acpi_platform_init(void); void acpi_pnp_init(void); -void acpi_int340x_thermal_init(void); int acpi_sysfs_init(void); void acpi_gpe_apply_masked_gpes(void); void acpi_container_init(void); diff --git a/drivers/acpi/irq.c b/drivers/acpi/irq.c index 76a856c32c4d..d1595156c86a 100644 --- a/drivers/acpi/irq.c +++ b/drivers/acpi/irq.c @@ -300,6 +300,25 @@ int acpi_irq_get(acpi_handle handle, unsigned int index, struct resource *res) } EXPORT_SYMBOL_GPL(acpi_irq_get); +const struct cpumask *acpi_irq_get_affinity(acpi_handle handle, + unsigned int index) +{ + struct irq_fwspec_info info; + struct irq_fwspec fwspec; + unsigned long flags; + + if (acpi_irq_parse_one(handle, index, &fwspec, &flags)) + return NULL; + + if (irq_populate_fwspec_info(&fwspec, &info)) + return NULL; + + if (!(info.flags & IRQ_FWSPEC_INFO_AFFINITY_VALID)) + return NULL; + + return info.affinity; +} + /** * acpi_set_irq_model - Setup the GSI irqdomain information * @model: the value assigned to acpi_irq_model diff --git a/drivers/acpi/osl.c b/drivers/acpi/osl.c index 5ff343096ece..05393a7315fe 100644 --- a/drivers/acpi/osl.c +++ b/drivers/acpi/osl.c @@ -398,7 +398,7 @@ static void acpi_os_drop_map_ref(struct acpi_ioremap *map) list_del_rcu(&map->list); INIT_RCU_WORK(&map->track.rwork, acpi_os_map_remove); - queue_rcu_work(system_wq, &map->track.rwork); + queue_rcu_work(system_percpu_wq, &map->track.rwork); } /** @@ -1694,8 +1694,8 @@ acpi_status __init acpi_os_initialize(void) acpi_status __init acpi_os_initialize1(void) { - kacpid_wq = alloc_workqueue("kacpid", 0, 1); - kacpi_notify_wq = alloc_workqueue("kacpi_notify", 0, 0); + kacpid_wq = alloc_workqueue("kacpid", WQ_PERCPU, 1); + kacpi_notify_wq = alloc_workqueue("kacpi_notify", WQ_PERCPU, 0); kacpi_hotplug_wq = alloc_ordered_workqueue("kacpi_hotplug", 0); BUG_ON(!kacpid_wq); BUG_ON(!kacpi_notify_wq); diff --git a/drivers/acpi/pptt.c b/drivers/acpi/pptt.c index 54676e3d82dd..de5f8c018333 100644 --- a/drivers/acpi/pptt.c +++ b/drivers/acpi/pptt.c @@ -21,6 +21,25 @@ #include <linux/cacheinfo.h> #include <acpi/processor.h> +/* + * The acpi_pptt_cache_v1 in actbl2.h, which is imported from acpica, + * only contains the cache_id field rather than all the fields of the + * Cache Type Structure. Use this alternative structure until it is + * resolved in acpica. + */ +struct acpi_pptt_cache_v1_full { + struct acpi_subtable_header header; + u16 reserved; + u32 flags; + u32 next_level_of_cache; + u32 size; + u32 number_of_sets; + u8 associativity; + u8 attributes; + u16 line_size; + u32 cache_id; +} __packed; + static struct acpi_subtable_header *fetch_pptt_subtable(struct acpi_table_header *table_hdr, u32 pptt_ref) { @@ -56,6 +75,18 @@ static struct acpi_pptt_cache *fetch_pptt_cache(struct acpi_table_header *table_ return (struct acpi_pptt_cache *)fetch_pptt_subtable(table_hdr, pptt_ref); } +static struct acpi_pptt_cache_v1_full *upgrade_pptt_cache(struct acpi_pptt_cache *cache) +{ + if (cache->header.length < sizeof(struct acpi_pptt_cache_v1_full)) + return NULL; + + /* No use for v1 if the only additional field is invalid */ + if (!(cache->flags & ACPI_PPTT_CACHE_ID_VALID)) + return NULL; + + return (struct acpi_pptt_cache_v1_full *)cache; +} + static struct acpi_subtable_header *acpi_get_pptt_resource(struct acpi_table_header *table_hdr, struct acpi_pptt_processor *node, int resource) @@ -177,14 +208,14 @@ acpi_find_cache_level(struct acpi_table_header *table_hdr, } /** - * acpi_count_levels() - Given a PPTT table, and a CPU node, count the cache - * levels and split cache levels (data/instruction). + * acpi_count_levels() - Given a PPTT table, and a CPU node, count the + * total number of levels and split cache levels (data/instruction). * @table_hdr: Pointer to the head of the PPTT table * @cpu_node: processor node we wish to count caches for - * @levels: Number of levels if success. * @split_levels: Number of split cache levels (data/instruction) if * success. Can by NULL. * + * Return: number of levels. * Given a processor node containing a processing unit, walk into it and count * how many levels exist solely for it, and then walk up each level until we hit * the root node (ignore the package level because it may be possible to have @@ -192,14 +223,18 @@ acpi_find_cache_level(struct acpi_table_header *table_hdr, * split cache levels (data/instruction) that exist at each level on the way * up. */ -static void acpi_count_levels(struct acpi_table_header *table_hdr, - struct acpi_pptt_processor *cpu_node, - unsigned int *levels, unsigned int *split_levels) +static int acpi_count_levels(struct acpi_table_header *table_hdr, + struct acpi_pptt_processor *cpu_node, + unsigned int *split_levels) { + int current_level = 0; + do { - acpi_find_cache_level(table_hdr, cpu_node, levels, split_levels, 0, 0); + acpi_find_cache_level(table_hdr, cpu_node, ¤t_level, split_levels, 0, 0); cpu_node = fetch_pptt_node(table_hdr, cpu_node->parent); } while (cpu_node); + + return current_level; } /** @@ -351,7 +386,6 @@ static struct acpi_pptt_cache *acpi_find_cache_node(struct acpi_table_header *ta * @this_leaf: Kernel cache info structure being updated * @found_cache: The PPTT node describing this cache instance * @cpu_node: A unique reference to describe this cache instance - * @revision: The revision of the PPTT table * * The ACPI spec implies that the fields in the cache structures are used to * extend and correct the information probed from the hardware. Lets only @@ -361,10 +395,9 @@ static struct acpi_pptt_cache *acpi_find_cache_node(struct acpi_table_header *ta */ static void update_cache_properties(struct cacheinfo *this_leaf, struct acpi_pptt_cache *found_cache, - struct acpi_pptt_processor *cpu_node, - u8 revision) + struct acpi_pptt_processor *cpu_node) { - struct acpi_pptt_cache_v1* found_cache_v1; + struct acpi_pptt_cache_v1_full *found_cache_v1; this_leaf->fw_token = cpu_node; if (found_cache->flags & ACPI_PPTT_SIZE_PROPERTY_VALID) @@ -414,9 +447,8 @@ static void update_cache_properties(struct cacheinfo *this_leaf, found_cache->flags & ACPI_PPTT_CACHE_TYPE_VALID) this_leaf->type = CACHE_TYPE_UNIFIED; - if (revision >= 3 && (found_cache->flags & ACPI_PPTT_CACHE_ID_VALID)) { - found_cache_v1 = ACPI_ADD_PTR(struct acpi_pptt_cache_v1, - found_cache, sizeof(struct acpi_pptt_cache)); + found_cache_v1 = upgrade_pptt_cache(found_cache); + if (found_cache_v1) { this_leaf->id = found_cache_v1->cache_id; this_leaf->attributes |= CACHE_ID; } @@ -441,8 +473,7 @@ static void cache_setup_acpi_cpu(struct acpi_table_header *table, pr_debug("found = %p %p\n", found_cache, cpu_node); if (found_cache) update_cache_properties(this_leaf, found_cache, - ACPI_TO_POINTER(ACPI_PTR_DIFF(cpu_node, table)), - table->revision); + ACPI_TO_POINTER(ACPI_PTR_DIFF(cpu_node, table))); index++; } @@ -645,7 +676,7 @@ int acpi_get_cache_info(unsigned int cpu, unsigned int *levels, if (!cpu_node) return -ENOENT; - acpi_count_levels(table, cpu_node, levels, split_levels); + *levels = acpi_count_levels(table, cpu_node, split_levels); pr_debug("Cache Setup: last_level=%d split_levels=%d\n", *levels, split_levels ? *split_levels : -1); @@ -817,3 +848,218 @@ int find_acpi_cpu_topology_hetero_id(unsigned int cpu) return find_acpi_cpu_topology_tag(cpu, PPTT_ABORT_PACKAGE, ACPI_PPTT_ACPI_IDENTICAL); } + +/** + * acpi_pptt_get_child_cpus() - Find all the CPUs below a PPTT + * processor hierarchy node + * + * @table_hdr: A reference to the PPTT table + * @parent_node: A pointer to the processor hierarchy node in the + * table_hdr + * @cpus: A cpumask to fill with the CPUs below @parent_node + * + * Walks up the PPTT from every possible CPU to find if the provided + * @parent_node is a parent of this CPU. + */ +static void acpi_pptt_get_child_cpus(struct acpi_table_header *table_hdr, + struct acpi_pptt_processor *parent_node, + cpumask_t *cpus) +{ + struct acpi_pptt_processor *cpu_node; + u32 acpi_id; + int cpu; + + cpumask_clear(cpus); + + for_each_possible_cpu(cpu) { + acpi_id = get_acpi_id_for_cpu(cpu); + cpu_node = acpi_find_processor_node(table_hdr, acpi_id); + + while (cpu_node) { + if (cpu_node == parent_node) { + cpumask_set_cpu(cpu, cpus); + break; + } + cpu_node = fetch_pptt_node(table_hdr, cpu_node->parent); + } + } +} + +/** + * acpi_pptt_get_cpus_from_container() - Populate a cpumask with all CPUs in a + * processor container + * @acpi_cpu_id: The UID of the processor container + * @cpus: The resulting CPU mask + * + * Find the specified Processor Container, and fill @cpus with all the cpus + * below it. + * + * Not all 'Processor Hierarchy' entries in the PPTT are either a CPU + * or a Processor Container, they may exist purely to describe a + * Private resource. CPUs have to be leaves, so a Processor Container + * is a non-leaf that has the 'ACPI Processor ID valid' flag set. + */ +void acpi_pptt_get_cpus_from_container(u32 acpi_cpu_id, cpumask_t *cpus) +{ + struct acpi_table_header *table_hdr; + struct acpi_subtable_header *entry; + unsigned long table_end; + u32 proc_sz; + + cpumask_clear(cpus); + + table_hdr = acpi_get_pptt(); + if (!table_hdr) + return; + + table_end = (unsigned long)table_hdr + table_hdr->length; + entry = ACPI_ADD_PTR(struct acpi_subtable_header, table_hdr, + sizeof(struct acpi_table_pptt)); + proc_sz = sizeof(struct acpi_pptt_processor); + while ((unsigned long)entry + proc_sz <= table_end) { + if (entry->type == ACPI_PPTT_TYPE_PROCESSOR) { + struct acpi_pptt_processor *cpu_node; + + cpu_node = (struct acpi_pptt_processor *)entry; + if (cpu_node->flags & ACPI_PPTT_ACPI_PROCESSOR_ID_VALID && + !acpi_pptt_leaf_node(table_hdr, cpu_node) && + cpu_node->acpi_processor_id == acpi_cpu_id) { + acpi_pptt_get_child_cpus(table_hdr, cpu_node, cpus); + break; + } + } + entry = ACPI_ADD_PTR(struct acpi_subtable_header, entry, + entry->length); + } +} + +/** + * find_acpi_cache_level_from_id() - Get the level of the specified cache + * @cache_id: The id field of the cache + * + * Determine the level relative to any CPU for the cache identified by + * cache_id. This allows the property to be found even if the CPUs are offline. + * + * The returned level can be used to group caches that are peers. + * + * The PPTT table must be rev 3 or later. + * + * If one CPU's L2 is shared with another CPU as L3, this function will return + * an unpredictable value. + * + * Return: -ENOENT if the PPTT doesn't exist, the revision isn't supported or + * the cache cannot be found. + * Otherwise returns a value which represents the level of the specified cache. + */ +int find_acpi_cache_level_from_id(u32 cache_id) +{ + int cpu; + struct acpi_table_header *table; + + table = acpi_get_pptt(); + if (!table) + return -ENOENT; + + if (table->revision < 3) + return -ENOENT; + + for_each_possible_cpu(cpu) { + bool empty; + int level = 1; + u32 acpi_cpu_id = get_acpi_id_for_cpu(cpu); + struct acpi_pptt_cache *cache; + struct acpi_pptt_processor *cpu_node; + + cpu_node = acpi_find_processor_node(table, acpi_cpu_id); + if (!cpu_node) + continue; + + do { + int cache_type[] = {CACHE_TYPE_INST, CACHE_TYPE_DATA, CACHE_TYPE_UNIFIED}; + + empty = true; + for (int i = 0; i < ARRAY_SIZE(cache_type); i++) { + struct acpi_pptt_cache_v1_full *cache_v1; + + cache = acpi_find_cache_node(table, acpi_cpu_id, cache_type[i], + level, &cpu_node); + if (!cache) + continue; + + empty = false; + + cache_v1 = upgrade_pptt_cache(cache); + if (cache_v1 && cache_v1->cache_id == cache_id) + return level; + } + level++; + } while (!empty); + } + + return -ENOENT; +} + +/** + * acpi_pptt_get_cpumask_from_cache_id() - Get the cpus associated with the + * specified cache + * @cache_id: The id field of the cache + * @cpus: Where to build the cpumask + * + * Determine which CPUs are below this cache in the PPTT. This allows the property + * to be found even if the CPUs are offline. + * + * The PPTT table must be rev 3 or later, + * + * Return: -ENOENT if the PPTT doesn't exist, or the cache cannot be found. + * Otherwise returns 0 and sets the cpus in the provided cpumask. + */ +int acpi_pptt_get_cpumask_from_cache_id(u32 cache_id, cpumask_t *cpus) +{ + int cpu; + struct acpi_table_header *table; + + cpumask_clear(cpus); + + table = acpi_get_pptt(); + if (!table) + return -ENOENT; + + if (table->revision < 3) + return -ENOENT; + + for_each_possible_cpu(cpu) { + bool empty; + int level = 1; + u32 acpi_cpu_id = get_acpi_id_for_cpu(cpu); + struct acpi_pptt_cache *cache; + struct acpi_pptt_processor *cpu_node; + + cpu_node = acpi_find_processor_node(table, acpi_cpu_id); + if (!cpu_node) + continue; + + do { + int cache_type[] = {CACHE_TYPE_INST, CACHE_TYPE_DATA, CACHE_TYPE_UNIFIED}; + + empty = true; + for (int i = 0; i < ARRAY_SIZE(cache_type); i++) { + struct acpi_pptt_cache_v1_full *cache_v1; + + cache = acpi_find_cache_node(table, acpi_cpu_id, cache_type[i], + level, &cpu_node); + + if (!cache) + continue; + + empty = false; + + cache_v1 = upgrade_pptt_cache(cache); + if (cache_v1 && cache_v1->cache_id == cache_id) + cpumask_set_cpu(cpu, cpus); + } + level++; + } while (!empty); + } + + return 0; +} diff --git a/drivers/acpi/prmt.c b/drivers/acpi/prmt.c index 6792d4385eee..7b8b5d2015ec 100644 --- a/drivers/acpi/prmt.c +++ b/drivers/acpi/prmt.c @@ -244,6 +244,12 @@ static struct prm_handler_info *find_prm_handler(const guid_t *guid) return (struct prm_handler_info *) find_guid_info(guid, GET_HANDLER); } +bool acpi_prm_handler_available(const guid_t *guid) +{ + return find_prm_handler(guid) && find_prm_module(guid); +} +EXPORT_SYMBOL_GPL(acpi_prm_handler_available); + /* In-coming PRM commands */ #define PRM_CMD_RUN_SERVICE 0 diff --git a/drivers/acpi/processor_core.c b/drivers/acpi/processor_core.c index 9b6b71a2ffb5..a4498357bd16 100644 --- a/drivers/acpi/processor_core.c +++ b/drivers/acpi/processor_core.c @@ -54,7 +54,7 @@ static int map_x2apic_id(struct acpi_subtable_header *entry, if (!(apic->lapic_flags & ACPI_MADT_ENABLED)) return -ENODEV; - if (device_declaration && (apic->uid == acpi_id)) { + if (apic->uid == acpi_id && (device_declaration || acpi_id < 255)) { *apic_id = apic->local_apic_id; return 0; } diff --git a/drivers/acpi/processor_driver.c b/drivers/acpi/processor_driver.c index 7644de24d2fa..65e779be64ff 100644 --- a/drivers/acpi/processor_driver.c +++ b/drivers/acpi/processor_driver.c @@ -166,7 +166,7 @@ static int __acpi_processor_start(struct acpi_device *device) if (result && !IS_ENABLED(CONFIG_ACPI_CPU_FREQ_PSS)) dev_dbg(&device->dev, "CPPC data invalid or not present\n"); - if (cpuidle_get_driver() == &acpi_idle_driver) + if (!cpuidle_get_driver() || cpuidle_get_driver() == &acpi_idle_driver) acpi_processor_power_init(pr); acpi_pss_perf_init(pr); diff --git a/drivers/acpi/processor_idle.c b/drivers/acpi/processor_idle.c index 4166090db642..89f2f08b2554 100644 --- a/drivers/acpi/processor_idle.c +++ b/drivers/acpi/processor_idle.c @@ -732,18 +732,16 @@ static int __cpuidle acpi_idle_enter_s2idle(struct cpuidle_device *dev, return 0; } -static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, - struct cpuidle_device *dev) +static void acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, + struct cpuidle_device *dev) { int i, count = ACPI_IDLE_STATE_START; struct acpi_processor_cx *cx; - struct cpuidle_state *state; if (max_cstate == 0) max_cstate = 1; for (i = 1; i < ACPI_PROCESSOR_MAX_POWER && i <= max_cstate; i++) { - state = &acpi_idle_driver.states[count]; cx = &pr->power.states[i]; if (!cx->valid) @@ -751,27 +749,13 @@ static int acpi_processor_setup_cpuidle_cx(struct acpi_processor *pr, per_cpu(acpi_cstate[count], dev->cpu) = cx; - if (lapic_timer_needs_broadcast(pr, cx)) - state->flags |= CPUIDLE_FLAG_TIMER_STOP; - - if (cx->type == ACPI_STATE_C3) { - state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; - if (pr->flags.bm_check) - state->flags |= CPUIDLE_FLAG_RCU_IDLE; - } - count++; if (count == CPUIDLE_STATE_MAX) break; } - - if (!count) - return -EINVAL; - - return 0; } -static int acpi_processor_setup_cstates(struct acpi_processor *pr) +static void acpi_processor_setup_cstates(struct acpi_processor *pr) { int i, count; struct acpi_processor_cx *cx; @@ -818,17 +802,21 @@ static int acpi_processor_setup_cstates(struct acpi_processor *pr) if (cx->type != ACPI_STATE_C1 && !acpi_idle_fallback_to_c1(pr)) state->enter_s2idle = acpi_idle_enter_s2idle; + if (lapic_timer_needs_broadcast(pr, cx)) + state->flags |= CPUIDLE_FLAG_TIMER_STOP; + + if (cx->type == ACPI_STATE_C3) { + state->flags |= CPUIDLE_FLAG_TLB_FLUSHED; + if (pr->flags.bm_check) + state->flags |= CPUIDLE_FLAG_RCU_IDLE; + } + count++; if (count == CPUIDLE_STATE_MAX) break; } drv->state_count = count; - - if (!count) - return -EINVAL; - - return 0; } static inline void acpi_processor_cstate_first_run_checks(void) @@ -1243,7 +1231,8 @@ static int acpi_processor_setup_cpuidle_states(struct acpi_processor *pr) if (pr->flags.has_lpi) return acpi_processor_setup_lpi_states(pr); - return acpi_processor_setup_cstates(pr); + acpi_processor_setup_cstates(pr); + return 0; } /** @@ -1263,7 +1252,8 @@ static int acpi_processor_setup_cpuidle_dev(struct acpi_processor *pr, if (pr->flags.has_lpi) return acpi_processor_ffh_lpi_probe(pr->id); - return acpi_processor_setup_cpuidle_cx(pr, dev); + acpi_processor_setup_cpuidle_cx(pr, dev); + return 0; } static int acpi_processor_get_power_info(struct acpi_processor *pr) diff --git a/drivers/acpi/property.c b/drivers/acpi/property.c index 43d5e457814e..18e90067d567 100644 --- a/drivers/acpi/property.c +++ b/drivers/acpi/property.c @@ -1280,7 +1280,7 @@ static int acpi_data_prop_read(const struct acpi_device_data *data, ret = acpi_copy_property_array_uint(items, (u64 *)val, nval); break; case DEV_PROP_STRING: - nval = min_t(u32, nval, obj->package.count); + nval = min(nval, obj->package.count); if (nval == 0) return -ENODATA; @@ -1329,13 +1329,14 @@ static int stop_on_next(struct acpi_device *adev, void *data) return 0; } -/** +/* * acpi_get_next_subnode - Return the next child node handle for a fwnode * @fwnode: Firmware node to find the next child node for. * @child: Handle to one of the device's child nodes or a null handle. */ -struct fwnode_handle *acpi_get_next_subnode(const struct fwnode_handle *fwnode, - struct fwnode_handle *child) +static struct fwnode_handle * +acpi_get_next_subnode(const struct fwnode_handle *fwnode, + struct fwnode_handle *child) { struct acpi_device *adev = to_acpi_device_node(fwnode); @@ -1472,7 +1473,7 @@ static struct fwnode_handle *acpi_graph_get_next_endpoint( if (!prev) { do { - port = fwnode_get_next_child_node(fwnode, port); + port = acpi_get_next_subnode(fwnode, port); /* * The names of the port nodes begin with "port@" * followed by the number of the port node and they also @@ -1490,14 +1491,17 @@ static struct fwnode_handle *acpi_graph_get_next_endpoint( if (!port) return NULL; - endpoint = fwnode_get_next_child_node(port, prev); - while (!endpoint) { - port = fwnode_get_next_child_node(fwnode, port); - if (!port) + do { + endpoint = acpi_get_next_subnode(port, prev); + if (endpoint) break; - if (is_acpi_graph_node(port, "port")) - endpoint = fwnode_get_next_child_node(port, NULL); - } + + prev = NULL; + + do { + port = acpi_get_next_subnode(fwnode, port); + } while (port && !is_acpi_graph_node(port, "port")); + } while (port); /* * The names of the endpoint nodes begin with "endpoint@" followed by @@ -1714,6 +1718,7 @@ static int acpi_fwnode_graph_parse_endpoint(const struct fwnode_handle *fwnode, if (fwnode_property_read_u32(fwnode, "reg", &endpoint->id)) fwnode_property_read_u32(fwnode, "endpoint", &endpoint->id); + fwnode_handle_put(port_fwnode); return 0; } diff --git a/drivers/acpi/scan.c b/drivers/acpi/scan.c index ef16d58b2949..416d87f9bd10 100644 --- a/drivers/acpi/scan.c +++ b/drivers/acpi/scan.c @@ -2397,7 +2397,7 @@ static bool acpi_scan_clear_dep_queue(struct acpi_device *adev) * initial enumeration of devices is complete, put it into the unbound * workqueue. */ - queue_work(system_unbound_wq, &cdw->work); + queue_work(system_dfl_wq, &cdw->work); return true; } @@ -2711,7 +2711,6 @@ void __init acpi_scan_init(void) acpi_watchdog_init(); acpi_pnp_init(); acpi_power_resources_init(); - acpi_int340x_thermal_init(); acpi_init_lpit(); acpi_scan_add_handler(&generic_device_handler); diff --git a/drivers/acpi/sleep.c b/drivers/acpi/sleep.c index c8ee8e42b0f6..68943b98333d 100644 --- a/drivers/acpi/sleep.c +++ b/drivers/acpi/sleep.c @@ -642,7 +642,7 @@ static int acpi_suspend_enter(suspend_state_t pm_state) /* * Disable all GPE and clear their status bits before interrupts are * enabled. Some GPEs (like wakeup GPEs) have no handlers and this can - * prevent them from producing spurious interrups. + * prevent them from producing spurious interrupts. * * acpi_leave_sleep_state() will reenable specific GPEs later. * diff --git a/drivers/acpi/sleep.h b/drivers/acpi/sleep.h index d960a238be4e..9c3cb109c5d2 100644 --- a/drivers/acpi/sleep.h +++ b/drivers/acpi/sleep.h @@ -17,10 +17,7 @@ static inline acpi_status acpi_set_waking_vector(u32 wakeup_address) extern int acpi_s2idle_begin(void); extern int acpi_s2idle_prepare(void); -extern int acpi_s2idle_prepare_late(void); -extern void acpi_s2idle_check(void); extern bool acpi_s2idle_wake(void); -extern void acpi_s2idle_restore_early(void); extern void acpi_s2idle_restore(void); extern void acpi_s2idle_end(void); diff --git a/drivers/acpi/tables.c b/drivers/acpi/tables.c index 57fc8bc56166..4286e4af1092 100644 --- a/drivers/acpi/tables.c +++ b/drivers/acpi/tables.c @@ -408,7 +408,7 @@ static const char table_sigs[][ACPI_NAMESEG_SIZE] __nonstring_array __initconst ACPI_SIG_PSDT, ACPI_SIG_RSDT, ACPI_SIG_XSDT, ACPI_SIG_SSDT, ACPI_SIG_IORT, ACPI_SIG_NFIT, ACPI_SIG_HMAT, ACPI_SIG_PPTT, ACPI_SIG_NHLT, ACPI_SIG_AEST, ACPI_SIG_CEDT, ACPI_SIG_AGDI, - ACPI_SIG_NBFT, ACPI_SIG_SWFT}; + ACPI_SIG_NBFT, ACPI_SIG_SWFT, ACPI_SIG_MPAM}; #define ACPI_HEADER_SIZE sizeof(struct acpi_table_header) diff --git a/drivers/acpi/thermal.c b/drivers/acpi/thermal.c index 8537395b417b..a511f9ea0267 100644 --- a/drivers/acpi/thermal.c +++ b/drivers/acpi/thermal.c @@ -1060,7 +1060,8 @@ static int __init acpi_thermal_init(void) } acpi_thermal_pm_queue = alloc_workqueue("acpi_thermal_pm", - WQ_HIGHPRI | WQ_MEM_RECLAIM, 0); + WQ_HIGHPRI | WQ_MEM_RECLAIM | WQ_PERCPU, + 0); if (!acpi_thermal_pm_queue) return -ENODEV; diff --git a/drivers/acpi/x86/lpss.c b/drivers/acpi/x86/lpss.c index 6daa6372f980..1dcb80ab0d23 100644 --- a/drivers/acpi/x86/lpss.c +++ b/drivers/acpi/x86/lpss.c @@ -181,7 +181,7 @@ static void byt_i2c_setup(struct lpss_private_data *pdata) acpi_status status; u64 uid; - /* Expected to always be successfull, but better safe then sorry */ + /* Expected to always be successful, but better safe then sorry */ if (!acpi_dev_uid_to_integer(pdata->adev, &uid) && uid) { /* Detect I2C bus shared with PUNIT and ignore its d3 status */ status = acpi_evaluate_integer(handle, "_SEM", NULL, &shared_host); diff --git a/drivers/acpi/x86/s2idle.c b/drivers/acpi/x86/s2idle.c index dd0b40b9bbe8..6d4d06236f61 100644 --- a/drivers/acpi/x86/s2idle.c +++ b/drivers/acpi/x86/s2idle.c @@ -299,34 +299,13 @@ free_acpi_buffer: ACPI_FREE(out_obj); } -/** - * acpi_get_lps0_constraint - Get the LPS0 constraint for a device. - * @adev: Device to get the constraint for. - * - * The LPS0 constraint is the shallowest (minimum) power state in which the - * device can be so as to allow the platform as a whole to achieve additional - * energy conservation by utilizing a system-wide low-power state. - * - * Returns: - * - ACPI power state value of the constraint for @adev on success. - * - Otherwise, ACPI_STATE_UNKNOWN. - */ -int acpi_get_lps0_constraint(struct acpi_device *adev) -{ - struct lpi_constraints *entry; - - for_each_lpi_constraint(entry) { - if (adev->handle == entry->handle) - return entry->min_dstate; - } - - return ACPI_STATE_UNKNOWN; -} - static void lpi_check_constraints(void) { struct lpi_constraints *entry; + if (IS_ERR_OR_NULL(lpi_constraints_table)) + return; + for_each_lpi_constraint(entry) { struct acpi_device *adev = acpi_fetch_acpi_dev(entry->handle); @@ -508,11 +487,6 @@ static int lps0_device_attach(struct acpi_device *adev, lps0_device_handle = adev->handle; - if (acpi_s2idle_vendor_amd()) - lpi_device_get_constraints_amd(); - else - lpi_device_get_constraints(); - /* * Use suspend-to-idle by default if ACPI_FADT_LOW_POWER_S0 is set in * the FADT and the default suspend mode was not set from the command @@ -539,7 +513,26 @@ static struct acpi_scan_handler lps0_handler = { .attach = lps0_device_attach, }; -int acpi_s2idle_prepare_late(void) +static int acpi_s2idle_begin_lps0(void) +{ + if (pm_debug_messages_on && !lpi_constraints_table) { + if (acpi_s2idle_vendor_amd()) + lpi_device_get_constraints_amd(); + else + lpi_device_get_constraints(); + + /* + * Try to retrieve the constraints only once because failures + * to do so usually are sticky. + */ + if (!lpi_constraints_table) + lpi_constraints_table = ERR_PTR(-ENODATA); + } + + return acpi_s2idle_begin(); +} + +static int acpi_s2idle_prepare_late_lps0(void) { struct acpi_s2idle_dev_ops *handler; @@ -585,7 +578,7 @@ int acpi_s2idle_prepare_late(void) return 0; } -void acpi_s2idle_check(void) +static void acpi_s2idle_check_lps0(void) { struct acpi_s2idle_dev_ops *handler; @@ -598,7 +591,7 @@ void acpi_s2idle_check(void) } } -void acpi_s2idle_restore_early(void) +static void acpi_s2idle_restore_early_lps0(void) { struct acpi_s2idle_dev_ops *handler; @@ -636,12 +629,12 @@ void acpi_s2idle_restore_early(void) } static const struct platform_s2idle_ops acpi_s2idle_ops_lps0 = { - .begin = acpi_s2idle_begin, + .begin = acpi_s2idle_begin_lps0, .prepare = acpi_s2idle_prepare, - .prepare_late = acpi_s2idle_prepare_late, - .check = acpi_s2idle_check, + .prepare_late = acpi_s2idle_prepare_late_lps0, + .check = acpi_s2idle_check_lps0, .wake = acpi_s2idle_wake, - .restore_early = acpi_s2idle_restore_early, + .restore_early = acpi_s2idle_restore_early_lps0, .restore = acpi_s2idle_restore, .end = acpi_s2idle_end, }; diff --git a/drivers/android/binder/error.rs b/drivers/android/binder/error.rs index 9921827267d0..b24497cfa292 100644 --- a/drivers/android/binder/error.rs +++ b/drivers/android/binder/error.rs @@ -2,6 +2,7 @@ // Copyright (C) 2025 Google LLC. +use kernel::fmt; use kernel::prelude::*; use crate::defs::*; @@ -76,8 +77,8 @@ impl From<kernel::alloc::AllocError> for BinderError { } } -impl core::fmt::Debug for BinderError { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { +impl fmt::Debug for BinderError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self.reply { BR_FAILED_REPLY => match self.source.as_ref() { Some(source) => f diff --git a/drivers/android/binder/freeze.rs b/drivers/android/binder/freeze.rs index 220de35ae85a..53b60035639a 100644 --- a/drivers/android/binder/freeze.rs +++ b/drivers/android/binder/freeze.rs @@ -331,7 +331,7 @@ impl Process { KVVec::with_capacity(8, GFP_KERNEL).unwrap_or_else(|_err| KVVec::new()); let mut inner = self.lock_with_nodes(); - let mut curr = inner.nodes.cursor_front(); + let mut curr = inner.nodes.cursor_front_mut(); while let Some(cursor) = curr { let (key, node) = cursor.current(); let key = *key; @@ -345,7 +345,7 @@ impl Process { // Find the node we were looking at and try again. If the set of nodes was changed, // then just proceed to the next node. This is ok because we don't guarantee the // inclusion of nodes that are added or removed in parallel with this operation. - curr = inner.nodes.cursor_lower_bound(&key); + curr = inner.nodes.cursor_lower_bound_mut(&key); continue; } diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index 7607353a5e92..e5237e9ec552 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -623,7 +623,7 @@ impl Process { " ref {}: desc {} {}node {debug_id} s {strong} w {weak}", r.debug_id, r.handle, - if dead { "dead " } else { "" }, + if dead { "dead " } else { "" } ); } } @@ -1320,7 +1320,7 @@ impl Process { { while let Some(node) = { let mut lock = self.inner.lock(); - lock.nodes.cursor_front().map(|c| c.remove_current().1) + lock.nodes.cursor_front_mut().map(|c| c.remove_current().1) } { node.to_key_value().1.release(); } diff --git a/drivers/android/binder/range_alloc/tree.rs b/drivers/android/binder/range_alloc/tree.rs index 7b1a248fcb02..838fdd2b47ea 100644 --- a/drivers/android/binder/range_alloc/tree.rs +++ b/drivers/android/binder/range_alloc/tree.rs @@ -207,7 +207,7 @@ impl<T> TreeRangeAllocator<T> { } pub(crate) fn reservation_abort(&mut self, offset: usize) -> Result<FreedRange> { - let mut cursor = self.tree.cursor_lower_bound(&offset).ok_or_else(|| { + let mut cursor = self.tree.cursor_lower_bound_mut(&offset).ok_or_else(|| { pr_warn!( "EINVAL from range_alloc.reservation_abort - offset: {}", offset diff --git a/drivers/android/binder/stats.rs b/drivers/android/binder/stats.rs index a83ec111d2cb..037002651941 100644 --- a/drivers/android/binder/stats.rs +++ b/drivers/android/binder/stats.rs @@ -61,7 +61,7 @@ impl BinderStats { mod strings { use core::str::from_utf8_unchecked; - use kernel::str::CStr; + use kernel::str::{CStr, CStrExt as _}; extern "C" { static binder_command_strings: [*const u8; super::BC_COUNT]; @@ -72,7 +72,7 @@ mod strings { // SAFETY: Accessing `binder_command_strings` is always safe. let c_str_ptr = unsafe { binder_command_strings[i] }; // SAFETY: The `binder_command_strings` array only contains nul-terminated strings. - let bytes = unsafe { CStr::from_char_ptr(c_str_ptr) }.as_bytes(); + let bytes = unsafe { CStr::from_char_ptr(c_str_ptr) }.to_bytes(); // SAFETY: The `binder_command_strings` array only contains strings with ascii-chars. unsafe { from_utf8_unchecked(bytes) } } @@ -81,7 +81,7 @@ mod strings { // SAFETY: Accessing `binder_return_strings` is always safe. let c_str_ptr = unsafe { binder_return_strings[i] }; // SAFETY: The `binder_command_strings` array only contains nul-terminated strings. - let bytes = unsafe { CStr::from_char_ptr(c_str_ptr) }.as_bytes(); + let bytes = unsafe { CStr::from_char_ptr(c_str_ptr) }.to_bytes(); // SAFETY: The `binder_command_strings` array only contains strings with ascii-chars. unsafe { from_utf8_unchecked(bytes) } } diff --git a/drivers/base/devtmpfs.c b/drivers/base/devtmpfs.c index 9d4e46ad8352..2f576ecf1832 100644 --- a/drivers/base/devtmpfs.c +++ b/drivers/base/devtmpfs.c @@ -180,7 +180,7 @@ static int dev_mkdir(const char *name, umode_t mode) if (IS_ERR(dentry)) return PTR_ERR(dentry); - dentry = vfs_mkdir(&nop_mnt_idmap, d_inode(path.dentry), dentry, mode); + dentry = vfs_mkdir(&nop_mnt_idmap, d_inode(path.dentry), dentry, mode, NULL); if (!IS_ERR(dentry)) /* mark as kernel-created inode */ d_inode(dentry)->i_private = &thread; @@ -231,7 +231,7 @@ static int handle_create(const char *nodename, umode_t mode, kuid_t uid, return PTR_ERR(dentry); err = vfs_mknod(&nop_mnt_idmap, d_inode(path.dentry), dentry, mode, - dev->devt); + dev->devt, NULL); if (!err) { struct iattr newattrs; @@ -261,7 +261,7 @@ static int dev_rmdir(const char *name) return PTR_ERR(dentry); if (d_inode(dentry)->i_private == &thread) err = vfs_rmdir(&nop_mnt_idmap, d_inode(parent.dentry), - dentry); + dentry, NULL); else err = -EPERM; diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c index 6942c62fa59d..bee3050a20d9 100644 --- a/drivers/base/firmware_loader/main.c +++ b/drivers/base/firmware_loader/main.c @@ -829,8 +829,6 @@ _request_firmware(const struct firmware **firmware_p, const char *name, size_t offset, u32 opt_flags) { struct firmware *fw = NULL; - struct cred *kern_cred = NULL; - const struct cred *old_cred; bool nondirect = false; int ret; @@ -871,45 +869,38 @@ _request_firmware(const struct firmware **firmware_p, const char *name, * called by a driver when serving an unrelated request from userland, we use * the kernel credentials to read the file. */ - kern_cred = prepare_kernel_cred(&init_task); - if (!kern_cred) { - ret = -ENOMEM; - goto out; - } - old_cred = override_creds(kern_cred); + scoped_with_kernel_creds() { + ret = fw_get_filesystem_firmware(device, fw->priv, "", NULL); - ret = fw_get_filesystem_firmware(device, fw->priv, "", NULL); - - /* Only full reads can support decompression, platform, and sysfs. */ - if (!(opt_flags & FW_OPT_PARTIAL)) - nondirect = true; + /* Only full reads can support decompression, platform, and sysfs. */ + if (!(opt_flags & FW_OPT_PARTIAL)) + nondirect = true; #ifdef CONFIG_FW_LOADER_COMPRESS_ZSTD - if (ret == -ENOENT && nondirect) - ret = fw_get_filesystem_firmware(device, fw->priv, ".zst", - fw_decompress_zstd); + if (ret == -ENOENT && nondirect) + ret = fw_get_filesystem_firmware(device, fw->priv, ".zst", + fw_decompress_zstd); #endif #ifdef CONFIG_FW_LOADER_COMPRESS_XZ - if (ret == -ENOENT && nondirect) - ret = fw_get_filesystem_firmware(device, fw->priv, ".xz", - fw_decompress_xz); + if (ret == -ENOENT && nondirect) + ret = fw_get_filesystem_firmware(device, fw->priv, ".xz", + fw_decompress_xz); #endif - if (ret == -ENOENT && nondirect) - ret = firmware_fallback_platform(fw->priv); + if (ret == -ENOENT && nondirect) + ret = firmware_fallback_platform(fw->priv); - if (ret) { - if (!(opt_flags & FW_OPT_NO_WARN)) - dev_warn(device, - "Direct firmware load for %s failed with error %d\n", - name, ret); - if (nondirect) - ret = firmware_fallback_sysfs(fw, name, device, - opt_flags, ret); - } else - ret = assign_fw(fw, device); - - revert_creds(old_cred); - put_cred(kern_cred); + if (ret) { + if (!(opt_flags & FW_OPT_NO_WARN)) + dev_warn(device, + "Direct firmware load for %s failed with error %d\n", + name, ret); + if (nondirect) + ret = firmware_fallback_sysfs(fw, name, device, + opt_flags, ret); + } else { + ret = assign_fw(fw, device); + } + } out: if (ret < 0) { diff --git a/drivers/base/memory.c b/drivers/base/memory.c index 6d84a02cfa5d..fc43f2703ae0 100644 --- a/drivers/base/memory.c +++ b/drivers/base/memory.c @@ -226,7 +226,6 @@ static int memory_block_online(struct memory_block *mem) unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; unsigned long nr_vmemmap_pages = 0; - struct memory_notify arg; struct zone *zone; int ret; @@ -246,19 +245,9 @@ static int memory_block_online(struct memory_block *mem) if (mem->altmap) nr_vmemmap_pages = mem->altmap->free; - arg.altmap_start_pfn = start_pfn; - arg.altmap_nr_pages = nr_vmemmap_pages; - arg.start_pfn = start_pfn + nr_vmemmap_pages; - arg.nr_pages = nr_pages - nr_vmemmap_pages; mem_hotplug_begin(); - ret = memory_notify(MEM_PREPARE_ONLINE, &arg); - ret = notifier_to_errno(ret); - if (ret) - goto out_notifier; - if (nr_vmemmap_pages) { - ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, - zone, mem->altmap->inaccessible); + ret = mhp_init_memmap_on_memory(start_pfn, nr_vmemmap_pages, zone); if (ret) goto out; } @@ -280,11 +269,7 @@ static int memory_block_online(struct memory_block *mem) nr_vmemmap_pages); mem->zone = zone; - mem_hotplug_done(); - return ret; out: - memory_notify(MEM_FINISH_OFFLINE, &arg); -out_notifier: mem_hotplug_done(); return ret; } @@ -297,7 +282,6 @@ static int memory_block_offline(struct memory_block *mem) unsigned long start_pfn = section_nr_to_pfn(mem->start_section_nr); unsigned long nr_pages = PAGES_PER_SECTION * sections_per_block; unsigned long nr_vmemmap_pages = 0; - struct memory_notify arg; int ret; if (!mem->zone) @@ -329,11 +313,6 @@ static int memory_block_offline(struct memory_block *mem) mhp_deinit_memmap_on_memory(start_pfn, nr_vmemmap_pages); mem->zone = NULL; - arg.altmap_start_pfn = start_pfn; - arg.altmap_nr_pages = nr_vmemmap_pages; - arg.start_pfn = start_pfn + nr_vmemmap_pages; - arg.nr_pages = nr_pages - nr_vmemmap_pages; - memory_notify(MEM_FINISH_OFFLINE, &arg); out: mem_hotplug_done(); return ret; diff --git a/drivers/base/platform.c b/drivers/base/platform.c index 09450349cf32..b45d41b018ca 100644 --- a/drivers/base/platform.c +++ b/drivers/base/platform.c @@ -150,25 +150,37 @@ devm_platform_ioremap_resource_byname(struct platform_device *pdev, EXPORT_SYMBOL_GPL(devm_platform_ioremap_resource_byname); #endif /* CONFIG_HAS_IOMEM */ +static const struct cpumask *get_irq_affinity(struct platform_device *dev, + unsigned int num) +{ + const struct cpumask *mask = NULL; +#ifndef CONFIG_SPARC + struct fwnode_handle *fwnode = dev_fwnode(&dev->dev); + + if (is_of_node(fwnode)) + mask = of_irq_get_affinity(to_of_node(fwnode), num); + else if (is_acpi_device_node(fwnode)) + mask = acpi_irq_get_affinity(ACPI_HANDLE_FWNODE(fwnode), num); +#endif + + return mask ?: cpu_possible_mask; +} + /** - * platform_get_irq_optional - get an optional IRQ for a device - * @dev: platform device - * @num: IRQ number index + * platform_get_irq_affinity - get an optional IRQ and its affinity for a device + * @dev: platform device + * @num: interrupt number index + * @affinity: optional cpumask pointer to get the affinity of a per-cpu interrupt * - * Gets an IRQ for a platform device. Device drivers should check the return - * value for errors so as to not pass a negative integer value to the - * request_irq() APIs. This is the same as platform_get_irq(), except that it - * does not print an error message if an IRQ can not be obtained. - * - * For example:: - * - * int irq = platform_get_irq_optional(pdev, 0); - * if (irq < 0) - * return irq; + * Gets an interupt for a platform device. Device drivers should check the + * return value for errors so as to not pass a negative integer value to + * the request_irq() APIs. Optional affinity information is provided in the + * affinity pointer if available, and NULL otherwise. * - * Return: non-zero IRQ number on success, negative error number on failure. + * Return: non-zero interrupt number on success, negative error number on failure. */ -int platform_get_irq_optional(struct platform_device *dev, unsigned int num) +int platform_get_irq_affinity(struct platform_device *dev, unsigned int num, + const struct cpumask **affinity) { int ret; #ifdef CONFIG_SPARC @@ -236,8 +248,37 @@ out_not_found: out: if (WARN(!ret, "0 is an invalid IRQ number\n")) return -EINVAL; + + if (ret > 0 && affinity) + *affinity = get_irq_affinity(dev, num); + return ret; } +EXPORT_SYMBOL_GPL(platform_get_irq_affinity); + +/** + * platform_get_irq_optional - get an optional interrupt for a device + * @dev: platform device + * @num: interrupt number index + * + * Gets an interrupt for a platform device. Device drivers should check the + * return value for errors so as to not pass a negative integer value to + * the request_irq() APIs. This is the same as platform_get_irq(), except + * that it does not print an error message if an interrupt can not be + * obtained. + * + * For example:: + * + * int irq = platform_get_irq_optional(pdev, 0); + * if (irq < 0) + * return irq; + * + * Return: non-zero interrupt number on success, negative error number on failure. + */ +int platform_get_irq_optional(struct platform_device *dev, unsigned int num) +{ + return platform_get_irq_affinity(dev, num, NULL); +} EXPORT_SYMBOL_GPL(platform_get_irq_optional); /** diff --git a/drivers/base/power/generic_ops.c b/drivers/base/power/generic_ops.c index 6502720bb564..af99bbcf281c 100644 --- a/drivers/base/power/generic_ops.c +++ b/drivers/base/power/generic_ops.c @@ -8,6 +8,13 @@ #include <linux/pm_runtime.h> #include <linux/export.h> +#define CALL_PM_OP(dev, op) \ +({ \ + struct device *_dev = (dev); \ + const struct dev_pm_ops *pm = _dev->driver ? _dev->driver->pm : NULL; \ + pm && pm->op ? pm->op(_dev) : 0; \ +}) + #ifdef CONFIG_PM /** * pm_generic_runtime_suspend - Generic runtime suspend callback for subsystems. @@ -19,12 +26,7 @@ */ int pm_generic_runtime_suspend(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int ret; - - ret = pm && pm->runtime_suspend ? pm->runtime_suspend(dev) : 0; - - return ret; + return CALL_PM_OP(dev, runtime_suspend); } EXPORT_SYMBOL_GPL(pm_generic_runtime_suspend); @@ -38,12 +40,7 @@ EXPORT_SYMBOL_GPL(pm_generic_runtime_suspend); */ int pm_generic_runtime_resume(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - int ret; - - ret = pm && pm->runtime_resume ? pm->runtime_resume(dev) : 0; - - return ret; + return CALL_PM_OP(dev, runtime_resume); } EXPORT_SYMBOL_GPL(pm_generic_runtime_resume); #endif /* CONFIG_PM */ @@ -72,9 +69,7 @@ int pm_generic_prepare(struct device *dev) */ int pm_generic_suspend_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->suspend_noirq ? pm->suspend_noirq(dev) : 0; + return CALL_PM_OP(dev, suspend_noirq); } EXPORT_SYMBOL_GPL(pm_generic_suspend_noirq); @@ -84,9 +79,7 @@ EXPORT_SYMBOL_GPL(pm_generic_suspend_noirq); */ int pm_generic_suspend_late(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->suspend_late ? pm->suspend_late(dev) : 0; + return CALL_PM_OP(dev, suspend_late); } EXPORT_SYMBOL_GPL(pm_generic_suspend_late); @@ -96,9 +89,7 @@ EXPORT_SYMBOL_GPL(pm_generic_suspend_late); */ int pm_generic_suspend(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->suspend ? pm->suspend(dev) : 0; + return CALL_PM_OP(dev, suspend); } EXPORT_SYMBOL_GPL(pm_generic_suspend); @@ -108,9 +99,7 @@ EXPORT_SYMBOL_GPL(pm_generic_suspend); */ int pm_generic_freeze_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->freeze_noirq ? pm->freeze_noirq(dev) : 0; + return CALL_PM_OP(dev, freeze_noirq); } EXPORT_SYMBOL_GPL(pm_generic_freeze_noirq); @@ -120,9 +109,7 @@ EXPORT_SYMBOL_GPL(pm_generic_freeze_noirq); */ int pm_generic_freeze(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->freeze ? pm->freeze(dev) : 0; + return CALL_PM_OP(dev, freeze); } EXPORT_SYMBOL_GPL(pm_generic_freeze); @@ -132,9 +119,7 @@ EXPORT_SYMBOL_GPL(pm_generic_freeze); */ int pm_generic_poweroff_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->poweroff_noirq ? pm->poweroff_noirq(dev) : 0; + return CALL_PM_OP(dev, poweroff_noirq); } EXPORT_SYMBOL_GPL(pm_generic_poweroff_noirq); @@ -144,9 +129,7 @@ EXPORT_SYMBOL_GPL(pm_generic_poweroff_noirq); */ int pm_generic_poweroff_late(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->poweroff_late ? pm->poweroff_late(dev) : 0; + return CALL_PM_OP(dev, poweroff_late); } EXPORT_SYMBOL_GPL(pm_generic_poweroff_late); @@ -156,9 +139,7 @@ EXPORT_SYMBOL_GPL(pm_generic_poweroff_late); */ int pm_generic_poweroff(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->poweroff ? pm->poweroff(dev) : 0; + return CALL_PM_OP(dev, poweroff); } EXPORT_SYMBOL_GPL(pm_generic_poweroff); @@ -168,9 +149,7 @@ EXPORT_SYMBOL_GPL(pm_generic_poweroff); */ int pm_generic_thaw_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->thaw_noirq ? pm->thaw_noirq(dev) : 0; + return CALL_PM_OP(dev, thaw_noirq); } EXPORT_SYMBOL_GPL(pm_generic_thaw_noirq); @@ -180,9 +159,7 @@ EXPORT_SYMBOL_GPL(pm_generic_thaw_noirq); */ int pm_generic_thaw(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->thaw ? pm->thaw(dev) : 0; + return CALL_PM_OP(dev, thaw); } EXPORT_SYMBOL_GPL(pm_generic_thaw); @@ -192,9 +169,7 @@ EXPORT_SYMBOL_GPL(pm_generic_thaw); */ int pm_generic_resume_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->resume_noirq ? pm->resume_noirq(dev) : 0; + return CALL_PM_OP(dev, resume_noirq); } EXPORT_SYMBOL_GPL(pm_generic_resume_noirq); @@ -204,9 +179,7 @@ EXPORT_SYMBOL_GPL(pm_generic_resume_noirq); */ int pm_generic_resume_early(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->resume_early ? pm->resume_early(dev) : 0; + return CALL_PM_OP(dev, resume_early); } EXPORT_SYMBOL_GPL(pm_generic_resume_early); @@ -216,9 +189,7 @@ EXPORT_SYMBOL_GPL(pm_generic_resume_early); */ int pm_generic_resume(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->resume ? pm->resume(dev) : 0; + return CALL_PM_OP(dev, resume); } EXPORT_SYMBOL_GPL(pm_generic_resume); @@ -228,9 +199,7 @@ EXPORT_SYMBOL_GPL(pm_generic_resume); */ int pm_generic_restore_noirq(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->restore_noirq ? pm->restore_noirq(dev) : 0; + return CALL_PM_OP(dev, restore_noirq); } EXPORT_SYMBOL_GPL(pm_generic_restore_noirq); @@ -240,9 +209,7 @@ EXPORT_SYMBOL_GPL(pm_generic_restore_noirq); */ int pm_generic_restore_early(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->restore_early ? pm->restore_early(dev) : 0; + return CALL_PM_OP(dev, restore_early); } EXPORT_SYMBOL_GPL(pm_generic_restore_early); @@ -252,9 +219,7 @@ EXPORT_SYMBOL_GPL(pm_generic_restore_early); */ int pm_generic_restore(struct device *dev) { - const struct dev_pm_ops *pm = dev->driver ? dev->driver->pm : NULL; - - return pm && pm->restore ? pm->restore(dev) : 0; + return CALL_PM_OP(dev, restore); } EXPORT_SYMBOL_GPL(pm_generic_restore); diff --git a/drivers/base/power/main.c b/drivers/base/power/main.c index 1de1cd72b616..a0225a83f50c 100644 --- a/drivers/base/power/main.c +++ b/drivers/base/power/main.c @@ -34,6 +34,7 @@ #include <linux/cpufreq.h> #include <linux/devfreq.h> #include <linux/timer.h> +#include <linux/nmi.h> #include "../base.h" #include "power.h" @@ -95,6 +96,8 @@ static const char *pm_verb(int event) return "restore"; case PM_EVENT_RECOVER: return "recover"; + case PM_EVENT_POWEROFF: + return "poweroff"; default: return "(unknown PM event)"; } @@ -367,6 +370,7 @@ static pm_callback_t pm_op(const struct dev_pm_ops *ops, pm_message_t state) case PM_EVENT_FREEZE: case PM_EVENT_QUIESCE: return ops->freeze; + case PM_EVENT_POWEROFF: case PM_EVENT_HIBERNATE: return ops->poweroff; case PM_EVENT_THAW: @@ -401,6 +405,7 @@ static pm_callback_t pm_late_early_op(const struct dev_pm_ops *ops, case PM_EVENT_FREEZE: case PM_EVENT_QUIESCE: return ops->freeze_late; + case PM_EVENT_POWEROFF: case PM_EVENT_HIBERNATE: return ops->poweroff_late; case PM_EVENT_THAW: @@ -435,6 +440,7 @@ static pm_callback_t pm_noirq_op(const struct dev_pm_ops *ops, pm_message_t stat case PM_EVENT_FREEZE: case PM_EVENT_QUIESCE: return ops->freeze_noirq; + case PM_EVENT_POWEROFF: case PM_EVENT_HIBERNATE: return ops->poweroff_noirq; case PM_EVENT_THAW: @@ -515,6 +521,11 @@ struct dpm_watchdog { #define DECLARE_DPM_WATCHDOG_ON_STACK(wd) \ struct dpm_watchdog wd +static bool __read_mostly dpm_watchdog_all_cpu_backtrace; +module_param(dpm_watchdog_all_cpu_backtrace, bool, 0644); +MODULE_PARM_DESC(dpm_watchdog_all_cpu_backtrace, + "Backtrace all CPUs on DPM watchdog timeout"); + /** * dpm_watchdog_handler - Driver suspend / resume watchdog handler. * @t: The timer that PM watchdog depends on. @@ -530,8 +541,12 @@ static void dpm_watchdog_handler(struct timer_list *t) unsigned int time_left; if (wd->fatal) { + unsigned int this_cpu = smp_processor_id(); + dev_emerg(wd->dev, "**** DPM device timeout ****\n"); show_stack(wd->tsk, NULL, KERN_EMERG); + if (dpm_watchdog_all_cpu_backtrace) + trigger_allbutcpu_cpu_backtrace(this_cpu); panic("%s %s: unrecoverable failure\n", dev_driver_string(wd->dev), dev_name(wd->dev)); } diff --git a/drivers/base/power/runtime.c b/drivers/base/power/runtime.c index 1b11a3cd4acc..62707738caa4 100644 --- a/drivers/base/power/runtime.c +++ b/drivers/base/power/runtime.c @@ -90,7 +90,7 @@ static void update_pm_runtime_accounting(struct device *dev) /* * Because ktime_get_mono_fast_ns() is not monotonic during * timekeeping updates, ensure that 'now' is after the last saved - * timesptamp. + * timestamp. */ if (now < last) return; @@ -217,7 +217,7 @@ static int dev_memalloc_noio(struct device *dev, void *data) * resume/suspend callback of any one of its ancestors(or the * block device itself), the deadlock may be triggered inside the * memory allocation since it might not complete until the block - * device becomes active and the involed page I/O finishes. The + * device becomes active and the involved page I/O finishes. The * situation is pointed out first by Alan Stern. Network device * are involved in iSCSI kind of situation. * @@ -1210,7 +1210,7 @@ EXPORT_SYMBOL_GPL(__pm_runtime_resume); * * Otherwise, if its runtime PM status is %RPM_ACTIVE and (1) @ign_usage_count * is set, or (2) @dev is not ignoring children and its active child count is - * nonero, or (3) the runtime PM usage counter of @dev is not zero, increment + * nonzero, or (3) the runtime PM usage counter of @dev is not zero, increment * the usage counter of @dev and return 1. * * Otherwise, return 0 without changing the usage counter. @@ -1664,9 +1664,12 @@ EXPORT_SYMBOL_GPL(devm_pm_runtime_get_noresume); * pm_runtime_forbid - Block runtime PM of a device. * @dev: Device to handle. * - * Increase the device's usage count and clear its power.runtime_auto flag, - * so that it cannot be suspended at run time until pm_runtime_allow() is called - * for it. + * Resume @dev if already suspended and block runtime suspend of @dev in such + * a way that it can be unblocked via the /sys/devices/.../power/control + * interface, or otherwise by calling pm_runtime_allow(). + * + * Calling this function many times in a row has the same effect as calling it + * once. */ void pm_runtime_forbid(struct device *dev) { @@ -1687,7 +1690,13 @@ EXPORT_SYMBOL_GPL(pm_runtime_forbid); * pm_runtime_allow - Unblock runtime PM of a device. * @dev: Device to handle. * - * Decrease the device's usage count and set its power.runtime_auto flag. + * Unblock runtime suspend of @dev after it has been blocked by + * pm_runtime_forbid() (for instance, if it has been blocked via the + * /sys/devices/.../power/control interface), check if @dev can be + * suspended and suspend it in that case. + * + * Calling this function many times in a row has the same effect as calling it + * once. */ void pm_runtime_allow(struct device *dev) { diff --git a/drivers/base/power/trace.c b/drivers/base/power/trace.c index cd6e559648b2..d8da7195bb00 100644 --- a/drivers/base/power/trace.c +++ b/drivers/base/power/trace.c @@ -238,10 +238,8 @@ int show_trace_dev_match(char *buf, size_t size) unsigned int hash = hash_string(DEVSEED, dev_name(dev), DEVHASH); if (hash == value) { - int len = snprintf(buf, size, "%s\n", + int len = scnprintf(buf, size, "%s\n", dev_driver_string(dev)); - if (len > size) - len = size; buf += len; ret += len; size -= len; diff --git a/drivers/base/power/wakeup.c b/drivers/base/power/wakeup.c index d1283ff1080b..1e1a0e7eeac5 100644 --- a/drivers/base/power/wakeup.c +++ b/drivers/base/power/wakeup.c @@ -189,17 +189,16 @@ static void wakeup_source_remove(struct wakeup_source *ws) if (WARN_ON(!ws)) return; + /* + * After shutting down the timer, wakeup_source_activate() will warn if + * the given wakeup source is passed to it. + */ + timer_shutdown_sync(&ws->timer); + raw_spin_lock_irqsave(&events_lock, flags); list_del_rcu(&ws->entry); raw_spin_unlock_irqrestore(&events_lock, flags); synchronize_srcu(&wakeup_srcu); - - timer_delete_sync(&ws->timer); - /* - * Clear timer.function to make wakeup_source_not_registered() treat - * this wakeup source as not registered. - */ - ws->timer.function = NULL; } /** @@ -506,14 +505,14 @@ int device_set_wakeup_enable(struct device *dev, bool enable) EXPORT_SYMBOL_GPL(device_set_wakeup_enable); /** - * wakeup_source_not_registered - validate the given wakeup source. + * wakeup_source_not_usable - validate the given wakeup source. * @ws: Wakeup source to be validated. */ -static bool wakeup_source_not_registered(struct wakeup_source *ws) +static bool wakeup_source_not_usable(struct wakeup_source *ws) { /* - * Use timer struct to check if the given source is initialized - * by wakeup_source_add. + * Use the timer struct to check if the given wakeup source has been + * initialized by wakeup_source_add() and it is not going away. */ return ws->timer.function != pm_wakeup_timer_fn; } @@ -558,8 +557,7 @@ static void wakeup_source_activate(struct wakeup_source *ws) { unsigned int cec; - if (WARN_ONCE(wakeup_source_not_registered(ws), - "unregistered wakeup source\n")) + if (WARN_ONCE(wakeup_source_not_usable(ws), "unusable wakeup source\n")) return; ws->active = true; diff --git a/drivers/block/drbd/drbd_bitmap.c b/drivers/block/drbd/drbd_bitmap.c index 85ca000a0564..d90fa3e7f4cf 100644 --- a/drivers/block/drbd/drbd_bitmap.c +++ b/drivers/block/drbd/drbd_bitmap.c @@ -1210,7 +1210,7 @@ static int bm_rw(struct drbd_device *device, const unsigned int flags, unsigned return err; } -/** +/* * drbd_bm_read() - Read the whole bitmap from its on disk location. * @device: DRBD device. */ @@ -1221,7 +1221,7 @@ int drbd_bm_read(struct drbd_device *device, return bm_rw(device, BM_AIO_READ, 0); } -/** +/* * drbd_bm_write() - Write the whole bitmap to its on disk location. * @device: DRBD device. * @@ -1233,7 +1233,7 @@ int drbd_bm_write(struct drbd_device *device, return bm_rw(device, 0, 0); } -/** +/* * drbd_bm_write_all() - Write the whole bitmap to its on disk location. * @device: DRBD device. * @@ -1255,7 +1255,7 @@ int drbd_bm_write_lazy(struct drbd_device *device, unsigned upper_idx) __must_ho return bm_rw(device, BM_AIO_COPY_PAGES, upper_idx); } -/** +/* * drbd_bm_write_copy_pages() - Write the whole bitmap to its on disk location. * @device: DRBD device. * @@ -1272,7 +1272,7 @@ int drbd_bm_write_copy_pages(struct drbd_device *device, return bm_rw(device, BM_AIO_COPY_PAGES, 0); } -/** +/* * drbd_bm_write_hinted() - Write bitmap pages with "hint" marks, if they have changed. * @device: DRBD device. */ diff --git a/drivers/block/drbd/drbd_receiver.c b/drivers/block/drbd/drbd_receiver.c index 33bc91665fe8..3de919b6f0e1 100644 --- a/drivers/block/drbd/drbd_receiver.c +++ b/drivers/block/drbd/drbd_receiver.c @@ -1736,13 +1736,13 @@ read_in_block(struct drbd_peer_device *peer_device, u64 id, sector_t sector, page = peer_req->pages; page_chain_for_each(page) { unsigned len = min_t(int, ds, PAGE_SIZE); - data = kmap(page); + data = kmap_local_page(page); err = drbd_recv_all_warn(peer_device->connection, data, len); if (drbd_insert_fault(device, DRBD_FAULT_RECEIVE)) { drbd_err(device, "Fault injection: Corrupting data on receive\n"); data[0] = data[0] ^ (unsigned long)-1; } - kunmap(page); + kunmap_local(data); if (err) { drbd_free_peer_req(device, peer_req); return NULL; @@ -1777,7 +1777,7 @@ static int drbd_drain_block(struct drbd_peer_device *peer_device, int data_size) page = drbd_alloc_pages(peer_device, 1, 1); - data = kmap(page); + data = kmap_local_page(page); while (data_size) { unsigned int len = min_t(int, data_size, PAGE_SIZE); @@ -1786,7 +1786,7 @@ static int drbd_drain_block(struct drbd_peer_device *peer_device, int data_size) break; data_size -= len; } - kunmap(page); + kunmap_local(data); drbd_free_pages(peer_device->device, page); return err; } diff --git a/drivers/block/floppy.c b/drivers/block/floppy.c index 5336c3c5ca36..c28786e0fe1c 100644 --- a/drivers/block/floppy.c +++ b/drivers/block/floppy.c @@ -329,7 +329,7 @@ static bool initialized; * This default is used whenever the current disk size is unknown. * [Now it is rather a minimum] */ -#define MAX_DISK_SIZE 4 /* 3984 */ +#define MAX_DISK_SIZE (PAGE_SIZE / 1024) /* * globals used by 'result()' diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 13ce229d450c..ebe751f39742 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -1908,6 +1908,10 @@ static void loop_handle_cmd(struct loop_cmd *cmd) goto failed; } + /* We can block in this context, so ignore REQ_NOWAIT. */ + if (rq->cmd_flags & REQ_NOWAIT) + rq->cmd_flags &= ~REQ_NOWAIT; + if (cmd_blkcg_css) kthread_associate_blkcg(cmd_blkcg_css); if (cmd_memcg_css) diff --git a/drivers/block/nbd.c b/drivers/block/nbd.c index a853c65ac65d..f6c33b21f69e 100644 --- a/drivers/block/nbd.c +++ b/drivers/block/nbd.c @@ -52,7 +52,6 @@ static DEFINE_IDR(nbd_index_idr); static DEFINE_MUTEX(nbd_index_mutex); static struct workqueue_struct *nbd_del_wq; -static struct cred *nbd_cred; static int nbd_total_devices = 0; struct nbd_sock { @@ -555,7 +554,6 @@ static int __sock_xmit(struct nbd_device *nbd, struct socket *sock, int send, int result; struct msghdr msg = {} ; unsigned int noreclaim_flag; - const struct cred *old_cred; if (unlikely(!sock)) { dev_err_ratelimited(disk_to_dev(nbd->disk), @@ -564,33 +562,32 @@ static int __sock_xmit(struct nbd_device *nbd, struct socket *sock, int send, return -EINVAL; } - old_cred = override_creds(nbd_cred); - msg.msg_iter = *iter; noreclaim_flag = memalloc_noreclaim_save(); - do { - sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC; - sock->sk->sk_use_task_frag = false; - msg.msg_flags = msg_flags | MSG_NOSIGNAL; - - if (send) - result = sock_sendmsg(sock, &msg); - else - result = sock_recvmsg(sock, &msg, msg.msg_flags); - - if (result <= 0) { - if (result == 0) - result = -EPIPE; /* short read */ - break; - } - if (sent) - *sent += result; - } while (msg_data_left(&msg)); - memalloc_noreclaim_restore(noreclaim_flag); + scoped_with_kernel_creds() { + do { + sock->sk->sk_allocation = GFP_NOIO | __GFP_MEMALLOC; + sock->sk->sk_use_task_frag = false; + msg.msg_flags = msg_flags | MSG_NOSIGNAL; + + if (send) + result = sock_sendmsg(sock, &msg); + else + result = sock_recvmsg(sock, &msg, msg.msg_flags); - revert_creds(old_cred); + if (result <= 0) { + if (result == 0) + result = -EPIPE; /* short read */ + break; + } + if (sent) + *sent += result; + } while (msg_data_left(&msg)); + } + + memalloc_noreclaim_restore(noreclaim_flag); return result; } @@ -1024,9 +1021,9 @@ static void recv_work(struct work_struct *work) nbd_mark_nsock_dead(nbd, nsock, 1); mutex_unlock(&nsock->tx_lock); - nbd_config_put(nbd); atomic_dec(&config->recv_threads); wake_up(&config->recv_wq); + nbd_config_put(nbd); kfree(args); } @@ -2241,12 +2238,13 @@ again: ret = nbd_start_device(nbd); out: - mutex_unlock(&nbd->config_lock); if (!ret) { set_bit(NBD_RT_HAS_CONFIG_REF, &config->runtime_flags); refcount_inc(&nbd->config_refs); nbd_connect_reply(info, nbd->index); } + mutex_unlock(&nbd->config_lock); + nbd_config_put(nbd); if (put_dev) nbd_put(nbd); @@ -2683,15 +2681,7 @@ static int __init nbd_init(void) return -ENOMEM; } - nbd_cred = prepare_kernel_cred(&init_task); - if (!nbd_cred) { - destroy_workqueue(nbd_del_wq); - unregister_blkdev(NBD_MAJOR, "nbd"); - return -ENOMEM; - } - if (genl_register_family(&nbd_genl_family)) { - put_cred(nbd_cred); destroy_workqueue(nbd_del_wq); unregister_blkdev(NBD_MAJOR, "nbd"); return -EINVAL; @@ -2746,7 +2736,6 @@ static void __exit nbd_cleanup(void) /* Also wait for nbd_dev_remove_work() completes */ destroy_workqueue(nbd_del_wq); - put_cred(nbd_cred); idr_destroy(&nbd_index_idr); unregister_blkdev(NBD_MAJOR, "nbd"); } diff --git a/drivers/block/null_blk/main.c b/drivers/block/null_blk/main.c index 0ee55f889cfd..c7c0fb79a6bf 100644 --- a/drivers/block/null_blk/main.c +++ b/drivers/block/null_blk/main.c @@ -1129,26 +1129,28 @@ again: return 0; } -static int copy_to_nullb(struct nullb *nullb, struct page *source, - unsigned int off, sector_t sector, size_t n, bool is_fua) +static blk_status_t copy_to_nullb(struct nullb *nullb, void *source, + loff_t pos, size_t n, bool is_fua) { size_t temp, count = 0; - unsigned int offset; struct nullb_page *t_page; + sector_t sector; while (count < n) { - temp = min_t(size_t, nullb->dev->blocksize, n - count); + temp = min3(nullb->dev->blocksize, n - count, + PAGE_SIZE - offset_in_page(pos)); + sector = pos >> SECTOR_SHIFT; if (null_cache_active(nullb) && !is_fua) null_make_cache_space(nullb, PAGE_SIZE); - offset = (sector & SECTOR_MASK) << SECTOR_SHIFT; t_page = null_insert_page(nullb, sector, !null_cache_active(nullb) || is_fua); if (!t_page) - return -ENOSPC; + return BLK_STS_NOSPC; - memcpy_page(t_page->page, offset, source, off + count, temp); + memcpy_to_page(t_page->page, offset_in_page(pos), + source + count, temp); __set_bit(sector & SECTOR_MASK, t_page->bitmap); @@ -1156,41 +1158,34 @@ static int copy_to_nullb(struct nullb *nullb, struct page *source, null_free_sector(nullb, sector, true); count += temp; - sector += temp >> SECTOR_SHIFT; + pos += temp; } - return 0; + return BLK_STS_OK; } -static int copy_from_nullb(struct nullb *nullb, struct page *dest, - unsigned int off, sector_t sector, size_t n) +static void copy_from_nullb(struct nullb *nullb, void *dest, loff_t pos, + size_t n) { size_t temp, count = 0; - unsigned int offset; struct nullb_page *t_page; + sector_t sector; while (count < n) { - temp = min_t(size_t, nullb->dev->blocksize, n - count); + temp = min3(nullb->dev->blocksize, n - count, + PAGE_SIZE - offset_in_page(pos)); + sector = pos >> SECTOR_SHIFT; - offset = (sector & SECTOR_MASK) << SECTOR_SHIFT; t_page = null_lookup_page(nullb, sector, false, !null_cache_active(nullb)); - if (t_page) - memcpy_page(dest, off + count, t_page->page, offset, - temp); + memcpy_from_page(dest + count, t_page->page, + offset_in_page(pos), temp); else - memzero_page(dest, off + count, temp); + memset(dest + count, 0, temp); count += temp; - sector += temp >> SECTOR_SHIFT; + pos += temp; } - return 0; -} - -static void nullb_fill_pattern(struct nullb *nullb, struct page *page, - unsigned int len, unsigned int off) -{ - memset_page(page, off, 0xff, len); } blk_status_t null_handle_discard(struct nullb_device *dev, @@ -1234,34 +1229,39 @@ static blk_status_t null_handle_flush(struct nullb *nullb) return errno_to_blk_status(err); } -static int null_transfer(struct nullb *nullb, struct page *page, - unsigned int len, unsigned int off, bool is_write, sector_t sector, +static blk_status_t null_transfer(struct nullb *nullb, struct page *page, + unsigned int len, unsigned int off, bool is_write, loff_t pos, bool is_fua) { struct nullb_device *dev = nullb->dev; + blk_status_t err = BLK_STS_OK; unsigned int valid_len = len; - int err = 0; + void *p; + p = kmap_local_page(page) + off; if (!is_write) { - if (dev->zoned) + if (dev->zoned) { valid_len = null_zone_valid_read_len(nullb, - sector, len); + pos >> SECTOR_SHIFT, len); + if (valid_len && valid_len != len) + valid_len -= pos & (SECTOR_SIZE - 1); + } if (valid_len) { - err = copy_from_nullb(nullb, page, off, - sector, valid_len); + copy_from_nullb(nullb, p, pos, valid_len); off += valid_len; len -= valid_len; } if (len) - nullb_fill_pattern(nullb, page, len, off); + memset(p + valid_len, 0xff, len); flush_dcache_page(page); } else { flush_dcache_page(page); - err = copy_to_nullb(nullb, page, off, sector, len, is_fua); + err = copy_to_nullb(nullb, p, pos, len, is_fua); } + kunmap_local(p); return err; } @@ -1274,9 +1274,9 @@ static blk_status_t null_handle_data_transfer(struct nullb_cmd *cmd, { struct request *rq = blk_mq_rq_from_pdu(cmd); struct nullb *nullb = cmd->nq->dev->nullb; - int err = 0; + blk_status_t err = BLK_STS_OK; unsigned int len; - sector_t sector = blk_rq_pos(rq); + loff_t pos = blk_rq_pos(rq) << SECTOR_SHIFT; unsigned int max_bytes = nr_sectors << SECTOR_SHIFT; unsigned int transferred_bytes = 0; struct req_iterator iter; @@ -1288,18 +1288,18 @@ static blk_status_t null_handle_data_transfer(struct nullb_cmd *cmd, if (transferred_bytes + len > max_bytes) len = max_bytes - transferred_bytes; err = null_transfer(nullb, bvec.bv_page, len, bvec.bv_offset, - op_is_write(req_op(rq)), sector, + op_is_write(req_op(rq)), pos, rq->cmd_flags & REQ_FUA); if (err) break; - sector += len >> SECTOR_SHIFT; + pos += len; transferred_bytes += len; if (transferred_bytes >= max_bytes) break; } spin_unlock_irq(&nullb->lock); - return errno_to_blk_status(err); + return err; } static inline blk_status_t null_handle_throttled(struct nullb_cmd *cmd) @@ -1949,7 +1949,7 @@ static int null_add_dev(struct nullb_device *dev) .logical_block_size = dev->blocksize, .physical_block_size = dev->blocksize, .max_hw_sectors = dev->max_sectors, - .dma_alignment = dev->blocksize - 1, + .dma_alignment = 1, }; struct nullb *nullb; diff --git a/drivers/block/null_blk/null_blk.h b/drivers/block/null_blk/null_blk.h index 7bb6128dbaaf..6c4c4bbe7dad 100644 --- a/drivers/block/null_blk/null_blk.h +++ b/drivers/block/null_blk/null_blk.h @@ -143,7 +143,8 @@ int null_init_zoned_dev(struct nullb_device *dev, struct queue_limits *lim); int null_register_zoned_dev(struct nullb *nullb); void null_free_zoned_dev(struct nullb_device *dev); int null_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data); + unsigned int nr_zones, + struct blk_report_zones_args *args); blk_status_t null_process_zoned_cmd(struct nullb_cmd *cmd, enum req_op op, sector_t sector, sector_t nr_sectors); size_t null_zone_valid_read_len(struct nullb *nullb, diff --git a/drivers/block/null_blk/zoned.c b/drivers/block/null_blk/zoned.c index 4e5728f45989..0ada35dc0989 100644 --- a/drivers/block/null_blk/zoned.c +++ b/drivers/block/null_blk/zoned.c @@ -191,7 +191,7 @@ void null_free_zoned_dev(struct nullb_device *dev) } int null_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data) + unsigned int nr_zones, struct blk_report_zones_args *args) { struct nullb *nullb = disk->private_data; struct nullb_device *dev = nullb->dev; @@ -225,7 +225,7 @@ int null_report_zones(struct gendisk *disk, sector_t sector, blkz.capacity = zone->capacity; null_unlock_zone(dev, zone); - error = cb(&blkz, i, data); + error = disk_report_zone(disk, &blkz, i, args); if (error) return error; } @@ -242,7 +242,7 @@ size_t null_zone_valid_read_len(struct nullb *nullb, { struct nullb_device *dev = nullb->dev; struct nullb_zone *zone = &dev->zones[null_zone_no(dev, sector)]; - unsigned int nr_sectors = len >> SECTOR_SHIFT; + unsigned int nr_sectors = DIV_ROUND_UP(len, SECTOR_SIZE); /* Read must be below the write pointer position */ if (zone->type == BLK_ZONE_TYPE_CONVENTIONAL || diff --git a/drivers/block/ps3disk.c b/drivers/block/ps3disk.c index dc9e4a14b885..8892f218a814 100644 --- a/drivers/block/ps3disk.c +++ b/drivers/block/ps3disk.c @@ -85,10 +85,14 @@ static void ps3disk_scatter_gather(struct ps3_storage_device *dev, struct bio_vec bvec; rq_for_each_segment(bvec, req, iter) { + dev_dbg(&dev->sbd.core, "%s:%u: %u sectors from %llu\n", + __func__, __LINE__, bio_sectors(iter.bio), + iter.bio->bi_iter.bi_sector); if (gather) memcpy_from_bvec(dev->bounce_buf + offset, &bvec); else memcpy_to_bvec(&bvec, dev->bounce_buf + offset); + offset += bvec.bv_len; } } diff --git a/drivers/block/rnbd/rnbd-proto.h b/drivers/block/rnbd/rnbd-proto.h index f35be51d213c..77360c2a6069 100644 --- a/drivers/block/rnbd/rnbd-proto.h +++ b/drivers/block/rnbd/rnbd-proto.h @@ -24,7 +24,7 @@ #define RTRS_PORT 1234 /** - * enum rnbd_msg_types - RNBD message types + * enum rnbd_msg_type - RNBD message types * @RNBD_MSG_SESS_INFO: initial session info from client to server * @RNBD_MSG_SESS_INFO_RSP: initial session info from server to client * @RNBD_MSG_OPEN: open (map) device request @@ -47,10 +47,11 @@ enum rnbd_msg_type { */ struct rnbd_msg_hdr { __le16 type; + /* private: */ __le16 __padding; }; -/** +/* * We allow to map RO many times and RW only once. We allow to map yet another * time RW, if MIGRATION is provided (second RW export can be required for * example for VM migration) @@ -78,6 +79,7 @@ static const __maybe_unused struct { struct rnbd_msg_sess_info { struct rnbd_msg_hdr hdr; u8 ver; + /* private: */ u8 reserved[31]; }; @@ -89,6 +91,7 @@ struct rnbd_msg_sess_info { struct rnbd_msg_sess_info_rsp { struct rnbd_msg_hdr hdr; u8 ver; + /* private: */ u8 reserved[31]; }; @@ -97,13 +100,16 @@ struct rnbd_msg_sess_info_rsp { * @hdr: message header * @access_mode: the mode to open remote device, valid values see: * enum rnbd_access_mode - * @device_name: device path on remote side + * @dev_name: device path on remote side */ struct rnbd_msg_open { struct rnbd_msg_hdr hdr; u8 access_mode; + /* private: */ u8 resv1; + /* public: */ s8 dev_name[NAME_MAX]; + /* private: */ u8 reserved[3]; }; @@ -155,6 +161,7 @@ struct rnbd_msg_open_rsp { __le16 secure_discard; u8 obsolete_rotational; u8 cache_policy; + /* private: */ u8 reserved[10]; }; @@ -187,7 +194,7 @@ struct rnbd_msg_io { * @RNBD_OP_DISCARD: discard sectors * @RNBD_OP_SECURE_ERASE: securely erase sectors * @RNBD_OP_WRITE_ZEROES: write zeroes sectors - + * * @RNBD_F_SYNC: request is sync (sync write or read) * @RNBD_F_FUA: forced unit access */ diff --git a/drivers/block/rnull/configfs.rs b/drivers/block/rnull/configfs.rs index 8498e9bae6fd..6713a6d92391 100644 --- a/drivers/block/rnull/configfs.rs +++ b/drivers/block/rnull/configfs.rs @@ -1,12 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 use super::{NullBlkDevice, THIS_MODULE}; -use core::fmt::{Display, Write}; use kernel::{ block::mq::gen_disk::{GenDisk, GenDiskBuilder}, c_str, configfs::{self, AttributeOperations}, - configfs_attrs, new_mutex, + configfs_attrs, + fmt::{self, Write as _}, + new_mutex, page::PAGE_SIZE, prelude::*, str::{kstrtobool_bytes, CString}, @@ -99,8 +100,8 @@ impl TryFrom<u8> for IRQMode { } } -impl Display for IRQMode { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { +impl fmt::Display for IRQMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::None => f.write_str("0")?, Self::Soft => f.write_str("1")?, diff --git a/drivers/block/rnull/rnull.rs b/drivers/block/rnull/rnull.rs index 1ec694d7f1a6..a9d5e575a2c4 100644 --- a/drivers/block/rnull/rnull.rs +++ b/drivers/block/rnull/rnull.rs @@ -17,8 +17,7 @@ use kernel::{ error::Result, pr_info, prelude::*, - sync::Arc, - types::ARef, + sync::{aref::ARef, Arc}, }; use pin_init::PinInit; diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 0c74a41a6753..2c715df63f23 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -155,12 +155,13 @@ struct ublk_uring_cmd_pdu { */ #define UBLK_REFCOUNT_INIT (REFCOUNT_MAX / 2) +union ublk_io_buf { + __u64 addr; + struct ublk_auto_buf_reg auto_reg; +}; + struct ublk_io { - /* userspace buffer address from io cmd */ - union { - __u64 addr; - struct ublk_auto_buf_reg buf; - }; + union ublk_io_buf buf; unsigned int flags; int res; @@ -203,15 +204,12 @@ struct ublk_queue { bool fail_io; /* copy of dev->state == UBLK_S_DEV_FAIL_IO */ spinlock_t cancel_lock; struct ublk_device *dev; - struct ublk_io ios[]; + struct ublk_io ios[] __counted_by(q_depth); }; struct ublk_device { struct gendisk *ub_disk; - char *__queues; - - unsigned int queue_size; struct ublksrv_ctrl_dev_info dev_info; struct blk_mq_tag_set tag_set; @@ -239,6 +237,8 @@ struct ublk_device { bool canceling; pid_t ublksrv_tgid; struct delayed_work exit_work; + + struct ublk_queue *queues[]; }; /* header of ublk_params */ @@ -265,7 +265,7 @@ static inline bool ublk_dev_is_zoned(const struct ublk_device *ub) return ub->dev_info.flags & UBLK_F_ZONED; } -static inline bool ublk_queue_is_zoned(struct ublk_queue *ubq) +static inline bool ublk_queue_is_zoned(const struct ublk_queue *ubq) { return ubq->flags & UBLK_F_ZONED; } @@ -368,7 +368,7 @@ static void *ublk_alloc_report_buffer(struct ublk_device *ublk, } static int ublk_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data) + unsigned int nr_zones, struct blk_report_zones_args *args) { struct ublk_device *ub = disk->private_data; unsigned int zone_size_sectors = disk->queue->limits.chunk_sectors; @@ -431,7 +431,7 @@ free_req: if (!zone->len) break; - ret = cb(zone, i, data); + ret = disk_report_zone(disk, zone, i, args); if (ret) goto out; @@ -499,7 +499,7 @@ static blk_status_t ublk_setup_iod_zoned(struct ublk_queue *ubq, iod->op_flags = ublk_op | ublk_req_build_flags(req); iod->nr_sectors = blk_rq_sectors(req); iod->start_sector = blk_rq_pos(req); - iod->addr = io->addr; + iod->addr = io->buf.addr; return BLK_STS_OK; } @@ -781,7 +781,7 @@ static noinline void ublk_put_device(struct ublk_device *ub) static inline struct ublk_queue *ublk_get_queue(struct ublk_device *dev, int qid) { - return (struct ublk_queue *)&(dev->__queues[qid * dev->queue_size]); + return dev->queues[qid]; } static inline bool ublk_rq_has_data(const struct request *rq) @@ -914,73 +914,6 @@ static const struct block_device_operations ub_fops = { .report_zones = ublk_report_zones, }; -#define UBLK_MAX_PIN_PAGES 32 - -struct ublk_io_iter { - struct page *pages[UBLK_MAX_PIN_PAGES]; - struct bio *bio; - struct bvec_iter iter; -}; - -/* return how many pages are copied */ -static void ublk_copy_io_pages(struct ublk_io_iter *data, - size_t total, size_t pg_off, int dir) -{ - unsigned done = 0; - unsigned pg_idx = 0; - - while (done < total) { - struct bio_vec bv = bio_iter_iovec(data->bio, data->iter); - unsigned int bytes = min3(bv.bv_len, (unsigned)total - done, - (unsigned)(PAGE_SIZE - pg_off)); - void *bv_buf = bvec_kmap_local(&bv); - void *pg_buf = kmap_local_page(data->pages[pg_idx]); - - if (dir == ITER_DEST) - memcpy(pg_buf + pg_off, bv_buf, bytes); - else - memcpy(bv_buf, pg_buf + pg_off, bytes); - - kunmap_local(pg_buf); - kunmap_local(bv_buf); - - /* advance page array */ - pg_off += bytes; - if (pg_off == PAGE_SIZE) { - pg_idx += 1; - pg_off = 0; - } - - done += bytes; - - /* advance bio */ - bio_advance_iter_single(data->bio, &data->iter, bytes); - if (!data->iter.bi_size) { - data->bio = data->bio->bi_next; - if (data->bio == NULL) - break; - data->iter = data->bio->bi_iter; - } - } -} - -static bool ublk_advance_io_iter(const struct request *req, - struct ublk_io_iter *iter, unsigned int offset) -{ - struct bio *bio = req->bio; - - for_each_bio(bio) { - if (bio->bi_iter.bi_size > offset) { - iter->bio = bio; - iter->iter = bio->bi_iter; - bio_advance_iter(iter->bio, &iter->iter, offset); - return true; - } - offset -= bio->bi_iter.bi_size; - } - return false; -} - /* * Copy data between request pages and io_iter, and 'offset' * is the start point of linear offset of request. @@ -988,34 +921,35 @@ static bool ublk_advance_io_iter(const struct request *req, static size_t ublk_copy_user_pages(const struct request *req, unsigned offset, struct iov_iter *uiter, int dir) { - struct ublk_io_iter iter; + struct req_iterator iter; + struct bio_vec bv; size_t done = 0; - if (!ublk_advance_io_iter(req, &iter, offset)) - return 0; + rq_for_each_segment(bv, req, iter) { + void *bv_buf; + size_t copied; - while (iov_iter_count(uiter) && iter.bio) { - unsigned nr_pages; - ssize_t len; - size_t off; - int i; - - len = iov_iter_get_pages2(uiter, iter.pages, - iov_iter_count(uiter), - UBLK_MAX_PIN_PAGES, &off); - if (len <= 0) - return done; - - ublk_copy_io_pages(&iter, len, off, dir); - nr_pages = DIV_ROUND_UP(len + off, PAGE_SIZE); - for (i = 0; i < nr_pages; i++) { - if (dir == ITER_DEST) - set_page_dirty(iter.pages[i]); - put_page(iter.pages[i]); + if (offset >= bv.bv_len) { + offset -= bv.bv_len; + continue; } - done += len; - } + bv.bv_offset += offset; + bv.bv_len -= offset; + bv_buf = bvec_kmap_local(&bv); + if (dir == ITER_DEST) + copied = copy_to_iter(bv_buf, bv.bv_len, uiter); + else + copied = copy_from_iter(bv_buf, bv.bv_len, uiter); + + kunmap_local(bv_buf); + + done += copied; + if (copied < bv.bv_len) + break; + + offset = 0; + } return done; } @@ -1030,8 +964,9 @@ static inline bool ublk_need_unmap_req(const struct request *req) (req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_DRV_IN); } -static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req, - const struct ublk_io *io) +static unsigned int ublk_map_io(const struct ublk_queue *ubq, + const struct request *req, + const struct ublk_io *io) { const unsigned int rq_bytes = blk_rq_bytes(req); @@ -1047,13 +982,13 @@ static int ublk_map_io(const struct ublk_queue *ubq, const struct request *req, struct iov_iter iter; const int dir = ITER_DEST; - import_ubuf(dir, u64_to_user_ptr(io->addr), rq_bytes, &iter); + import_ubuf(dir, u64_to_user_ptr(io->buf.addr), rq_bytes, &iter); return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; } -static int ublk_unmap_io(bool need_map, +static unsigned int ublk_unmap_io(bool need_map, const struct request *req, const struct ublk_io *io) { @@ -1068,7 +1003,7 @@ static int ublk_unmap_io(bool need_map, WARN_ON_ONCE(io->res > rq_bytes); - import_ubuf(dir, u64_to_user_ptr(io->addr), io->res, &iter); + import_ubuf(dir, u64_to_user_ptr(io->buf.addr), io->res, &iter); return ublk_copy_user_pages(req, 0, &iter, dir); } return rq_bytes; @@ -1134,7 +1069,7 @@ static blk_status_t ublk_setup_iod(struct ublk_queue *ubq, struct request *req) iod->op_flags = ublk_op | ublk_req_build_flags(req); iod->nr_sectors = blk_rq_sectors(req); iod->start_sector = blk_rq_pos(req); - iod->addr = io->addr; + iod->addr = io->buf.addr; return BLK_STS_OK; } @@ -1233,45 +1168,65 @@ static inline void __ublk_abort_rq(struct ublk_queue *ubq, } static void -ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, struct ublk_io *io) +ublk_auto_buf_reg_fallback(const struct ublk_queue *ubq, unsigned tag) { - unsigned tag = io - ubq->ios; struct ublksrv_io_desc *iod = ublk_get_iod(ubq, tag); iod->op_flags |= UBLK_IO_F_NEED_REG_BUF; } -static bool ublk_auto_buf_reg(const struct ublk_queue *ubq, struct request *req, - struct ublk_io *io, unsigned int issue_flags) +enum auto_buf_reg_res { + AUTO_BUF_REG_FAIL, + AUTO_BUF_REG_FALLBACK, + AUTO_BUF_REG_OK, +}; + +static void ublk_prep_auto_buf_reg_io(const struct ublk_queue *ubq, + struct request *req, struct ublk_io *io, + struct io_uring_cmd *cmd, + enum auto_buf_reg_res res) +{ + if (res == AUTO_BUF_REG_OK) { + io->task_registered_buffers = 1; + io->buf_ctx_handle = io_uring_cmd_ctx_handle(cmd); + io->flags |= UBLK_IO_FLAG_AUTO_BUF_REG; + } + ublk_init_req_ref(ubq, io); + __ublk_prep_compl_io_cmd(io, req); +} + +static enum auto_buf_reg_res +__ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req, + struct ublk_io *io, struct io_uring_cmd *cmd, + unsigned int issue_flags) { int ret; - ret = io_buffer_register_bvec(io->cmd, req, ublk_io_release, - io->buf.index, issue_flags); + ret = io_buffer_register_bvec(cmd, req, ublk_io_release, + io->buf.auto_reg.index, issue_flags); if (ret) { - if (io->buf.flags & UBLK_AUTO_BUF_REG_FALLBACK) { - ublk_auto_buf_reg_fallback(ubq, io); - return true; + if (io->buf.auto_reg.flags & UBLK_AUTO_BUF_REG_FALLBACK) { + ublk_auto_buf_reg_fallback(ubq, req->tag); + return AUTO_BUF_REG_FALLBACK; } blk_mq_end_request(req, BLK_STS_IOERR); - return false; + return AUTO_BUF_REG_FAIL; } - io->task_registered_buffers = 1; - io->buf_ctx_handle = io_uring_cmd_ctx_handle(io->cmd); - io->flags |= UBLK_IO_FLAG_AUTO_BUF_REG; - return true; + return AUTO_BUF_REG_OK; } -static bool ublk_prep_auto_buf_reg(struct ublk_queue *ubq, - struct request *req, struct ublk_io *io, - unsigned int issue_flags) +static void ublk_do_auto_buf_reg(const struct ublk_queue *ubq, struct request *req, + struct ublk_io *io, struct io_uring_cmd *cmd, + unsigned int issue_flags) { - ublk_init_req_ref(ubq, io); - if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) - return ublk_auto_buf_reg(ubq, req, io, issue_flags); + enum auto_buf_reg_res res = __ublk_do_auto_buf_reg(ubq, req, io, cmd, + issue_flags); - return true; + if (res != AUTO_BUF_REG_FAIL) { + ublk_prep_auto_buf_reg_io(ubq, req, io, cmd, res); + io_uring_cmd_done(cmd, UBLK_IO_RES_OK, issue_flags); + } } static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, @@ -1302,10 +1257,9 @@ static bool ublk_start_io(const struct ublk_queue *ubq, struct request *req, return true; } -static void ublk_dispatch_req(struct ublk_queue *ubq, - struct request *req, - unsigned int issue_flags) +static void ublk_dispatch_req(struct ublk_queue *ubq, struct request *req) { + unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS; int tag = req->tag; struct ublk_io *io = &ubq->ios[tag]; @@ -1344,17 +1298,21 @@ static void ublk_dispatch_req(struct ublk_queue *ubq, if (!ublk_start_io(ubq, req, io)) return; - if (ublk_prep_auto_buf_reg(ubq, req, io, issue_flags)) + if (ublk_support_auto_buf_reg(ubq) && ublk_rq_has_data(req)) { + ublk_do_auto_buf_reg(ubq, req, io, io->cmd, issue_flags); + } else { + ublk_init_req_ref(ubq, io); ublk_complete_io_cmd(io, req, UBLK_IO_RES_OK, issue_flags); + } } -static void ublk_cmd_tw_cb(struct io_uring_cmd *cmd, - unsigned int issue_flags) +static void ublk_cmd_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req); struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); struct ublk_queue *ubq = pdu->ubq; - ublk_dispatch_req(ubq, pdu->req, issue_flags); + ublk_dispatch_req(ubq, pdu->req); } static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq) @@ -1366,9 +1324,9 @@ static void ublk_queue_cmd(struct ublk_queue *ubq, struct request *rq) io_uring_cmd_complete_in_task(cmd, ublk_cmd_tw_cb); } -static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd, - unsigned int issue_flags) +static void ublk_cmd_list_tw_cb(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req); struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); struct request *rq = pdu->req_list; struct request *next; @@ -1376,7 +1334,7 @@ static void ublk_cmd_list_tw_cb(struct io_uring_cmd *cmd, do { next = rq->rq_next; rq->rq_next = NULL; - ublk_dispatch_req(rq->mq_hctx->driver_data, rq, issue_flags); + ublk_dispatch_req(rq->mq_hctx->driver_data, rq); rq = next; } while (rq); } @@ -1537,7 +1495,7 @@ static void ublk_queue_reinit(struct ublk_device *ub, struct ublk_queue *ubq) */ io->flags &= UBLK_IO_FLAG_CANCELED; io->cmd = NULL; - io->addr = 0; + io->buf.addr = 0; /* * old task is PF_EXITING, put it now @@ -2098,13 +2056,16 @@ static inline int ublk_check_cmd_op(u32 cmd_op) static inline int ublk_set_auto_buf_reg(struct ublk_io *io, struct io_uring_cmd *cmd) { - io->buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + struct ublk_auto_buf_reg buf; - if (io->buf.reserved0 || io->buf.reserved1) + buf = ublk_sqe_addr_to_auto_buf_reg(READ_ONCE(cmd->sqe->addr)); + + if (buf.reserved0 || buf.reserved1) return -EINVAL; - if (io->buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) + if (buf.flags & ~UBLK_AUTO_BUF_REG_F_MASK) return -EINVAL; + io->buf.auto_reg = buf; return 0; } @@ -2126,7 +2087,7 @@ static int ublk_handle_auto_buf_reg(struct ublk_io *io, * this ublk request gets stuck. */ if (io->buf_ctx_handle == io_uring_cmd_ctx_handle(cmd)) - *buf_idx = io->buf.index; + *buf_idx = io->buf.auto_reg.index; } return ublk_set_auto_buf_reg(io, cmd); @@ -2154,7 +2115,7 @@ ublk_config_io_buf(const struct ublk_device *ub, struct ublk_io *io, if (ublk_dev_support_auto_buf_reg(ub)) return ublk_handle_auto_buf_reg(io, cmd, buf_idx); - io->addr = buf_addr; + io->buf.addr = buf_addr; return 0; } @@ -2272,39 +2233,41 @@ static int ublk_check_fetch_buf(const struct ublk_device *ub, __u64 buf_addr) return 0; } -static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, - struct ublk_io *io, __u64 buf_addr) +static int __ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, + struct ublk_io *io) { - int ret = 0; - - /* - * When handling FETCH command for setting up ublk uring queue, - * ub->mutex is the innermost lock, and we won't block for handling - * FETCH, so it is fine even for IO_URING_F_NONBLOCK. - */ - mutex_lock(&ub->mutex); /* UBLK_IO_FETCH_REQ is only allowed before dev is setup */ - if (ublk_dev_ready(ub)) { - ret = -EBUSY; - goto out; - } + if (ublk_dev_ready(ub)) + return -EBUSY; /* allow each command to be FETCHed at most once */ - if (io->flags & UBLK_IO_FLAG_ACTIVE) { - ret = -EINVAL; - goto out; - } + if (io->flags & UBLK_IO_FLAG_ACTIVE) + return -EINVAL; WARN_ON_ONCE(io->flags & UBLK_IO_FLAG_OWNED_BY_SRV); ublk_fill_io_cmd(io, cmd); - ret = ublk_config_io_buf(ub, io, cmd, buf_addr, NULL); - if (ret) - goto out; WRITE_ONCE(io->task, get_task_struct(current)); ublk_mark_io_ready(ub); -out: + + return 0; +} + +static int ublk_fetch(struct io_uring_cmd *cmd, struct ublk_device *ub, + struct ublk_io *io, __u64 buf_addr) +{ + int ret; + + /* + * When handling FETCH command for setting up ublk uring queue, + * ub->mutex is the innermost lock, and we won't block for handling + * FETCH, so it is fine even for IO_URING_F_NONBLOCK. + */ + mutex_lock(&ub->mutex); + ret = __ublk_fetch(cmd, ub, io); + if (!ret) + ret = ublk_config_io_buf(ub, io, cmd, buf_addr, NULL); mutex_unlock(&ub->mutex); return ret; } @@ -2351,7 +2314,7 @@ static bool ublk_get_data(const struct ublk_queue *ubq, struct ublk_io *io, */ io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA; /* update iod->addr because ublksrv may have passed a new io buffer */ - ublk_get_iod(ubq, req->tag)->addr = io->addr; + ublk_get_iod(ubq, req->tag)->addr = io->buf.addr; pr_devel("%s: update iod->addr: qid %d tag %d io_flags %x addr %llx\n", __func__, ubq->q_id, req->tag, io->flags, ublk_get_iod(ubq, req->tag)->addr); @@ -2367,7 +2330,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, u16 buf_idx = UBLK_INVALID_BUF_IDX; struct ublk_device *ub = cmd->file->private_data; struct ublk_queue *ubq; - struct ublk_io *io; + struct ublk_io *io = NULL; u32 cmd_op = cmd->cmd_op; u16 q_id = READ_ONCE(ub_src->q_id); u16 tag = READ_ONCE(ub_src->tag); @@ -2488,7 +2451,7 @@ static int ublk_ch_uring_cmd_local(struct io_uring_cmd *cmd, out: pr_devel("%s: complete: cmd op %d, tag %d ret %x io_flags %x\n", - __func__, cmd_op, tag, ret, io->flags); + __func__, cmd_op, tag, ret, io ? io->flags : 0); return ret; } @@ -2523,9 +2486,10 @@ fail_put: return NULL; } -static void ublk_ch_uring_cmd_cb(struct io_uring_cmd *cmd, - unsigned int issue_flags) +static void ublk_ch_uring_cmd_cb(struct io_tw_req tw_req, io_tw_token_t tw) { + unsigned int issue_flags = IO_URING_CMD_TASK_WORK_ISSUE_FLAGS; + struct io_uring_cmd *cmd = io_uring_cmd_from_tw(tw_req); int ret = ublk_ch_uring_cmd_local(cmd, issue_flags); if (ret != -EIOCBQUEUED) @@ -2575,9 +2539,6 @@ static struct request *ublk_check_and_get_req(struct kiocb *iocb, size_t buf_off; u16 tag, q_id; - if (!ub) - return ERR_PTR(-EACCES); - if (!user_backed_iter(iter)) return ERR_PTR(-EACCES); @@ -2603,9 +2564,6 @@ static struct request *ublk_check_and_get_req(struct kiocb *iocb, if (!req) return ERR_PTR(-EINVAL); - if (!req->mq_hctx || !req->mq_hctx->driver_data) - goto fail; - if (!ublk_check_ubuf_dir(req, dir)) goto fail; @@ -2662,9 +2620,13 @@ static const struct file_operations ublk_ch_fops = { static void ublk_deinit_queue(struct ublk_device *ub, int q_id) { - int size = ublk_queue_cmd_buf_size(ub); - struct ublk_queue *ubq = ublk_get_queue(ub, q_id); - int i; + struct ublk_queue *ubq = ub->queues[q_id]; + int size, i; + + if (!ubq) + return; + + size = ublk_queue_cmd_buf_size(ub); for (i = 0; i < ubq->q_depth; i++) { struct ublk_io *io = &ubq->ios[i]; @@ -2676,57 +2638,76 @@ static void ublk_deinit_queue(struct ublk_device *ub, int q_id) if (ubq->io_cmd_buf) free_pages((unsigned long)ubq->io_cmd_buf, get_order(size)); + + kvfree(ubq); + ub->queues[q_id] = NULL; +} + +static int ublk_get_queue_numa_node(struct ublk_device *ub, int q_id) +{ + unsigned int cpu; + + /* Find first CPU mapped to this queue */ + for_each_possible_cpu(cpu) { + if (ub->tag_set.map[HCTX_TYPE_DEFAULT].mq_map[cpu] == q_id) + return cpu_to_node(cpu); + } + + return NUMA_NO_NODE; } static int ublk_init_queue(struct ublk_device *ub, int q_id) { - struct ublk_queue *ubq = ublk_get_queue(ub, q_id); + int depth = ub->dev_info.queue_depth; gfp_t gfp_flags = GFP_KERNEL | __GFP_ZERO; - void *ptr; + struct ublk_queue *ubq; + struct page *page; + int numa_node; int size; + /* Determine NUMA node based on queue's CPU affinity */ + numa_node = ublk_get_queue_numa_node(ub, q_id); + + /* Allocate queue structure on local NUMA node */ + ubq = kvzalloc_node(struct_size(ubq, ios, depth), GFP_KERNEL, + numa_node); + if (!ubq) + return -ENOMEM; + spin_lock_init(&ubq->cancel_lock); ubq->flags = ub->dev_info.flags; ubq->q_id = q_id; - ubq->q_depth = ub->dev_info.queue_depth; + ubq->q_depth = depth; size = ublk_queue_cmd_buf_size(ub); - ptr = (void *) __get_free_pages(gfp_flags, get_order(size)); - if (!ptr) + /* Allocate I/O command buffer on local NUMA node */ + page = alloc_pages_node(numa_node, gfp_flags, get_order(size)); + if (!page) { + kvfree(ubq); return -ENOMEM; + } + ubq->io_cmd_buf = page_address(page); - ubq->io_cmd_buf = ptr; + ub->queues[q_id] = ubq; ubq->dev = ub; return 0; } static void ublk_deinit_queues(struct ublk_device *ub) { - int nr_queues = ub->dev_info.nr_hw_queues; int i; - if (!ub->__queues) - return; - - for (i = 0; i < nr_queues; i++) + for (i = 0; i < ub->dev_info.nr_hw_queues; i++) ublk_deinit_queue(ub, i); - kvfree(ub->__queues); } static int ublk_init_queues(struct ublk_device *ub) { - int nr_queues = ub->dev_info.nr_hw_queues; - int depth = ub->dev_info.queue_depth; - int ubq_size = sizeof(struct ublk_queue) + depth * sizeof(struct ublk_io); - int i, ret = -ENOMEM; + int i, ret; - ub->queue_size = ubq_size; - ub->__queues = kvcalloc(nr_queues, ubq_size, GFP_KERNEL); - if (!ub->__queues) - return ret; - - for (i = 0; i < nr_queues; i++) { - if (ublk_init_queue(ub, i)) + for (i = 0; i < ub->dev_info.nr_hw_queues; i++) { + ret = ublk_init_queue(ub, i); + if (ret) goto fail; } @@ -3128,7 +3109,7 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) goto out_unlock; ret = -ENOMEM; - ub = kzalloc(sizeof(*ub), GFP_KERNEL); + ub = kzalloc(struct_size(ub, queues, info.nr_hw_queues), GFP_KERNEL); if (!ub) goto out_unlock; mutex_init(&ub->mutex); @@ -3178,17 +3159,17 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) ub->dev_info.nr_hw_queues, nr_cpu_ids); ublk_align_max_io_size(ub); - ret = ublk_init_queues(ub); + ret = ublk_add_tag_set(ub); if (ret) goto out_free_dev_number; - ret = ublk_add_tag_set(ub); + ret = ublk_init_queues(ub); if (ret) - goto out_deinit_queues; + goto out_free_tag_set; ret = -EFAULT; if (copy_to_user(argp, &ub->dev_info, sizeof(info))) - goto out_free_tag_set; + goto out_deinit_queues; /* * Add the char dev so that ublksrv daemon can be setup. @@ -3197,10 +3178,10 @@ static int ublk_ctrl_add_dev(const struct ublksrv_ctrl_cmd *header) ret = ublk_add_chdev(ub); goto out_unlock; -out_free_tag_set: - blk_mq_free_tag_set(&ub->tag_set); out_deinit_queues: ublk_deinit_queues(ub); +out_free_tag_set: + blk_mq_free_tag_set(&ub->tag_set); out_free_dev_number: ublk_free_dev_number(ub); out_free_ub: diff --git a/drivers/block/virtio_blk.c b/drivers/block/virtio_blk.c index f061420dfb10..357434bdae99 100644 --- a/drivers/block/virtio_blk.c +++ b/drivers/block/virtio_blk.c @@ -584,7 +584,8 @@ out: static int virtblk_parse_zone(struct virtio_blk *vblk, struct virtio_blk_zone_descriptor *entry, - unsigned int idx, report_zones_cb cb, void *data) + unsigned int idx, + struct blk_report_zones_args *args) { struct blk_zone zone = { }; @@ -650,12 +651,12 @@ static int virtblk_parse_zone(struct virtio_blk *vblk, * The callback below checks the validity of the reported * entry data, no need to further validate it here. */ - return cb(&zone, idx, data); + return disk_report_zone(vblk->disk, &zone, idx, args); } static int virtblk_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, - void *data) + unsigned int nr_zones, + struct blk_report_zones_args *args) { struct virtio_blk *vblk = disk->private_data; struct virtio_blk_zone_report *report; @@ -693,7 +694,7 @@ static int virtblk_report_zones(struct gendisk *disk, sector_t sector, for (i = 0; i < nz && zone_idx < nr_zones; i++) { ret = virtblk_parse_zone(vblk, &report->zones[i], - zone_idx, cb, data); + zone_idx, args); if (ret) goto fail_report; @@ -1026,8 +1027,13 @@ static int init_vq(struct virtio_blk *vblk) out: kfree(vqs); kfree(vqs_info); - if (err) + if (err) { kfree(vblk->vqs); + /* + * Set to NULL to prevent freeing vqs again during freezing. + */ + vblk->vqs = NULL; + } return err; } @@ -1598,6 +1604,12 @@ static int virtblk_freeze_priv(struct virtio_device *vdev) vdev->config->del_vqs(vdev); kfree(vblk->vqs); + /* + * Set to NULL to prevent freeing vqs again after a failed vqs + * allocation during resume. Note that kfree() already handles NULL + * pointers safely. + */ + vblk->vqs = NULL; return 0; } diff --git a/drivers/block/zloop.c b/drivers/block/zloop.c index a423228e201b..3f50321aa4a7 100644 --- a/drivers/block/zloop.c +++ b/drivers/block/zloop.c @@ -32,6 +32,8 @@ enum { ZLOOP_OPT_NR_QUEUES = (1 << 6), ZLOOP_OPT_QUEUE_DEPTH = (1 << 7), ZLOOP_OPT_BUFFERED_IO = (1 << 8), + ZLOOP_OPT_ZONE_APPEND = (1 << 9), + ZLOOP_OPT_ORDERED_ZONE_APPEND = (1 << 10), }; static const match_table_t zloop_opt_tokens = { @@ -44,6 +46,8 @@ static const match_table_t zloop_opt_tokens = { { ZLOOP_OPT_NR_QUEUES, "nr_queues=%u" }, { ZLOOP_OPT_QUEUE_DEPTH, "queue_depth=%u" }, { ZLOOP_OPT_BUFFERED_IO, "buffered_io" }, + { ZLOOP_OPT_ZONE_APPEND, "zone_append=%u" }, + { ZLOOP_OPT_ORDERED_ZONE_APPEND, "ordered_zone_append" }, { ZLOOP_OPT_ERR, NULL } }; @@ -56,6 +60,8 @@ static const match_table_t zloop_opt_tokens = { #define ZLOOP_DEF_NR_QUEUES 1 #define ZLOOP_DEF_QUEUE_DEPTH 128 #define ZLOOP_DEF_BUFFERED_IO false +#define ZLOOP_DEF_ZONE_APPEND true +#define ZLOOP_DEF_ORDERED_ZONE_APPEND false /* Arbitrary limit on the zone size (16GB). */ #define ZLOOP_MAX_ZONE_SIZE_MB 16384 @@ -71,6 +77,8 @@ struct zloop_options { unsigned int nr_queues; unsigned int queue_depth; bool buffered_io; + bool zone_append; + bool ordered_zone_append; }; /* @@ -92,6 +100,7 @@ struct zloop_zone { unsigned long flags; struct mutex lock; + spinlock_t wp_lock; enum blk_zone_cond cond; sector_t start; sector_t wp; @@ -108,6 +117,8 @@ struct zloop_device { struct workqueue_struct *workqueue; bool buffered_io; + bool zone_append; + bool ordered_zone_append; const char *base_dir; struct file *data_dir; @@ -147,6 +158,7 @@ static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no) struct zloop_zone *zone = &zlo->zones[zone_no]; struct kstat stat; sector_t file_sectors; + unsigned long flags; int ret; lockdep_assert_held(&zone->lock); @@ -172,16 +184,18 @@ static int zloop_update_seq_zone(struct zloop_device *zlo, unsigned int zone_no) return -EINVAL; } + spin_lock_irqsave(&zone->wp_lock, flags); if (!file_sectors) { zone->cond = BLK_ZONE_COND_EMPTY; zone->wp = zone->start; } else if (file_sectors == zlo->zone_capacity) { zone->cond = BLK_ZONE_COND_FULL; - zone->wp = zone->start + zlo->zone_size; + zone->wp = ULLONG_MAX; } else { zone->cond = BLK_ZONE_COND_CLOSED; zone->wp = zone->start + file_sectors; } + spin_unlock_irqrestore(&zone->wp_lock, flags); return 0; } @@ -225,6 +239,7 @@ unlock: static int zloop_close_zone(struct zloop_device *zlo, unsigned int zone_no) { struct zloop_zone *zone = &zlo->zones[zone_no]; + unsigned long flags; int ret = 0; if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) @@ -243,10 +258,12 @@ static int zloop_close_zone(struct zloop_device *zlo, unsigned int zone_no) break; case BLK_ZONE_COND_IMP_OPEN: case BLK_ZONE_COND_EXP_OPEN: + spin_lock_irqsave(&zone->wp_lock, flags); if (zone->wp == zone->start) zone->cond = BLK_ZONE_COND_EMPTY; else zone->cond = BLK_ZONE_COND_CLOSED; + spin_unlock_irqrestore(&zone->wp_lock, flags); break; case BLK_ZONE_COND_EMPTY: case BLK_ZONE_COND_FULL: @@ -264,6 +281,7 @@ unlock: static int zloop_reset_zone(struct zloop_device *zlo, unsigned int zone_no) { struct zloop_zone *zone = &zlo->zones[zone_no]; + unsigned long flags; int ret = 0; if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) @@ -281,9 +299,11 @@ static int zloop_reset_zone(struct zloop_device *zlo, unsigned int zone_no) goto unlock; } + spin_lock_irqsave(&zone->wp_lock, flags); zone->cond = BLK_ZONE_COND_EMPTY; zone->wp = zone->start; clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); + spin_unlock_irqrestore(&zone->wp_lock, flags); unlock: mutex_unlock(&zone->lock); @@ -308,6 +328,7 @@ static int zloop_reset_all_zones(struct zloop_device *zlo) static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no) { struct zloop_zone *zone = &zlo->zones[zone_no]; + unsigned long flags; int ret = 0; if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) @@ -325,9 +346,11 @@ static int zloop_finish_zone(struct zloop_device *zlo, unsigned int zone_no) goto unlock; } + spin_lock_irqsave(&zone->wp_lock, flags); zone->cond = BLK_ZONE_COND_FULL; - zone->wp = zone->start + zlo->zone_size; + zone->wp = ULLONG_MAX; clear_bit(ZLOOP_ZONE_SEQ_ERROR, &zone->flags); + spin_unlock_irqrestore(&zone->wp_lock, flags); unlock: mutex_unlock(&zone->lock); @@ -369,6 +392,7 @@ static void zloop_rw(struct zloop_cmd *cmd) struct zloop_zone *zone; struct iov_iter iter; struct bio_vec tmp; + unsigned long flags; sector_t zone_end; int nr_bvec = 0; int ret; @@ -378,6 +402,11 @@ static void zloop_rw(struct zloop_cmd *cmd) cmd->nr_sectors = nr_sectors; cmd->ret = 0; + if (WARN_ON_ONCE(is_append && !zlo->zone_append)) { + ret = -EIO; + goto out; + } + /* We should never get an I/O beyond the device capacity. */ if (WARN_ON_ONCE(zone_no >= zlo->nr_zones)) { ret = -EIO; @@ -406,16 +435,31 @@ static void zloop_rw(struct zloop_cmd *cmd) if (!test_bit(ZLOOP_ZONE_CONV, &zone->flags) && is_write) { mutex_lock(&zone->lock); - if (is_append) { - sector = zone->wp; - cmd->sector = sector; - } + spin_lock_irqsave(&zone->wp_lock, flags); /* - * Write operations must be aligned to the write pointer and - * fully contained within the zone capacity. + * Zone append operations always go at the current write + * pointer, but regular write operations must already be + * aligned to the write pointer when submitted. */ - if (sector != zone->wp || zone->wp + nr_sectors > zone_end) { + if (is_append) { + /* + * If ordered zone append is in use, we already checked + * and set the target sector in zloop_queue_rq(). + */ + if (!zlo->ordered_zone_append) { + if (zone->cond == BLK_ZONE_COND_FULL || + zone->wp + nr_sectors > zone_end) { + spin_unlock_irqrestore(&zone->wp_lock, + flags); + ret = -EIO; + goto unlock; + } + sector = zone->wp; + } + cmd->sector = sector; + } else if (sector != zone->wp) { + spin_unlock_irqrestore(&zone->wp_lock, flags); pr_err("Zone %u: unaligned write: sect %llu, wp %llu\n", zone_no, sector, zone->wp); ret = -EIO; @@ -428,13 +472,19 @@ static void zloop_rw(struct zloop_cmd *cmd) zone->cond = BLK_ZONE_COND_IMP_OPEN; /* - * Advance the write pointer of sequential zones. If the write - * fails, the wp position will be corrected when the next I/O - * copmpletes. + * Advance the write pointer, unless ordered zone append is in + * use. If the write fails, the write pointer position will be + * corrected when the next I/O starts execution. */ - zone->wp += nr_sectors; - if (zone->wp == zone_end) - zone->cond = BLK_ZONE_COND_FULL; + if (!is_append || !zlo->ordered_zone_append) { + zone->wp += nr_sectors; + if (zone->wp == zone_end) { + zone->cond = BLK_ZONE_COND_FULL; + zone->wp = ULLONG_MAX; + } + } + + spin_unlock_irqrestore(&zone->wp_lock, flags); } rq_for_each_bvec(tmp, rq, rq_iter) @@ -498,6 +548,10 @@ static void zloop_handle_cmd(struct zloop_cmd *cmd) struct request *rq = blk_mq_rq_from_pdu(cmd); struct zloop_device *zlo = rq->q->queuedata; + /* We can block in this context, so ignore REQ_NOWAIT. */ + if (rq->cmd_flags & REQ_NOWAIT) + rq->cmd_flags &= ~REQ_NOWAIT; + switch (req_op(rq)) { case REQ_OP_READ: case REQ_OP_WRITE: @@ -608,6 +662,35 @@ static void zloop_complete_rq(struct request *rq) blk_mq_end_request(rq, sts); } +static bool zloop_set_zone_append_sector(struct request *rq) +{ + struct zloop_device *zlo = rq->q->queuedata; + unsigned int zone_no = rq_zone_no(rq); + struct zloop_zone *zone = &zlo->zones[zone_no]; + sector_t zone_end = zone->start + zlo->zone_capacity; + sector_t nr_sectors = blk_rq_sectors(rq); + unsigned long flags; + + spin_lock_irqsave(&zone->wp_lock, flags); + + if (zone->cond == BLK_ZONE_COND_FULL || + zone->wp + nr_sectors > zone_end) { + spin_unlock_irqrestore(&zone->wp_lock, flags); + return false; + } + + rq->__sector = zone->wp; + zone->wp += blk_rq_sectors(rq); + if (zone->wp >= zone_end) { + zone->cond = BLK_ZONE_COND_FULL; + zone->wp = ULLONG_MAX; + } + + spin_unlock_irqrestore(&zone->wp_lock, flags); + + return true; +} + static blk_status_t zloop_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) { @@ -618,6 +701,16 @@ static blk_status_t zloop_queue_rq(struct blk_mq_hw_ctx *hctx, if (zlo->state == Zlo_deleting) return BLK_STS_IOERR; + /* + * If we need to strongly order zone append operations, set the request + * sector to the zone write pointer location now instead of when the + * command work runs. + */ + if (zlo->ordered_zone_append && req_op(rq) == REQ_OP_ZONE_APPEND) { + if (!zloop_set_zone_append_sector(rq)) + return BLK_STS_IOERR; + } + blk_mq_start_request(rq); INIT_WORK(&cmd->work, zloop_cmd_workfn); @@ -647,11 +740,12 @@ static int zloop_open(struct gendisk *disk, blk_mode_t mode) } static int zloop_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data) + unsigned int nr_zones, struct blk_report_zones_args *args) { struct zloop_device *zlo = disk->private_data; struct blk_zone blkz = {}; unsigned int first, i; + unsigned long flags; int ret; first = disk_zone_no(disk, sector); @@ -675,7 +769,9 @@ static int zloop_report_zones(struct gendisk *disk, sector_t sector, blkz.start = zone->start; blkz.len = zlo->zone_size; + spin_lock_irqsave(&zone->wp_lock, flags); blkz.wp = zone->wp; + spin_unlock_irqrestore(&zone->wp_lock, flags); blkz.cond = zone->cond; if (test_bit(ZLOOP_ZONE_CONV, &zone->flags)) { blkz.type = BLK_ZONE_TYPE_CONVENTIONAL; @@ -687,7 +783,7 @@ static int zloop_report_zones(struct gendisk *disk, sector_t sector, mutex_unlock(&zone->lock); - ret = cb(&blkz, i, data); + ret = disk_report_zone(disk, &blkz, i, args); if (ret) return ret; } @@ -783,6 +879,7 @@ static int zloop_init_zone(struct zloop_device *zlo, struct zloop_options *opts, int ret; mutex_init(&zone->lock); + spin_lock_init(&zone->wp_lock); zone->start = (sector_t)zone_no << zlo->zone_shift; if (!restore) @@ -884,7 +981,6 @@ static int zloop_ctl_add(struct zloop_options *opts) { struct queue_limits lim = { .max_hw_sectors = SZ_1M >> SECTOR_SHIFT, - .max_hw_zone_append_sectors = SZ_1M >> SECTOR_SHIFT, .chunk_sectors = opts->zone_size, .features = BLK_FEAT_ZONED, }; @@ -936,6 +1032,9 @@ static int zloop_ctl_add(struct zloop_options *opts) zlo->nr_zones = nr_zones; zlo->nr_conv_zones = opts->nr_conv_zones; zlo->buffered_io = opts->buffered_io; + zlo->zone_append = opts->zone_append; + if (zlo->zone_append) + zlo->ordered_zone_append = opts->ordered_zone_append; zlo->workqueue = alloc_workqueue("zloop%d", WQ_UNBOUND | WQ_FREEZABLE, opts->nr_queues * opts->queue_depth, zlo->id); @@ -976,6 +1075,8 @@ static int zloop_ctl_add(struct zloop_options *opts) lim.physical_block_size = zlo->block_size; lim.logical_block_size = zlo->block_size; + if (zlo->zone_append) + lim.max_hw_zone_append_sectors = lim.max_hw_sectors; zlo->tag_set.ops = &zloop_mq_ops; zlo->tag_set.nr_hw_queues = opts->nr_queues; @@ -1016,10 +1117,14 @@ static int zloop_ctl_add(struct zloop_options *opts) zlo->state = Zlo_live; mutex_unlock(&zloop_ctl_mutex); - pr_info("Added device %d: %u zones of %llu MB, %u B block size\n", + pr_info("zloop: device %d, %u zones of %llu MiB, %u B block size\n", zlo->id, zlo->nr_zones, ((sector_t)zlo->zone_size << SECTOR_SHIFT) >> 20, zlo->block_size); + pr_info("zloop%d: using %s%s zone append\n", + zlo->id, + zlo->ordered_zone_append ? "ordered " : "", + zlo->zone_append ? "native" : "emulated"); return 0; @@ -1106,6 +1211,8 @@ static int zloop_parse_options(struct zloop_options *opts, const char *buf) opts->nr_queues = ZLOOP_DEF_NR_QUEUES; opts->queue_depth = ZLOOP_DEF_QUEUE_DEPTH; opts->buffered_io = ZLOOP_DEF_BUFFERED_IO; + opts->zone_append = ZLOOP_DEF_ZONE_APPEND; + opts->ordered_zone_append = ZLOOP_DEF_ORDERED_ZONE_APPEND; if (!buf) return 0; @@ -1215,6 +1322,21 @@ static int zloop_parse_options(struct zloop_options *opts, const char *buf) case ZLOOP_OPT_BUFFERED_IO: opts->buffered_io = true; break; + case ZLOOP_OPT_ZONE_APPEND: + if (match_uint(args, &token)) { + ret = -EINVAL; + goto out; + } + if (token != 0 && token != 1) { + pr_err("Invalid zone_append value\n"); + ret = -EINVAL; + goto out; + } + opts->zone_append = token; + break; + case ZLOOP_OPT_ORDERED_ZONE_APPEND: + opts->ordered_zone_append = true; + break; case ZLOOP_OPT_ERR: default: pr_warn("unknown parameter or missing value '%s'\n", p); diff --git a/drivers/bus/fsl-mc/mc-sys.c b/drivers/bus/fsl-mc/mc-sys.c index b22c59d57c8f..31037f41893e 100644 --- a/drivers/bus/fsl-mc/mc-sys.c +++ b/drivers/bus/fsl-mc/mc-sys.c @@ -248,7 +248,7 @@ int mc_send_command(struct fsl_mc_io *mc_io, struct fsl_mc_command *cmd) enum mc_cmd_status status; unsigned long irq_flags = 0; - if (in_irq() && !(mc_io->flags & FSL_MC_IO_ATOMIC_CONTEXT_PORTAL)) + if (in_hardirq() && !(mc_io->flags & FSL_MC_IO_ATOMIC_CONTEXT_PORTAL)) return -EINVAL; if (mc_io->flags & FSL_MC_IO_ATOMIC_CONTEXT_PORTAL) diff --git a/drivers/char/hw_random/bcm2835-rng.c b/drivers/char/hw_random/bcm2835-rng.c index aa2b135e3ee2..6d6ac409efcf 100644 --- a/drivers/char/hw_random/bcm2835-rng.c +++ b/drivers/char/hw_random/bcm2835-rng.c @@ -138,12 +138,11 @@ static const struct of_device_id bcm2835_rng_of_match[] = { { .compatible = "brcm,bcm6368-rng"}, {}, }; +MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match); static int bcm2835_rng_probe(struct platform_device *pdev) { - const struct bcm2835_rng_of_data *of_data; struct device *dev = &pdev->dev; - const struct of_device_id *rng_id; struct bcm2835_rng_priv *priv; int err; @@ -171,12 +170,10 @@ static int bcm2835_rng_probe(struct platform_device *pdev) priv->rng.cleanup = bcm2835_rng_cleanup; if (dev_of_node(dev)) { - rng_id = of_match_node(bcm2835_rng_of_match, dev->of_node); - if (!rng_id) - return -EINVAL; + const struct bcm2835_rng_of_data *of_data; /* Check for rng init function, execute it */ - of_data = rng_id->data; + of_data = of_device_get_match_data(dev); if (of_data) priv->mask_interrupts = of_data->mask_interrupts; } @@ -191,8 +188,6 @@ static int bcm2835_rng_probe(struct platform_device *pdev) return err; } -MODULE_DEVICE_TABLE(of, bcm2835_rng_of_match); - static const struct platform_device_id bcm2835_rng_devtype[] = { { .name = "bcm2835-rng" }, { .name = "bcm63xx-rng" }, diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 018316f54621..96d7fe41b373 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -341,6 +341,9 @@ static ssize_t rng_current_store(struct device *dev, if (sysfs_streq(buf, "")) { err = enable_best_rng(); + } else if (sysfs_streq(buf, "none")) { + cur_rng_set_by_user = 1; + drop_current_rng(); } else { list_for_each_entry(rng, &rng_list, list) { if (sysfs_streq(rng->name, buf)) { @@ -392,7 +395,7 @@ static ssize_t rng_available_show(struct device *dev, strlcat(buf, rng->name, PAGE_SIZE); strlcat(buf, " ", PAGE_SIZE); } - strlcat(buf, "\n", PAGE_SIZE); + strlcat(buf, "none\n", PAGE_SIZE); mutex_unlock(&rng_mutex); return strlen(buf); @@ -542,10 +545,10 @@ int hwrng_register(struct hwrng *rng) init_completion(&rng->dying); /* Adjust quality field to always have a proper value */ - rng->quality = min_t(u16, min_t(u16, default_quality, 1024), rng->quality ?: 1024); + rng->quality = min3(default_quality, 1024, rng->quality ?: 1024); - if (!current_rng || - (!cur_rng_set_by_user && rng->quality > current_rng->quality)) { + if (!cur_rng_set_by_user && + (!current_rng || rng->quality > current_rng->quality)) { /* * Set new rng as current as the new rng source * provides better entropy quality and was not diff --git a/drivers/char/hw_random/s390-trng.c b/drivers/char/hw_random/s390-trng.c index d27e32e9bfee..3024d5e9fd61 100644 --- a/drivers/char/hw_random/s390-trng.c +++ b/drivers/char/hw_random/s390-trng.c @@ -9,8 +9,7 @@ * Author(s): Harald Freudenberger <freude@de.ibm.com> */ -#define KMSG_COMPONENT "trng" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "trng: " fmt #include <linux/hw_random.h> #include <linux/kernel.h> diff --git a/drivers/char/ipmi/ipmi_si_intf.c b/drivers/char/ipmi/ipmi_si_intf.c index 70e55f5ff85e..5459ffdde8dc 100644 --- a/drivers/char/ipmi/ipmi_si_intf.c +++ b/drivers/char/ipmi/ipmi_si_intf.c @@ -275,8 +275,7 @@ void debug_timestamp(struct smi_info *smi_info, char *msg) struct timespec64 t; ktime_get_ts64(&t); - dev_dbg(smi_info->io.dev, "**%s: %lld.%9.9ld\n", - msg, t.tv_sec, t.tv_nsec); + dev_dbg(smi_info->io.dev, "**%s: %ptSp\n", msg, &t); } #else #define debug_timestamp(smi_info, x) diff --git a/drivers/char/ipmi/ipmi_ssif.c b/drivers/char/ipmi/ipmi_ssif.c index 1b63f7d2fcda..ef1582a029f4 100644 --- a/drivers/char/ipmi/ipmi_ssif.c +++ b/drivers/char/ipmi/ipmi_ssif.c @@ -1083,10 +1083,8 @@ static int sender(void *send_info, struct ipmi_smi_msg *msg) struct timespec64 t; ktime_get_real_ts64(&t); - dev_dbg(&ssif_info->client->dev, - "**Enqueue %02x %02x: %lld.%6.6ld\n", - msg->data[0], msg->data[1], - (long long)t.tv_sec, (long)t.tv_nsec / NSEC_PER_USEC); + dev_dbg(&ssif_info->client->dev, "**Enqueue %02x %02x: %ptSp\n", + msg->data[0], msg->data[1], &t); } return IPMI_CC_NO_ERROR; } diff --git a/drivers/char/random.c b/drivers/char/random.c index b8b24b6ed3fe..bab03c7c4194 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -259,8 +259,8 @@ static void crng_reseed(struct work_struct *work) u8 key[CHACHA_KEY_SIZE]; /* Immediately schedule the next reseeding, so that it fires sooner rather than later. */ - if (likely(system_unbound_wq)) - queue_delayed_work(system_unbound_wq, &next_reseed, crng_reseed_interval()); + if (likely(system_dfl_wq)) + queue_delayed_work(system_dfl_wq, &next_reseed, crng_reseed_interval()); extract_entropy(key, sizeof(key)); @@ -427,7 +427,7 @@ static void _get_random_bytes(void *buf, size_t len) /* * This returns random bytes in arbitrary quantities. The quality of the - * random bytes is good as /dev/urandom. In order to ensure that the + * random bytes is as good as /dev/urandom. In order to ensure that the * randomness provided by this function is okay, the function * wait_for_random_bytes() should be called and return 0 at least once * at any point prior. @@ -491,7 +491,7 @@ out_zero_chacha: /* * Batched entropy returns random integers. The quality of the random - * number is good as /dev/urandom. In order to ensure that the randomness + * number is as good as /dev/urandom. In order to ensure that the randomness * provided by this function is okay, the function wait_for_random_bytes() * should be called and return 0 at least once at any point prior. */ @@ -636,7 +636,7 @@ enum { }; static struct { - struct blake2s_state hash; + struct blake2s_ctx hash; spinlock_t lock; unsigned int init_bits; } input_pool = { @@ -701,7 +701,7 @@ static void extract_entropy(void *buf, size_t len) /* next_key = HASHPRF(seed, RDSEED || 0) */ block.counter = 0; - blake2s(next_key, (u8 *)&block, seed, sizeof(next_key), sizeof(block), sizeof(seed)); + blake2s(seed, sizeof(seed), (const u8 *)&block, sizeof(block), next_key, sizeof(next_key)); blake2s_init_key(&input_pool.hash, BLAKE2S_HASH_SIZE, next_key, sizeof(next_key)); spin_unlock_irqrestore(&input_pool.lock, flags); @@ -711,7 +711,7 @@ static void extract_entropy(void *buf, size_t len) i = min_t(size_t, len, BLAKE2S_HASH_SIZE); /* output = HASHPRF(seed, RDSEED || ++counter) */ ++block.counter; - blake2s(buf, (u8 *)&block, seed, i, sizeof(block), sizeof(seed)); + blake2s(seed, sizeof(seed), (const u8 *)&block, sizeof(block), buf, i); len -= i; buf += i; } @@ -741,8 +741,8 @@ static void __cold _credit_init_bits(size_t bits) if (orig < POOL_READY_BITS && new >= POOL_READY_BITS) { crng_reseed(NULL); /* Sets crng_init to CRNG_READY under base_crng.lock. */ - if (static_key_initialized && system_unbound_wq) - queue_work(system_unbound_wq, &set_ready); + if (system_dfl_wq) + queue_work(system_dfl_wq, &set_ready); atomic_notifier_call_chain(&random_ready_notifier, 0, NULL); #ifdef CONFIG_VDSO_GETRANDOM WRITE_ONCE(vdso_k_rng_data->is_ready, true); @@ -794,7 +794,7 @@ static void __cold _credit_init_bits(size_t bits) * * add_bootloader_randomness() is called by bootloader drivers, such as EFI * and device tree, and credits its input depending on whether or not the - * command line option 'random.trust_bootloader'. + * command line option 'random.trust_bootloader' is set. * * add_vmfork_randomness() adds a unique (but not necessarily secret) ID * representing the current instance of a VM to the pool, without crediting, @@ -915,9 +915,8 @@ void __init random_init(void) add_latent_entropy(); /* - * If we were initialized by the cpu or bootloader before jump labels - * or workqueues are initialized, then we should enable the static - * branch here, where it's guaranteed that these have been initialized. + * If we were initialized by the cpu or bootloader before workqueues + * are initialized, then we should enable the static branch here. */ if (!static_branch_likely(&crng_is_ready) && crng_init >= CRNG_READY) crng_set_ready(NULL); @@ -1296,6 +1295,7 @@ static void __cold try_to_generate_entropy(void) struct entropy_timer_state *stack = PTR_ALIGN((void *)stack_bytes, SMP_CACHE_BYTES); unsigned int i, num_different = 0; unsigned long last = random_get_entropy(); + cpumask_var_t timer_cpus; int cpu = -1; for (i = 0; i < NUM_TRIAL_SAMPLES - 1; ++i) { @@ -1310,13 +1310,15 @@ static void __cold try_to_generate_entropy(void) atomic_set(&stack->samples, 0); timer_setup_on_stack(&stack->timer, entropy_timer, 0); + if (!alloc_cpumask_var(&timer_cpus, GFP_KERNEL)) + goto out; + while (!crng_ready() && !signal_pending(current)) { /* * Check !timer_pending() and then ensure that any previous callback has finished * executing by checking timer_delete_sync_try(), before queueing the next one. */ if (!timer_pending(&stack->timer) && timer_delete_sync_try(&stack->timer) >= 0) { - struct cpumask timer_cpus; unsigned int num_cpus; /* @@ -1326,19 +1328,19 @@ static void __cold try_to_generate_entropy(void) preempt_disable(); /* Only schedule callbacks on timer CPUs that are online. */ - cpumask_and(&timer_cpus, housekeeping_cpumask(HK_TYPE_TIMER), cpu_online_mask); - num_cpus = cpumask_weight(&timer_cpus); + cpumask_and(timer_cpus, housekeeping_cpumask(HK_TYPE_TIMER), cpu_online_mask); + num_cpus = cpumask_weight(timer_cpus); /* In very bizarre case of misconfiguration, fallback to all online. */ if (unlikely(num_cpus == 0)) { - timer_cpus = *cpu_online_mask; - num_cpus = cpumask_weight(&timer_cpus); + *timer_cpus = *cpu_online_mask; + num_cpus = cpumask_weight(timer_cpus); } /* Basic CPU round-robin, which avoids the current CPU. */ do { - cpu = cpumask_next(cpu, &timer_cpus); + cpu = cpumask_next(cpu, timer_cpus); if (cpu >= nr_cpu_ids) - cpu = cpumask_first(&timer_cpus); + cpu = cpumask_first(timer_cpus); } while (cpu == smp_processor_id() && num_cpus > 1); /* Expiring the timer at `jiffies` means it's the next tick. */ @@ -1354,6 +1356,8 @@ static void __cold try_to_generate_entropy(void) } mix_pool_bytes(&stack->entropy, sizeof(stack->entropy)); + free_cpumask_var(timer_cpus); +out: timer_delete_sync(&stack->timer); timer_destroy_on_stack(&stack->timer); } diff --git a/drivers/char/tpm/tpm2-cmd.c b/drivers/char/tpm/tpm2-cmd.c index 7d77f6fbc152..5532e53a2dd3 100644 --- a/drivers/char/tpm/tpm2-cmd.c +++ b/drivers/char/tpm/tpm2-cmd.c @@ -18,7 +18,7 @@ static bool disable_pcr_integrity; module_param(disable_pcr_integrity, bool, 0444); MODULE_PARM_DESC(disable_pcr_integrity, "Disable integrity protection of TPM2_PCR_Extend"); -static struct tpm2_hash tpm2_hash_map[] = { +struct tpm2_hash tpm2_hash_map[] = { {HASH_ALGO_SHA1, TPM_ALG_SHA1}, {HASH_ALGO_SHA256, TPM_ALG_SHA256}, {HASH_ALGO_SHA384, TPM_ALG_SHA384}, @@ -26,6 +26,18 @@ static struct tpm2_hash tpm2_hash_map[] = { {HASH_ALGO_SM3_256, TPM_ALG_SM3_256}, }; +int tpm2_find_hash_alg(unsigned int crypto_id) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(tpm2_hash_map); i++) + if (crypto_id == tpm2_hash_map[i].crypto_id) + return tpm2_hash_map[i].tpm_id; + + return -EINVAL; +} +EXPORT_SYMBOL_GPL(tpm2_find_hash_alg); + int tpm2_get_timeouts(struct tpm_chip *chip) { chip->timeout_a = msecs_to_jiffies(TPM2_TIMEOUT_A); diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index ffcd23668763..aa59e5b13351 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -782,4 +782,15 @@ config NXP_STM_TIMER Enables the support for NXP System Timer Module found in the s32g NXP platform series. +config RTK_SYSTIMER + bool "Realtek SYSTIMER support" + depends on ARM || ARM64 + depends on ARCH_REALTEK || COMPILE_TEST + select TIMER_OF + help + This option enables the driver that registers the global 1 MHz hardware + counter as a clock event device on Realtek SoCs. Make sure to enable + this option only when building for a Realtek platform or for compilation + testing. + endmenu diff --git a/drivers/clocksource/Makefile b/drivers/clocksource/Makefile index ec4452ee958f..b46376af6b49 100644 --- a/drivers/clocksource/Makefile +++ b/drivers/clocksource/Makefile @@ -95,3 +95,4 @@ obj-$(CONFIG_CLKSRC_LOONGSON1_PWM) += timer-loongson1-pwm.o obj-$(CONFIG_EP93XX_TIMER) += timer-ep93xx.o obj-$(CONFIG_RALINK_TIMER) += timer-ralink.o obj-$(CONFIG_NXP_STM_TIMER) += timer-nxp-stm.o +obj-$(CONFIG_RTK_SYSTIMER) += timer-realtek.o diff --git a/drivers/clocksource/arm_arch_timer_mmio.c b/drivers/clocksource/arm_arch_timer_mmio.c index ebe1987d651e..d10362692fdd 100644 --- a/drivers/clocksource/arm_arch_timer_mmio.c +++ b/drivers/clocksource/arm_arch_timer_mmio.c @@ -426,6 +426,7 @@ static struct platform_driver arch_timer_mmio_drv = { .driver = { .name = "arch-timer-mmio", .of_match_table = arch_timer_mmio_of_table, + .suppress_bind_attrs = true, }, .probe = arch_timer_mmio_probe, }; @@ -434,6 +435,7 @@ builtin_platform_driver(arch_timer_mmio_drv); static struct platform_driver arch_timer_mmio_acpi_drv = { .driver = { .name = "gtdt-arm-mmio-timer", + .suppress_bind_attrs = true, }, .probe = arch_timer_mmio_probe, }; diff --git a/drivers/clocksource/sh_cmt.c b/drivers/clocksource/sh_cmt.c index 385eb94bbe7c..791b298c995b 100644 --- a/drivers/clocksource/sh_cmt.c +++ b/drivers/clocksource/sh_cmt.c @@ -355,14 +355,6 @@ static int sh_cmt_enable(struct sh_cmt_channel *ch) dev_pm_syscore_device(&ch->cmt->pdev->dev, true); - /* enable clock */ - ret = clk_enable(ch->cmt->clk); - if (ret) { - dev_err(&ch->cmt->pdev->dev, "ch%u: cannot enable clock\n", - ch->index); - goto err0; - } - /* make sure channel is disabled */ sh_cmt_start_stop_ch(ch, 0); @@ -384,19 +376,12 @@ static int sh_cmt_enable(struct sh_cmt_channel *ch) if (ret || sh_cmt_read_cmcnt(ch)) { dev_err(&ch->cmt->pdev->dev, "ch%u: cannot clear CMCNT\n", ch->index); - ret = -ETIMEDOUT; - goto err1; + return -ETIMEDOUT; } /* enable channel */ sh_cmt_start_stop_ch(ch, 1); return 0; - err1: - /* stop clock */ - clk_disable(ch->cmt->clk); - - err0: - return ret; } static void sh_cmt_disable(struct sh_cmt_channel *ch) @@ -407,9 +392,6 @@ static void sh_cmt_disable(struct sh_cmt_channel *ch) /* disable interrupts in CMT block */ sh_cmt_write_cmcsr(ch, 0); - /* stop clock */ - clk_disable(ch->cmt->clk); - dev_pm_syscore_device(&ch->cmt->pdev->dev, false); } @@ -583,8 +565,6 @@ static int sh_cmt_start_clocksource(struct sh_cmt_channel *ch) int ret = 0; unsigned long flags; - pm_runtime_get_sync(&ch->cmt->pdev->dev); - raw_spin_lock_irqsave(&ch->lock, flags); if (!(ch->flags & (FLAG_CLOCKEVENT | FLAG_CLOCKSOURCE))) @@ -619,8 +599,6 @@ static void sh_cmt_stop_clocksource(struct sh_cmt_channel *ch) sh_cmt_disable(ch); raw_spin_unlock_irqrestore(&ch->lock, flags); - - pm_runtime_put(&ch->cmt->pdev->dev); } static int sh_cmt_start_clockevent(struct sh_cmt_channel *ch) @@ -630,10 +608,8 @@ static int sh_cmt_start_clockevent(struct sh_cmt_channel *ch) raw_spin_lock_irqsave(&ch->lock, flags); - if (!(ch->flags & (FLAG_CLOCKEVENT | FLAG_CLOCKSOURCE))) { - pm_runtime_get_sync(&ch->cmt->pdev->dev); + if (!(ch->flags & (FLAG_CLOCKEVENT | FLAG_CLOCKSOURCE))) ret = sh_cmt_enable(ch); - } if (ret) goto out; @@ -656,10 +632,8 @@ static void sh_cmt_stop_clockevent(struct sh_cmt_channel *ch) ch->flags &= ~FLAG_CLOCKEVENT; - if (f && !(ch->flags & (FLAG_CLOCKEVENT | FLAG_CLOCKSOURCE))) { + if (f && !(ch->flags & (FLAG_CLOCKEVENT | FLAG_CLOCKSOURCE))) sh_cmt_disable(ch); - pm_runtime_put(&ch->cmt->pdev->dev); - } /* adjust the timeout to maximum if only clocksource left */ if (ch->flags & FLAG_CLOCKSOURCE) @@ -1134,8 +1108,6 @@ static int sh_cmt_setup(struct sh_cmt_device *cmt, struct platform_device *pdev) mask &= ~(1 << hwidx); } - clk_disable(cmt->clk); - platform_set_drvdata(pdev, cmt); return 0; @@ -1183,8 +1155,6 @@ static int sh_cmt_probe(struct platform_device *pdev) out: if (cmt->has_clockevent || cmt->has_clocksource) pm_runtime_irq_safe(&pdev->dev); - else - pm_runtime_idle(&pdev->dev); return 0; } diff --git a/drivers/clocksource/timer-nxp-pit.c b/drivers/clocksource/timer-nxp-pit.c index 2d0a3554b6bf..d1740f18f718 100644 --- a/drivers/clocksource/timer-nxp-pit.c +++ b/drivers/clocksource/timer-nxp-pit.c @@ -374,9 +374,10 @@ static struct platform_driver nxp_pit_driver = { .driver = { .name = "nxp-pit", .of_match_table = pit_timer_of_match, + .suppress_bind_attrs = true, }, .probe = pit_timer_probe, }; -module_platform_driver(nxp_pit_driver); +builtin_platform_driver(nxp_pit_driver); TIMER_OF_DECLARE(vf610, "fsl,vf610-pit", pit_timer_init); diff --git a/drivers/clocksource/timer-nxp-stm.c b/drivers/clocksource/timer-nxp-stm.c index bbc40623728f..1ab907233f48 100644 --- a/drivers/clocksource/timer-nxp-stm.c +++ b/drivers/clocksource/timer-nxp-stm.c @@ -177,15 +177,15 @@ static void nxp_stm_clocksource_resume(struct clocksource *cs) nxp_stm_clocksource_enable(cs); } -static void __init devm_clocksource_unregister(void *data) +static void devm_clocksource_unregister(void *data) { struct stm_timer *stm_timer = data; clocksource_unregister(&stm_timer->cs); } -static int __init nxp_stm_clocksource_init(struct device *dev, struct stm_timer *stm_timer, - const char *name, void __iomem *base, struct clk *clk) +static int nxp_stm_clocksource_init(struct device *dev, struct stm_timer *stm_timer, + const char *name, void __iomem *base, struct clk *clk) { int ret; @@ -208,10 +208,8 @@ static int __init nxp_stm_clocksource_init(struct device *dev, struct stm_timer return ret; ret = devm_add_action_or_reset(dev, devm_clocksource_unregister, stm_timer); - if (ret) { - clocksource_unregister(&stm_timer->cs); + if (ret) return ret; - } stm_sched_clock = stm_timer; @@ -298,9 +296,9 @@ static void nxp_stm_clockevent_resume(struct clock_event_device *ced) nxp_stm_module_get(stm_timer); } -static int __init nxp_stm_clockevent_per_cpu_init(struct device *dev, struct stm_timer *stm_timer, - const char *name, void __iomem *base, int irq, - struct clk *clk, int cpu) +static int nxp_stm_clockevent_per_cpu_init(struct device *dev, struct stm_timer *stm_timer, + const char *name, void __iomem *base, int irq, + struct clk *clk, int cpu) { stm_timer->base = base; stm_timer->rate = clk_get_rate(clk); @@ -388,7 +386,7 @@ static irqreturn_t nxp_stm_module_interrupt(int irq, void *dev_id) return IRQ_HANDLED; } -static int __init nxp_stm_timer_probe(struct platform_device *pdev) +static int nxp_stm_timer_probe(struct platform_device *pdev) { struct stm_timer *stm_timer; struct device *dev = &pdev->dev; @@ -484,14 +482,15 @@ static const struct of_device_id nxp_stm_of_match[] = { }; MODULE_DEVICE_TABLE(of, nxp_stm_of_match); -static struct platform_driver nxp_stm_probe = { +static struct platform_driver nxp_stm_driver = { .probe = nxp_stm_timer_probe, .driver = { .name = "nxp-stm", .of_match_table = nxp_stm_of_match, + .suppress_bind_attrs = true, }, }; -module_platform_driver(nxp_stm_probe); +builtin_platform_driver(nxp_stm_driver); MODULE_DESCRIPTION("NXP System Timer Module driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/clocksource/timer-ralink.c b/drivers/clocksource/timer-ralink.c index 6ecdb4228f76..68434d9ed910 100644 --- a/drivers/clocksource/timer-ralink.c +++ b/drivers/clocksource/timer-ralink.c @@ -130,14 +130,15 @@ static int __init ralink_systick_init(struct device_node *np) systick.dev.irq = irq_of_parse_and_map(np, 0); if (!systick.dev.irq) { pr_err("%pOFn: request_irq failed", np); - return -EINVAL; + ret = -EINVAL; + goto err_iounmap; } ret = clocksource_mmio_init(systick.membase + SYSTICK_COUNT, np->name, SYSTICK_FREQ, 301, 16, clocksource_mmio_readl_up); if (ret) - return ret; + goto err_free_irq; clockevents_register_device(&systick.dev); @@ -145,6 +146,12 @@ static int __init ralink_systick_init(struct device_node *np) np, systick.dev.mult, systick.dev.shift); return 0; + +err_free_irq: + irq_dispose_mapping(systick.dev.irq); +err_iounmap: + iounmap(systick.membase); + return ret; } TIMER_OF_DECLARE(systick, "ralink,cevt-systick", ralink_systick_init); diff --git a/drivers/clocksource/timer-rda.c b/drivers/clocksource/timer-rda.c index fd1199c189bf..0be8e05970e2 100644 --- a/drivers/clocksource/timer-rda.c +++ b/drivers/clocksource/timer-rda.c @@ -13,6 +13,7 @@ #include <linux/init.h> #include <linux/interrupt.h> +#include <linux/sched_clock.h> #include "timer-of.h" @@ -153,7 +154,7 @@ static struct timer_of rda_ostimer_of = { }, }; -static u64 rda_hwtimer_read(struct clocksource *cs) +static u64 rda_hwtimer_clocksource_read(void) { void __iomem *base = timer_of_base(&rda_ostimer_of); u32 lo, hi; @@ -167,6 +168,11 @@ static u64 rda_hwtimer_read(struct clocksource *cs) return ((u64)hi << 32) | lo; } +static u64 rda_hwtimer_read(struct clocksource *cs) +{ + return rda_hwtimer_clocksource_read(); +} + static struct clocksource rda_hwtimer_clocksource = { .name = "rda-timer", .rating = 400, @@ -185,6 +191,7 @@ static int __init rda_timer_init(struct device_node *np) return ret; clocksource_register_hz(&rda_hwtimer_clocksource, rate); + sched_clock_register(rda_hwtimer_clocksource_read, 64, rate); clockevents_config_and_register(&rda_ostimer_of.clkevt, rate, 0x2, UINT_MAX); diff --git a/drivers/clocksource/timer-realtek.c b/drivers/clocksource/timer-realtek.c new file mode 100644 index 000000000000..4f0439de9939 --- /dev/null +++ b/drivers/clocksource/timer-realtek.c @@ -0,0 +1,150 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025 Realtek Semiconductor Corp. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/irqflags.h> +#include <linux/interrupt.h> +#include "timer-of.h" + +#define ENBL 1 +#define DSBL 0 + +#define SYSTIMER_RATE 1000000 +#define SYSTIMER_MIN_DELTA 0x64 +#define SYSTIMER_MAX_DELTA ULONG_MAX + +/* SYSTIMER Register Offset (RTK Internal Use) */ +#define TS_LW_OFST 0x0 +#define TS_HW_OFST 0x4 +#define TS_CMP_VAL_LW_OFST 0x8 +#define TS_CMP_VAL_HW_OFST 0xC +#define TS_CMP_CTRL_OFST 0x10 +#define TS_CMP_STAT_OFST 0x14 + +/* SYSTIMER CMP CTRL REG Mask */ +#define TS_CMP_EN_MASK 0x1 +#define TS_WR_EN0_MASK 0x2 + +static void __iomem *systimer_base; + +static u64 rtk_ts64_read(void) +{ + u32 low, high; + u64 ts; + + /* Caution: Read LSB word (TS_LW_OFST) first then MSB (TS_HW_OFST) */ + low = readl(systimer_base + TS_LW_OFST); + high = readl(systimer_base + TS_HW_OFST); + ts = ((u64)high << 32) | low; + + return ts; +} + +static void rtk_cmp_value_write(u64 value) +{ + u32 high, low; + + low = value & 0xFFFFFFFF; + high = value >> 32; + + writel(high, systimer_base + TS_CMP_VAL_HW_OFST); + writel(low, systimer_base + TS_CMP_VAL_LW_OFST); +} + +static inline void rtk_cmp_en_write(bool cmp_en) +{ + u32 val; + + val = TS_WR_EN0_MASK; + if (cmp_en == ENBL) + val |= TS_CMP_EN_MASK; + + writel(val, systimer_base + TS_CMP_CTRL_OFST); +} + +static int rtk_syst_clkevt_next_event(unsigned long cycles, struct clock_event_device *clkevt) +{ + u64 cmp_val; + + rtk_cmp_en_write(DSBL); + cmp_val = rtk_ts64_read(); + + /* Set CMP value to current timestamp plus delta_us */ + rtk_cmp_value_write(cmp_val + cycles); + rtk_cmp_en_write(ENBL); + return 0; +} + +static irqreturn_t rtk_ts_match_intr_handler(int irq, void *dev_id) +{ + struct clock_event_device *clkevt = dev_id; + void __iomem *reg_base; + u32 val; + + /* Disable TS CMP Match */ + rtk_cmp_en_write(DSBL); + + /* Clear TS CMP INTR */ + reg_base = systimer_base + TS_CMP_STAT_OFST; + val = readl(reg_base) & TS_CMP_EN_MASK; + writel(val | TS_CMP_EN_MASK, reg_base); + clkevt->event_handler(clkevt); + + return IRQ_HANDLED; +} + +static int rtk_syst_shutdown(struct clock_event_device *clkevt) +{ + void __iomem *reg_base; + u64 cmp_val = 0; + + /* Disable TS CMP Match */ + rtk_cmp_en_write(DSBL); + /* Set compare value to 0 */ + rtk_cmp_value_write(cmp_val); + + /* Clear TS CMP INTR */ + reg_base = systimer_base + TS_CMP_STAT_OFST; + writel(TS_CMP_EN_MASK, reg_base); + return 0; +} + +static struct timer_of rtk_timer_to = { + .flags = TIMER_OF_IRQ | TIMER_OF_BASE, + + .clkevt = { + .name = "rtk-clkevt", + .rating = 300, + .cpumask = cpu_possible_mask, + .features = CLOCK_EVT_FEAT_DYNIRQ | + CLOCK_EVT_FEAT_ONESHOT, + .set_next_event = rtk_syst_clkevt_next_event, + .set_state_oneshot = rtk_syst_shutdown, + .set_state_shutdown = rtk_syst_shutdown, + }, + + .of_irq = { + .flags = IRQF_TIMER | IRQF_IRQPOLL, + .handler = rtk_ts_match_intr_handler, + }, +}; + +static int __init rtk_systimer_init(struct device_node *node) +{ + int ret; + + ret = timer_of_init(node, &rtk_timer_to); + if (ret) + return ret; + + systimer_base = timer_of_base(&rtk_timer_to); + clockevents_config_and_register(&rtk_timer_to.clkevt, SYSTIMER_RATE, + SYSTIMER_MIN_DELTA, SYSTIMER_MAX_DELTA); + + return 0; +} + +TIMER_OF_DECLARE(rtk_systimer, "realtek,rtd1625-systimer", rtk_systimer_init); diff --git a/drivers/clocksource/timer-sp804.c b/drivers/clocksource/timer-sp804.c index cd1916c05325..e82a95ea4724 100644 --- a/drivers/clocksource/timer-sp804.c +++ b/drivers/clocksource/timer-sp804.c @@ -21,6 +21,10 @@ #include <linux/of_irq.h> #include <linux/sched_clock.h> +#ifdef CONFIG_ARM +#include <linux/delay.h> +#endif + #include "timer-sp.h" /* Hisilicon 64-bit timer(a variant of ARM SP804) */ @@ -102,6 +106,23 @@ static u64 notrace sp804_read(void) return ~readl_relaxed(sched_clkevt->value); } +#ifdef CONFIG_ARM +static struct delay_timer delay; +static unsigned long sp804_read_delay_timer_read(void) +{ + return sp804_read(); +} + +static void sp804_register_delay_timer(int freq) +{ + delay.freq = freq; + delay.read_current_timer = sp804_read_delay_timer_read; + register_current_timer_delay(&delay); +} +#else +static inline void sp804_register_delay_timer(int freq) {} +#endif + static int __init sp804_clocksource_and_sched_clock_init(void __iomem *base, const char *name, struct clk *clk, @@ -114,6 +135,8 @@ static int __init sp804_clocksource_and_sched_clock_init(void __iomem *base, if (rate < 0) return -EINVAL; + sp804_register_delay_timer(rate); + clkevt = sp804_clkevt_get(base); writel(0, clkevt->ctrl); @@ -318,6 +341,7 @@ static int __init sp804_of_init(struct device_node *np, struct sp804_timer *time if (ret) goto err; } + initialized = true; return 0; diff --git a/drivers/clocksource/timer-sprd.c b/drivers/clocksource/timer-sprd.c index 430cb99d8d79..2c07dd2af760 100644 --- a/drivers/clocksource/timer-sprd.c +++ b/drivers/clocksource/timer-sprd.c @@ -30,6 +30,7 @@ #define TIMER_VALUE_SHDW_HI 0x1c #define TIMER_VALUE_LO_MASK GENMASK(31, 0) +#define TIMER_VALUE_HI_MASK GENMASK(31, 0) static void sprd_timer_enable(void __iomem *base, u32 flag) { @@ -162,15 +163,26 @@ static struct timer_of suspend_to = { static u64 sprd_suspend_timer_read(struct clocksource *cs) { - return ~(u64)readl_relaxed(timer_of_base(&suspend_to) + - TIMER_VALUE_SHDW_LO) & cs->mask; + u32 lo, hi; + + do { + hi = readl_relaxed(timer_of_base(&suspend_to) + + TIMER_VALUE_SHDW_HI); + lo = readl_relaxed(timer_of_base(&suspend_to) + + TIMER_VALUE_SHDW_LO); + } while (hi != readl_relaxed(timer_of_base(&suspend_to) + TIMER_VALUE_SHDW_HI)); + + return ~(((u64)hi << 32) | lo); } static int sprd_suspend_timer_enable(struct clocksource *cs) { - sprd_timer_update_counter(timer_of_base(&suspend_to), - TIMER_VALUE_LO_MASK); - sprd_timer_enable(timer_of_base(&suspend_to), TIMER_CTL_PERIOD_MODE); + writel_relaxed(TIMER_VALUE_LO_MASK, + timer_of_base(&suspend_to) + TIMER_LOAD_LO); + writel_relaxed(TIMER_VALUE_HI_MASK, + timer_of_base(&suspend_to) + TIMER_LOAD_HI); + sprd_timer_enable(timer_of_base(&suspend_to), + TIMER_CTL_PERIOD_MODE|TIMER_CTL_64BIT_WIDTH); return 0; } @@ -186,7 +198,7 @@ static struct clocksource suspend_clocksource = { .read = sprd_suspend_timer_read, .enable = sprd_suspend_timer_enable, .disable = sprd_suspend_timer_disable, - .mask = CLOCKSOURCE_MASK(32), + .mask = CLOCKSOURCE_MASK(64), .flags = CLOCK_SOURCE_IS_CONTINUOUS | CLOCK_SOURCE_SUSPEND_NONSTOP, }; diff --git a/drivers/clocksource/timer-stm32-lp.c b/drivers/clocksource/timer-stm32-lp.c index c2a699f5c1dd..3d804128c765 100644 --- a/drivers/clocksource/timer-stm32-lp.c +++ b/drivers/clocksource/timer-stm32-lp.c @@ -289,5 +289,4 @@ static struct platform_driver stm32_clkevent_lp_driver = { }; module_platform_driver(stm32_clkevent_lp_driver); -MODULE_ALIAS("platform:stm32-lptimer-timer"); MODULE_DESCRIPTION("STMicroelectronics STM32 clockevent low power driver"); diff --git a/drivers/counter/microchip-tcb-capture.c b/drivers/counter/microchip-tcb-capture.c index 1a299d1f350b..19d457ae4c3b 100644 --- a/drivers/counter/microchip-tcb-capture.c +++ b/drivers/counter/microchip-tcb-capture.c @@ -451,7 +451,7 @@ static void mchp_tc_irq_remove(void *ptr) static int mchp_tc_irq_enable(struct counter_device *const counter, int irq) { struct mchp_tc_data *const priv = counter_priv(counter); - int ret = devm_request_irq(counter->parent, irq, mchp_tc_isr, 0, + int ret = devm_request_irq(counter->parent, irq, mchp_tc_isr, IRQF_SHARED, dev_name(counter->parent), counter); if (ret < 0) diff --git a/drivers/cpufreq/acpi-cpufreq.c b/drivers/cpufreq/acpi-cpufreq.c index 083d8369a591..e73a66785d69 100644 --- a/drivers/cpufreq/acpi-cpufreq.c +++ b/drivers/cpufreq/acpi-cpufreq.c @@ -395,7 +395,7 @@ static unsigned int check_freqs(struct cpufreq_policy *policy, cur_freq = extract_freq(policy, get_cur_val(mask, data)); if (cur_freq == freq) return 1; - udelay(10); + usleep_range(10, 15); } return 0; } diff --git a/drivers/cpufreq/amd-pstate.c b/drivers/cpufreq/amd-pstate.c index b44f0f7a5ba1..c45bc98721d2 100644 --- a/drivers/cpufreq/amd-pstate.c +++ b/drivers/cpufreq/amd-pstate.c @@ -65,13 +65,13 @@ static const char * const amd_pstate_mode_string[] = { [AMD_PSTATE_PASSIVE] = "passive", [AMD_PSTATE_ACTIVE] = "active", [AMD_PSTATE_GUIDED] = "guided", - NULL, }; +static_assert(ARRAY_SIZE(amd_pstate_mode_string) == AMD_PSTATE_MAX); const char *amd_pstate_get_mode_string(enum amd_pstate_mode mode) { - if (mode < 0 || mode >= AMD_PSTATE_MAX) - return NULL; + if (mode < AMD_PSTATE_UNDEFINED || mode >= AMD_PSTATE_MAX) + mode = AMD_PSTATE_UNDEFINED; return amd_pstate_mode_string[mode]; } EXPORT_SYMBOL_GPL(amd_pstate_get_mode_string); @@ -110,6 +110,7 @@ enum energy_perf_value_index { EPP_INDEX_BALANCE_PERFORMANCE, EPP_INDEX_BALANCE_POWERSAVE, EPP_INDEX_POWERSAVE, + EPP_INDEX_MAX, }; static const char * const energy_perf_strings[] = { @@ -118,8 +119,8 @@ static const char * const energy_perf_strings[] = { [EPP_INDEX_BALANCE_PERFORMANCE] = "balance_performance", [EPP_INDEX_BALANCE_POWERSAVE] = "balance_power", [EPP_INDEX_POWERSAVE] = "power", - NULL }; +static_assert(ARRAY_SIZE(energy_perf_strings) == EPP_INDEX_MAX); static unsigned int epp_values[] = { [EPP_INDEX_DEFAULT] = 0, @@ -127,7 +128,8 @@ static unsigned int epp_values[] = { [EPP_INDEX_BALANCE_PERFORMANCE] = AMD_CPPC_EPP_BALANCE_PERFORMANCE, [EPP_INDEX_BALANCE_POWERSAVE] = AMD_CPPC_EPP_BALANCE_POWERSAVE, [EPP_INDEX_POWERSAVE] = AMD_CPPC_EPP_POWERSAVE, - }; +}; +static_assert(ARRAY_SIZE(epp_values) == EPP_INDEX_MAX); typedef int (*cppc_mode_transition_fn)(int); @@ -183,7 +185,7 @@ static inline int get_mode_idx_from_str(const char *str, size_t size) { int i; - for (i=0; i < AMD_PSTATE_MAX; i++) { + for (i = 0; i < AMD_PSTATE_MAX; i++) { if (!strncmp(str, amd_pstate_mode_string[i], size)) return i; } @@ -1137,16 +1139,15 @@ static ssize_t show_amd_pstate_hw_prefcore(struct cpufreq_policy *policy, static ssize_t show_energy_performance_available_preferences( struct cpufreq_policy *policy, char *buf) { - int i = 0; - int offset = 0; + int offset = 0, i; struct amd_cpudata *cpudata = policy->driver_data; if (cpudata->policy == CPUFREQ_POLICY_PERFORMANCE) return sysfs_emit_at(buf, offset, "%s\n", energy_perf_strings[EPP_INDEX_PERFORMANCE]); - while (energy_perf_strings[i] != NULL) - offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i++]); + for (i = 0; i < ARRAY_SIZE(energy_perf_strings); i++) + offset += sysfs_emit_at(buf, offset, "%s ", energy_perf_strings[i]); offset += sysfs_emit_at(buf, offset, "\n"); @@ -1157,15 +1158,10 @@ static ssize_t store_energy_performance_preference( struct cpufreq_policy *policy, const char *buf, size_t count) { struct amd_cpudata *cpudata = policy->driver_data; - char str_preference[21]; ssize_t ret; u8 epp; - ret = sscanf(buf, "%20s", str_preference); - if (ret != 1) - return -EINVAL; - - ret = match_string(energy_perf_strings, -1, str_preference); + ret = sysfs_match_string(energy_perf_strings, buf); if (ret < 0) return -EINVAL; @@ -1282,7 +1278,7 @@ static int amd_pstate_change_mode_without_dvr_change(int mode) if (cpu_feature_enabled(X86_FEATURE_CPPC) || cppc_state == AMD_PSTATE_ACTIVE) return 0; - for_each_present_cpu(cpu) { + for_each_online_cpu(cpu) { cppc_set_auto_sel(cpu, (cppc_state == AMD_PSTATE_PASSIVE) ? 0 : 1); } @@ -1353,9 +1349,8 @@ int amd_pstate_update_status(const char *buf, size_t size) return -EINVAL; mode_idx = get_mode_idx_from_str(buf, size); - - if (mode_idx < 0 || mode_idx >= AMD_PSTATE_MAX) - return -EINVAL; + if (mode_idx < 0) + return mode_idx; if (mode_state_machine[cppc_state][mode_idx]) { guard(mutex)(&amd_pstate_driver_lock); diff --git a/drivers/cpufreq/cppc_cpufreq.c b/drivers/cpufreq/cppc_cpufreq.c index e23d9abea135..9eac77c4f294 100644 --- a/drivers/cpufreq/cppc_cpufreq.c +++ b/drivers/cpufreq/cppc_cpufreq.c @@ -142,16 +142,15 @@ static void cppc_cpufreq_cpu_fie_init(struct cpufreq_policy *policy) init_irq_work(&cppc_fi->irq_work, cppc_irq_work); ret = cppc_get_perf_ctrs(cpu, &cppc_fi->prev_perf_fb_ctrs); - if (ret) { - pr_warn("%s: failed to read perf counters for cpu:%d: %d\n", - __func__, cpu, ret); - /* - * Don't abort if the CPU was offline while the driver - * was getting registered. - */ - if (cpu_online(cpu)) - return; + /* + * Don't abort as the CPU was offline while the driver was + * getting registered. + */ + if (ret && cpu_online(cpu)) { + pr_debug("%s: failed to read perf counters for cpu:%d: %d\n", + __func__, cpu, ret); + return; } } diff --git a/drivers/cpufreq/cpufreq-dt-platdev.c b/drivers/cpufreq/cpufreq-dt-platdev.c index cd1816a12bb9..dc11b62399ad 100644 --- a/drivers/cpufreq/cpufreq-dt-platdev.c +++ b/drivers/cpufreq/cpufreq-dt-platdev.c @@ -87,6 +87,7 @@ static const struct of_device_id allowlist[] __initconst = { { .compatible = "st-ericsson,u9540", }, { .compatible = "starfive,jh7110", }, + { .compatible = "starfive,jh7110s", }, { .compatible = "ti,omap2", }, { .compatible = "ti,omap4", }, diff --git a/drivers/cpufreq/cpufreq-nforce2.c b/drivers/cpufreq/cpufreq-nforce2.c index fedad1081973..fbbbe501cf2d 100644 --- a/drivers/cpufreq/cpufreq-nforce2.c +++ b/drivers/cpufreq/cpufreq-nforce2.c @@ -145,6 +145,8 @@ static unsigned int nforce2_fsb_read(int bootfsb) pci_read_config_dword(nforce2_sub5, NFORCE2_BOOTFSB, &fsb); fsb /= 1000000; + pci_dev_put(nforce2_sub5); + /* Check if PLL register is already set */ pci_read_config_byte(nforce2_dev, NFORCE2_PLLENABLE, (u8 *)&temp); @@ -426,6 +428,7 @@ static int __init nforce2_init(void) static void __exit nforce2_exit(void) { cpufreq_unregister_driver(&nforce2_driver); + pci_dev_put(nforce2_dev); } module_init(nforce2_init); diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 852e024facc3..4472bb1ec83c 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -1421,9 +1421,12 @@ static int cpufreq_policy_online(struct cpufreq_policy *policy, * If there is a problem with its frequency table, take it * offline and drop it. */ - ret = cpufreq_table_validate_and_sort(policy); - if (ret) - goto out_offline_policy; + if (policy->freq_table_sorted != CPUFREQ_TABLE_SORTED_ASCENDING && + policy->freq_table_sorted != CPUFREQ_TABLE_SORTED_DESCENDING) { + ret = cpufreq_table_validate_and_sort(policy); + if (ret) + goto out_offline_policy; + } /* related_cpus should at least include policy->cpus. */ cpumask_copy(policy->related_cpus, policy->cpus); @@ -2550,7 +2553,7 @@ void cpufreq_unregister_governor(struct cpufreq_governor *governor) for_each_inactive_policy(policy) { if (!strcmp(policy->last_governor, governor->name)) { policy->governor = NULL; - strcpy(policy->last_governor, "\0"); + policy->last_governor[0] = '\0'; } } read_unlock_irqrestore(&cpufreq_driver_lock, flags); diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c index 492a10f1bdbf..ec4abe374573 100644 --- a/drivers/cpufreq/intel_pstate.c +++ b/drivers/cpufreq/intel_pstate.c @@ -575,13 +575,18 @@ static void intel_pstate_hybrid_hwp_adjust(struct cpudata *cpu) int scaling = cpu->pstate.scaling; int freq; - pr_debug("CPU%d: perf_ctl_max_phys = %d\n", cpu->cpu, perf_ctl_max_phys); - pr_debug("CPU%d: perf_ctl_turbo = %d\n", cpu->cpu, perf_ctl_turbo); - pr_debug("CPU%d: perf_ctl_scaling = %d\n", cpu->cpu, perf_ctl_scaling); + pr_debug("CPU%d: PERF_CTL max_phys = %d\n", cpu->cpu, perf_ctl_max_phys); + pr_debug("CPU%d: PERF_CTL turbo = %d\n", cpu->cpu, perf_ctl_turbo); + pr_debug("CPU%d: PERF_CTL scaling = %d\n", cpu->cpu, perf_ctl_scaling); pr_debug("CPU%d: HWP_CAP guaranteed = %d\n", cpu->cpu, cpu->pstate.max_pstate); pr_debug("CPU%d: HWP_CAP highest = %d\n", cpu->cpu, cpu->pstate.turbo_pstate); pr_debug("CPU%d: HWP-to-frequency scaling factor: %d\n", cpu->cpu, scaling); + if (scaling == perf_ctl_scaling) + return; + + hwp_is_hybrid = true; + cpu->pstate.turbo_freq = rounddown(cpu->pstate.turbo_pstate * scaling, perf_ctl_scaling); cpu->pstate.max_freq = rounddown(cpu->pstate.max_pstate * scaling, @@ -909,6 +914,11 @@ static struct freq_attr *hwp_cpufreq_attrs[] = { [HWP_CPUFREQ_ATTR_COUNT] = NULL, }; +static u8 hybrid_get_cpu_type(unsigned int cpu) +{ + return cpu_data(cpu).topo.intel_type; +} + static bool no_cas __ro_after_init; static struct cpudata *hybrid_max_perf_cpu __read_mostly; @@ -925,11 +935,8 @@ static int hybrid_active_power(struct device *dev, unsigned long *power, unsigned long *freq) { /* - * Create "utilization bins" of 0-40%, 40%-60%, 60%-80%, and 80%-100% - * of the maximum capacity such that two CPUs of the same type will be - * regarded as equally attractive if the utilization of each of them - * falls into the same bin, which should prevent tasks from being - * migrated between them too often. + * Create four "states" corresponding to 40%, 60%, 80%, and 100% of the + * full capacity. * * For this purpose, return the "frequency" of 2 for the first * performance level and otherwise leave the value set by the caller. @@ -943,38 +950,40 @@ static int hybrid_active_power(struct device *dev, unsigned long *power, return 0; } +static bool hybrid_has_l3(unsigned int cpu) +{ + struct cpu_cacheinfo *cacheinfo = get_cpu_cacheinfo(cpu); + unsigned int i; + + if (!cacheinfo) + return false; + + for (i = 0; i < cacheinfo->num_leaves; i++) { + if (cacheinfo->info_list[i].level == 3) + return true; + } + + return false; +} + static int hybrid_get_cost(struct device *dev, unsigned long freq, unsigned long *cost) { - struct pstate_data *pstate = &all_cpu_data[dev->id]->pstate; - struct cpu_cacheinfo *cacheinfo = get_cpu_cacheinfo(dev->id); - + /* Facilitate load balancing between CPUs of the same type. */ + *cost = freq; /* - * The smaller the perf-to-frequency scaling factor, the larger the IPC - * ratio between the given CPU and the least capable CPU in the system. - * Regard that IPC ratio as the primary cost component and assume that - * the scaling factors for different CPU types will differ by at least - * 5% and they will not be above INTEL_PSTATE_CORE_SCALING. + * Adjust the cost depending on CPU type. * - * Add the freq value to the cost, so that the cost of running on CPUs - * of the same type in different "utilization bins" is different. - */ - *cost = div_u64(100ULL * INTEL_PSTATE_CORE_SCALING, pstate->scaling) + freq; - /* - * Increase the cost slightly for CPUs able to access L3 to avoid - * touching it in case some other CPUs of the same type can do the work - * without it. + * The idea is to start loading up LPE-cores before E-cores and start + * to populate E-cores when LPE-cores are utilized above 60% of the + * capacity. Similarly, P-cores start to be populated when E-cores are + * utilized above 60% of the capacity. */ - if (cacheinfo) { - unsigned int i; - - /* Check if L3 cache is there. */ - for (i = 0; i < cacheinfo->num_leaves; i++) { - if (cacheinfo->info_list[i].level == 3) { - *cost += 2; - break; - } - } + if (hybrid_get_cpu_type(dev->id) == INTEL_CPU_TYPE_ATOM) { + if (hybrid_has_l3(dev->id)) /* E-core */ + *cost += 1; + } else { /* P-core */ + *cost += 2; } return 0; @@ -1037,9 +1046,9 @@ static void hybrid_set_cpu_capacity(struct cpudata *cpu) topology_set_cpu_scale(cpu->cpu, arch_scale_cpu_capacity(cpu->cpu)); - pr_debug("CPU%d: perf = %u, max. perf = %u, base perf = %d\n", cpu->cpu, - cpu->capacity_perf, hybrid_max_perf_cpu->capacity_perf, - cpu->pstate.max_pstate_physical); + pr_debug("CPU%d: capacity perf = %u, base perf = %u, sys max perf = %u\n", + cpu->cpu, cpu->capacity_perf, cpu->pstate.max_pstate_physical, + hybrid_max_perf_cpu->capacity_perf); } static void hybrid_clear_cpu_capacity(unsigned int cpunum) @@ -1384,7 +1393,8 @@ static void set_power_ctl_ee_state(bool input) { u64 power_ctl; - mutex_lock(&intel_pstate_driver_lock); + guard(mutex)(&intel_pstate_driver_lock); + rdmsrq(MSR_IA32_POWER_CTL, power_ctl); if (input) { power_ctl &= ~BIT(MSR_IA32_POWER_CTL_BIT_EE); @@ -1394,7 +1404,6 @@ static void set_power_ctl_ee_state(bool input) power_ctl_ee_state = POWER_CTL_EE_DISABLE; } wrmsrq(MSR_IA32_POWER_CTL, power_ctl); - mutex_unlock(&intel_pstate_driver_lock); } static void intel_pstate_hwp_enable(struct cpudata *cpudata); @@ -1516,13 +1525,9 @@ static int intel_pstate_update_status(const char *buf, size_t size); static ssize_t show_status(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - ssize_t ret; - - mutex_lock(&intel_pstate_driver_lock); - ret = intel_pstate_show_status(buf); - mutex_unlock(&intel_pstate_driver_lock); + guard(mutex)(&intel_pstate_driver_lock); - return ret; + return intel_pstate_show_status(buf); } static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, @@ -1531,11 +1536,13 @@ static ssize_t store_status(struct kobject *a, struct kobj_attribute *b, char *p = memchr(buf, '\n', count); int ret; - mutex_lock(&intel_pstate_driver_lock); + guard(mutex)(&intel_pstate_driver_lock); + ret = intel_pstate_update_status(buf, p ? p - buf : count); - mutex_unlock(&intel_pstate_driver_lock); + if (ret < 0) + return ret; - return ret < 0 ? ret : count; + return count; } static ssize_t show_turbo_pct(struct kobject *kobj, @@ -1545,12 +1552,10 @@ static ssize_t show_turbo_pct(struct kobject *kobj, int total, no_turbo, turbo_pct; uint32_t turbo_fp; - mutex_lock(&intel_pstate_driver_lock); + guard(mutex)(&intel_pstate_driver_lock); - if (!intel_pstate_driver) { - mutex_unlock(&intel_pstate_driver_lock); + if (!intel_pstate_driver) return -EAGAIN; - } cpu = all_cpu_data[0]; @@ -1559,8 +1564,6 @@ static ssize_t show_turbo_pct(struct kobject *kobj, turbo_fp = div_fp(no_turbo, total); turbo_pct = 100 - fp_toint(mul_fp(turbo_fp, int_tofp(100))); - mutex_unlock(&intel_pstate_driver_lock); - return sprintf(buf, "%u\n", turbo_pct); } @@ -1570,38 +1573,26 @@ static ssize_t show_num_pstates(struct kobject *kobj, struct cpudata *cpu; int total; - mutex_lock(&intel_pstate_driver_lock); + guard(mutex)(&intel_pstate_driver_lock); - if (!intel_pstate_driver) { - mutex_unlock(&intel_pstate_driver_lock); + if (!intel_pstate_driver) return -EAGAIN; - } cpu = all_cpu_data[0]; total = cpu->pstate.turbo_pstate - cpu->pstate.min_pstate + 1; - mutex_unlock(&intel_pstate_driver_lock); - return sprintf(buf, "%u\n", total); } static ssize_t show_no_turbo(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - ssize_t ret; + guard(mutex)(&intel_pstate_driver_lock); - mutex_lock(&intel_pstate_driver_lock); - - if (!intel_pstate_driver) { - mutex_unlock(&intel_pstate_driver_lock); + if (!intel_pstate_driver) return -EAGAIN; - } - - ret = sprintf(buf, "%u\n", global.no_turbo); - - mutex_unlock(&intel_pstate_driver_lock); - return ret; + return sprintf(buf, "%u\n", global.no_turbo); } static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, @@ -1613,29 +1604,25 @@ static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, if (sscanf(buf, "%u", &input) != 1) return -EINVAL; - mutex_lock(&intel_pstate_driver_lock); + guard(mutex)(&intel_pstate_driver_lock); - if (!intel_pstate_driver) { - count = -EAGAIN; - goto unlock_driver; - } + if (!intel_pstate_driver) + return -EAGAIN; no_turbo = !!clamp_t(int, input, 0, 1); WRITE_ONCE(global.turbo_disabled, turbo_is_disabled()); if (global.turbo_disabled && !no_turbo) { pr_notice("Turbo disabled by BIOS or unavailable on processor\n"); - count = -EPERM; if (global.no_turbo) - goto unlock_driver; - else - no_turbo = 1; - } + return -EPERM; - if (no_turbo == global.no_turbo) { - goto unlock_driver; + no_turbo = 1; } + if (no_turbo == global.no_turbo) + return count; + WRITE_ONCE(global.no_turbo, no_turbo); mutex_lock(&intel_pstate_limits_lock); @@ -1654,9 +1641,6 @@ static ssize_t store_no_turbo(struct kobject *a, struct kobj_attribute *b, intel_pstate_update_limits_for_all(); arch_set_max_freq_ratio(no_turbo); -unlock_driver: - mutex_unlock(&intel_pstate_driver_lock); - return count; } @@ -1706,12 +1690,10 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b, if (ret != 1) return -EINVAL; - mutex_lock(&intel_pstate_driver_lock); + guard(mutex)(&intel_pstate_driver_lock); - if (!intel_pstate_driver) { - mutex_unlock(&intel_pstate_driver_lock); + if (!intel_pstate_driver) return -EAGAIN; - } mutex_lock(&intel_pstate_limits_lock); @@ -1724,8 +1706,6 @@ static ssize_t store_max_perf_pct(struct kobject *a, struct kobj_attribute *b, else update_qos_requests(FREQ_QOS_MAX); - mutex_unlock(&intel_pstate_driver_lock); - return count; } @@ -1739,12 +1719,10 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct kobj_attribute *b, if (ret != 1) return -EINVAL; - mutex_lock(&intel_pstate_driver_lock); + guard(mutex)(&intel_pstate_driver_lock); - if (!intel_pstate_driver) { - mutex_unlock(&intel_pstate_driver_lock); + if (!intel_pstate_driver) return -EAGAIN; - } mutex_lock(&intel_pstate_limits_lock); @@ -1758,8 +1736,6 @@ static ssize_t store_min_perf_pct(struct kobject *a, struct kobj_attribute *b, else update_qos_requests(FREQ_QOS_MIN); - mutex_unlock(&intel_pstate_driver_lock); - return count; } @@ -1780,10 +1756,10 @@ static ssize_t store_hwp_dynamic_boost(struct kobject *a, if (ret) return ret; - mutex_lock(&intel_pstate_driver_lock); + guard(mutex)(&intel_pstate_driver_lock); + hwp_boost = !!input; intel_pstate_update_policies(); - mutex_unlock(&intel_pstate_driver_lock); return count; } @@ -2072,6 +2048,18 @@ static void intel_pstate_hwp_enable(struct cpudata *cpudata) intel_pstate_update_epp_defaults(cpudata); } +static u64 get_perf_ctl_val(int pstate) +{ + u64 val; + + val = (u64)pstate << 8; + if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled) && + cpu_feature_enabled(X86_FEATURE_IDA)) + val |= (u64)1 << 32; + + return val; +} + static int atom_get_min_pstate(int not_used) { u64 value; @@ -2098,15 +2086,10 @@ static int atom_get_turbo_pstate(int not_used) static u64 atom_get_val(struct cpudata *cpudata, int pstate) { - u64 val; + u64 val = get_perf_ctl_val(pstate); int32_t vid_fp; u32 vid; - val = (u64)pstate << 8; - if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled) && - cpu_feature_enabled(X86_FEATURE_IDA)) - val |= (u64)1 << 32; - vid_fp = cpudata->vid.min + mul_fp( int_tofp(pstate - cpudata->pstate.min_pstate), cpudata->vid.ratio); @@ -2266,14 +2249,7 @@ static int core_get_turbo_pstate(int cpu) static u64 core_get_val(struct cpudata *cpudata, int pstate) { - u64 val; - - val = (u64)pstate << 8; - if (READ_ONCE(global.no_turbo) && !READ_ONCE(global.turbo_disabled) && - cpu_feature_enabled(X86_FEATURE_IDA)) - val |= (u64)1 << 32; - - return val; + return get_perf_ctl_val(pstate); } static int knl_get_aperf_mperf_shift(void) @@ -2297,18 +2273,14 @@ static int knl_get_turbo_pstate(int cpu) static int hwp_get_cpu_scaling(int cpu) { if (hybrid_scaling_factor) { - struct cpuinfo_x86 *c = &cpu_data(cpu); - u8 cpu_type = c->topo.intel_type; - /* * Return the hybrid scaling factor for P-cores and use the * default core scaling for E-cores. */ - if (cpu_type == INTEL_CPU_TYPE_CORE) + if (hybrid_get_cpu_type(cpu) == INTEL_CPU_TYPE_CORE) return hybrid_scaling_factor; - if (cpu_type == INTEL_CPU_TYPE_ATOM) - return core_get_scaling(); + return core_get_scaling(); } /* Use core scaling on non-hybrid systems. */ @@ -2343,11 +2315,10 @@ static void intel_pstate_set_min_pstate(struct cpudata *cpu) static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) { - int perf_ctl_max_phys = pstate_funcs.get_max_physical(cpu->cpu); int perf_ctl_scaling = pstate_funcs.get_scaling(); + cpu->pstate.max_pstate_physical = pstate_funcs.get_max_physical(cpu->cpu); cpu->pstate.min_pstate = pstate_funcs.get_min(cpu->cpu); - cpu->pstate.max_pstate_physical = perf_ctl_max_phys; cpu->pstate.perf_ctl_scaling = perf_ctl_scaling; if (hwp_active && !hwp_mode_bdw) { @@ -2355,10 +2326,7 @@ static void intel_pstate_get_cpu_pstates(struct cpudata *cpu) if (pstate_funcs.get_cpu_scaling) { cpu->pstate.scaling = pstate_funcs.get_cpu_scaling(cpu->cpu); - if (cpu->pstate.scaling != perf_ctl_scaling) { - intel_pstate_hybrid_hwp_adjust(cpu); - hwp_is_hybrid = true; - } + intel_pstate_hybrid_hwp_adjust(cpu); } else { cpu->pstate.scaling = perf_ctl_scaling; } @@ -2760,6 +2728,7 @@ static const struct x86_cpu_id intel_pstate_cpu_oob_ids[] __initconst = { X86_MATCH(INTEL_ATOM_CRESTMONT, core_funcs), X86_MATCH(INTEL_ATOM_CRESTMONT_X, core_funcs), X86_MATCH(INTEL_ATOM_DARKMONT_X, core_funcs), + X86_MATCH(INTEL_DIAMONDRAPIDS_X, core_funcs), {} }; #endif @@ -3912,9 +3881,9 @@ hwp_cpu_matched: } - mutex_lock(&intel_pstate_driver_lock); - rc = intel_pstate_register_driver(default_driver); - mutex_unlock(&intel_pstate_driver_lock); + scoped_guard(mutex, &intel_pstate_driver_lock) { + rc = intel_pstate_register_driver(default_driver); + } if (rc) { intel_pstate_sysfs_remove(); return rc; diff --git a/drivers/cpufreq/qcom-cpufreq-nvmem.c b/drivers/cpufreq/qcom-cpufreq-nvmem.c index 765a5bb81829..81e16b5a0245 100644 --- a/drivers/cpufreq/qcom-cpufreq-nvmem.c +++ b/drivers/cpufreq/qcom-cpufreq-nvmem.c @@ -256,13 +256,22 @@ len_error: return ret; } +static const struct of_device_id qcom_cpufreq_ipq806x_match_list[] __maybe_unused = { + { .compatible = "qcom,ipq8062", .data = (const void *)QCOM_ID_IPQ8062 }, + { .compatible = "qcom,ipq8064", .data = (const void *)QCOM_ID_IPQ8064 }, + { .compatible = "qcom,ipq8065", .data = (const void *)QCOM_ID_IPQ8065 }, + { .compatible = "qcom,ipq8066", .data = (const void *)QCOM_ID_IPQ8066 }, + { .compatible = "qcom,ipq8068", .data = (const void *)QCOM_ID_IPQ8068 }, + { .compatible = "qcom,ipq8069", .data = (const void *)QCOM_ID_IPQ8069 }, +}; + static int qcom_cpufreq_ipq8064_name_version(struct device *cpu_dev, struct nvmem_cell *speedbin_nvmem, char **pvs_name, struct qcom_cpufreq_drv *drv) { + int msm_id = -1, ret = 0; int speed = 0, pvs = 0; - int msm_id, ret = 0; u8 *speedbin; size_t len; @@ -279,8 +288,30 @@ static int qcom_cpufreq_ipq8064_name_version(struct device *cpu_dev, get_krait_bin_format_a(cpu_dev, &speed, &pvs, speedbin); ret = qcom_smem_get_soc_id(&msm_id); - if (ret) + if (ret == -ENODEV) { + const struct of_device_id *match; + struct device_node *root; + + root = of_find_node_by_path("/"); + if (!root) { + ret = -ENODEV; + goto exit; + } + + /* Fallback to compatible match with no SMEM initialized */ + match = of_match_node(qcom_cpufreq_ipq806x_match_list, root); + of_node_put(root); + if (!match) { + ret = -ENODEV; + goto exit; + } + + /* We found a matching device, get the msm_id from the data entry */ + msm_id = (int)(uintptr_t)match->data; + ret = 0; + } else if (ret) { goto exit; + } switch (msm_id) { case QCOM_ID_IPQ8062: diff --git a/drivers/cpufreq/s5pv210-cpufreq.c b/drivers/cpufreq/s5pv210-cpufreq.c index 4215621deb3f..ba8a1c96427a 100644 --- a/drivers/cpufreq/s5pv210-cpufreq.c +++ b/drivers/cpufreq/s5pv210-cpufreq.c @@ -518,7 +518,7 @@ static int s5pv210_cpu_init(struct cpufreq_policy *policy) if (policy->cpu != 0) { ret = -EINVAL; - goto out_dmc1; + goto out; } /* @@ -530,7 +530,7 @@ static int s5pv210_cpu_init(struct cpufreq_policy *policy) if ((mem_type != LPDDR) && (mem_type != LPDDR2)) { pr_err("CPUFreq doesn't support this memory type\n"); ret = -EINVAL; - goto out_dmc1; + goto out; } /* Find current refresh counter and frequency each DMC */ @@ -544,6 +544,8 @@ static int s5pv210_cpu_init(struct cpufreq_policy *policy) cpufreq_generic_init(policy, s5pv210_freq_table, 40000); return 0; +out: + clk_put(dmc1_clk); out_dmc1: clk_put(dmc0_clk); out_dmc0: diff --git a/drivers/cpufreq/tegra186-cpufreq.c b/drivers/cpufreq/tegra186-cpufreq.c index 136ab102f636..34ed943c5f34 100644 --- a/drivers/cpufreq/tegra186-cpufreq.c +++ b/drivers/cpufreq/tegra186-cpufreq.c @@ -8,6 +8,7 @@ #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> +#include <linux/units.h> #include <soc/tegra/bpmp.h> #include <soc/tegra/bpmp-abi.h> @@ -58,7 +59,7 @@ static const struct tegra186_cpufreq_cpu tegra186_cpus[] = { }; struct tegra186_cpufreq_cluster { - struct cpufreq_frequency_table *table; + struct cpufreq_frequency_table *bpmp_lut; u32 ref_clk_khz; u32 div; }; @@ -66,16 +67,119 @@ struct tegra186_cpufreq_cluster { struct tegra186_cpufreq_data { void __iomem *regs; const struct tegra186_cpufreq_cpu *cpus; + bool icc_dram_bw_scaling; struct tegra186_cpufreq_cluster clusters[]; }; +static int tegra_cpufreq_set_bw(struct cpufreq_policy *policy, unsigned long freq_khz) +{ + struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); + struct device *dev; + int ret; + + dev = get_cpu_device(policy->cpu); + if (!dev) + return -ENODEV; + + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_exact(dev, freq_khz * HZ_PER_KHZ, true); + if (IS_ERR(opp)) + return PTR_ERR(opp); + + ret = dev_pm_opp_set_opp(dev, opp); + if (ret) + data->icc_dram_bw_scaling = false; + + return ret; +} + +static int tegra_cpufreq_init_cpufreq_table(struct cpufreq_policy *policy, + struct cpufreq_frequency_table *bpmp_lut, + struct cpufreq_frequency_table **opp_table) +{ + struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); + struct cpufreq_frequency_table *freq_table = NULL; + struct cpufreq_frequency_table *pos; + struct device *cpu_dev; + unsigned long rate; + int ret, max_opps; + int j = 0; + + cpu_dev = get_cpu_device(policy->cpu); + if (!cpu_dev) { + pr_err("%s: failed to get cpu%d device\n", __func__, policy->cpu); + return -ENODEV; + } + + /* Initialize OPP table mentioned in operating-points-v2 property in DT */ + ret = dev_pm_opp_of_add_table_indexed(cpu_dev, 0); + if (ret) { + dev_err(cpu_dev, "Invalid or empty opp table in device tree\n"); + data->icc_dram_bw_scaling = false; + return ret; + } + + max_opps = dev_pm_opp_get_opp_count(cpu_dev); + if (max_opps <= 0) { + dev_err(cpu_dev, "Failed to add OPPs\n"); + return max_opps; + } + + /* Disable all opps and cross-validate against LUT later */ + for (rate = 0; ; rate++) { + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_ceil(cpu_dev, &rate); + if (IS_ERR(opp)) + break; + + dev_pm_opp_disable(cpu_dev, rate); + } + + freq_table = kcalloc((max_opps + 1), sizeof(*freq_table), GFP_KERNEL); + if (!freq_table) + return -ENOMEM; + + /* + * Cross check the frequencies from BPMP-FW LUT against the OPP's present in DT. + * Enable only those DT OPP's which are present in LUT also. + */ + cpufreq_for_each_valid_entry(pos, bpmp_lut) { + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_exact(cpu_dev, pos->frequency * HZ_PER_KHZ, false); + if (IS_ERR(opp)) + continue; + + ret = dev_pm_opp_enable(cpu_dev, pos->frequency * HZ_PER_KHZ); + if (ret < 0) + return ret; + + freq_table[j].driver_data = pos->driver_data; + freq_table[j].frequency = pos->frequency; + j++; + } + + freq_table[j].driver_data = pos->driver_data; + freq_table[j].frequency = CPUFREQ_TABLE_END; + + *opp_table = &freq_table[0]; + + dev_pm_opp_set_sharing_cpus(cpu_dev, policy->cpus); + + /* Prime interconnect data */ + tegra_cpufreq_set_bw(policy, freq_table[j - 1].frequency); + + return ret; +} + static int tegra186_cpufreq_init(struct cpufreq_policy *policy) { struct tegra186_cpufreq_data *data = cpufreq_get_driver_data(); unsigned int cluster = data->cpus[policy->cpu].bpmp_cluster_id; + struct cpufreq_frequency_table *freq_table; + struct cpufreq_frequency_table *bpmp_lut; u32 cpu; + int ret; - policy->freq_table = data->clusters[cluster].table; policy->cpuinfo.transition_latency = 300 * 1000; policy->driver_data = NULL; @@ -85,6 +189,20 @@ static int tegra186_cpufreq_init(struct cpufreq_policy *policy) cpumask_set_cpu(cpu, policy->cpus); } + bpmp_lut = data->clusters[cluster].bpmp_lut; + + if (data->icc_dram_bw_scaling) { + ret = tegra_cpufreq_init_cpufreq_table(policy, bpmp_lut, &freq_table); + if (!ret) { + policy->freq_table = freq_table; + return 0; + } + } + + data->icc_dram_bw_scaling = false; + policy->freq_table = bpmp_lut; + pr_info("OPP tables missing from DT, EMC frequency scaling disabled\n"); + return 0; } @@ -102,6 +220,10 @@ static int tegra186_cpufreq_set_target(struct cpufreq_policy *policy, writel(edvd_val, data->regs + edvd_offset); } + if (data->icc_dram_bw_scaling) + tegra_cpufreq_set_bw(policy, tbl->frequency); + + return 0; } @@ -134,7 +256,7 @@ static struct cpufreq_driver tegra186_cpufreq_driver = { .init = tegra186_cpufreq_init, }; -static struct cpufreq_frequency_table *init_vhint_table( +static struct cpufreq_frequency_table *tegra_cpufreq_bpmp_read_lut( struct platform_device *pdev, struct tegra_bpmp *bpmp, struct tegra186_cpufreq_cluster *cluster, unsigned int cluster_id, int *num_rates) @@ -229,6 +351,7 @@ static int tegra186_cpufreq_probe(struct platform_device *pdev) { struct tegra186_cpufreq_data *data; struct tegra_bpmp *bpmp; + struct device *cpu_dev; unsigned int i = 0, err, edvd_offset; int num_rates = 0; u32 edvd_val, cpu; @@ -254,9 +377,9 @@ static int tegra186_cpufreq_probe(struct platform_device *pdev) for (i = 0; i < TEGRA186_NUM_CLUSTERS; i++) { struct tegra186_cpufreq_cluster *cluster = &data->clusters[i]; - cluster->table = init_vhint_table(pdev, bpmp, cluster, i, &num_rates); - if (IS_ERR(cluster->table)) { - err = PTR_ERR(cluster->table); + cluster->bpmp_lut = tegra_cpufreq_bpmp_read_lut(pdev, bpmp, cluster, i, &num_rates); + if (IS_ERR(cluster->bpmp_lut)) { + err = PTR_ERR(cluster->bpmp_lut); goto put_bpmp; } else if (!num_rates) { err = -EINVAL; @@ -265,7 +388,7 @@ static int tegra186_cpufreq_probe(struct platform_device *pdev) for (cpu = 0; cpu < ARRAY_SIZE(tegra186_cpus); cpu++) { if (data->cpus[cpu].bpmp_cluster_id == i) { - edvd_val = cluster->table[num_rates - 1].driver_data; + edvd_val = cluster->bpmp_lut[num_rates - 1].driver_data; edvd_offset = data->cpus[cpu].edvd_offset; writel(edvd_val, data->regs + edvd_offset); } @@ -274,6 +397,19 @@ static int tegra186_cpufreq_probe(struct platform_device *pdev) tegra186_cpufreq_driver.driver_data = data; + /* Check for optional OPPv2 and interconnect paths on CPU0 to enable ICC scaling */ + cpu_dev = get_cpu_device(0); + if (!cpu_dev) { + err = -EPROBE_DEFER; + goto put_bpmp; + } + + if (dev_pm_opp_of_get_opp_desc_node(cpu_dev)) { + err = dev_pm_opp_of_find_icc_paths(cpu_dev, NULL); + if (!err) + data->icc_dram_bw_scaling = true; + } + err = cpufreq_register_driver(&tegra186_cpufreq_driver); put_bpmp: diff --git a/drivers/cpufreq/tegra194-cpufreq.c b/drivers/cpufreq/tegra194-cpufreq.c index 9b4f516f313e..695599e1001f 100644 --- a/drivers/cpufreq/tegra194-cpufreq.c +++ b/drivers/cpufreq/tegra194-cpufreq.c @@ -750,7 +750,8 @@ static int tegra194_cpufreq_probe(struct platform_device *pdev) if (IS_ERR(bpmp)) return PTR_ERR(bpmp); - read_counters_wq = alloc_workqueue("read_counters_wq", __WQ_LEGACY, 1); + read_counters_wq = alloc_workqueue("read_counters_wq", + __WQ_LEGACY | WQ_PERCPU, 1); if (!read_counters_wq) { dev_err(&pdev->dev, "fail to create_workqueue\n"); err = -EINVAL; diff --git a/drivers/cpuidle/cpuidle.c b/drivers/cpuidle/cpuidle.c index 56132e843c99..c7876e9e024f 100644 --- a/drivers/cpuidle/cpuidle.c +++ b/drivers/cpuidle/cpuidle.c @@ -184,20 +184,22 @@ static noinstr void enter_s2idle_proper(struct cpuidle_driver *drv, * cpuidle_enter_s2idle - Enter an idle state suitable for suspend-to-idle. * @drv: cpuidle driver for the given CPU. * @dev: cpuidle device for the given CPU. + * @latency_limit_ns: Idle state exit latency limit * * If there are states with the ->enter_s2idle callback, find the deepest of * them and enter it with frozen tick. */ -int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev) +int cpuidle_enter_s2idle(struct cpuidle_driver *drv, struct cpuidle_device *dev, + u64 latency_limit_ns) { int index; /* - * Find the deepest state with ->enter_s2idle present, which guarantees - * that interrupts won't be enabled when it exits and allows the tick to - * be frozen safely. + * Find the deepest state with ->enter_s2idle present that meets the + * specified latency limit, which guarantees that interrupts won't be + * enabled when it exits and allows the tick to be frozen safely. */ - index = find_deepest_state(drv, dev, U64_MAX, 0, true); + index = find_deepest_state(drv, dev, latency_limit_ns, 0, true); if (index > 0) { enter_s2idle_proper(drv, dev, index); local_irq_enable(); diff --git a/drivers/cpuidle/driver.c b/drivers/cpuidle/driver.c index 9bbfa594c442..370664c47e65 100644 --- a/drivers/cpuidle/driver.c +++ b/drivers/cpuidle/driver.c @@ -8,6 +8,8 @@ * This code is licenced under the GPL. */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + #include <linux/mutex.h> #include <linux/module.h> #include <linux/sched.h> @@ -193,6 +195,14 @@ static void __cpuidle_driver_init(struct cpuidle_driver *drv) s->exit_latency_ns = 0; else s->exit_latency = div_u64(s->exit_latency_ns, NSEC_PER_USEC); + + /* + * Warn if the exit latency of a CPU idle state exceeds its + * target residency which is assumed to never happen in cpuidle + * in multiple places. + */ + if (s->exit_latency_ns > s->target_residency_ns) + pr_warn("Idle state %d target residency too low\n", i); } } diff --git a/drivers/cpuidle/governor.c b/drivers/cpuidle/governor.c index 0d0f9751ff8f..5d0e7f78c6c5 100644 --- a/drivers/cpuidle/governor.c +++ b/drivers/cpuidle/governor.c @@ -111,6 +111,10 @@ s64 cpuidle_governor_latency_req(unsigned int cpu) struct device *device = get_cpu_device(cpu); int device_req = dev_pm_qos_raw_resume_latency(device); int global_req = cpu_latency_qos_limit(); + int global_wake_req = cpu_wakeup_latency_qos_limit(); + + if (global_req > global_wake_req) + global_req = global_wake_req; if (device_req > global_req) device_req = global_req; diff --git a/drivers/cpuidle/governors/menu.c b/drivers/cpuidle/governors/menu.c index 23239b0c04f9..64d6f7a1c776 100644 --- a/drivers/cpuidle/governors/menu.c +++ b/drivers/cpuidle/governors/menu.c @@ -317,12 +317,13 @@ static int menu_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, } /* - * Use a physical idle state, not busy polling, unless a timer - * is going to trigger soon enough or the exit latency of the - * idle state in question is greater than the predicted idle - * duration. + * Use a physical idle state instead of busy polling so long as + * its target residency is below the residency threshold, its + * exit latency is not greater than the predicted idle duration, + * and the next timer doesn't expire soon. */ if ((drv->states[idx].flags & CPUIDLE_FLAG_POLLING) && + s->target_residency_ns < RESIDENCY_THRESHOLD_NS && s->target_residency_ns <= data->next_timer_ns && s->exit_latency_ns <= predicted_ns) { predicted_ns = s->target_residency_ns; diff --git a/drivers/cpuidle/governors/teo.c b/drivers/cpuidle/governors/teo.c index bfa55c1eab5b..81ac5fd58a1c 100644 --- a/drivers/cpuidle/governors/teo.c +++ b/drivers/cpuidle/governors/teo.c @@ -76,7 +76,7 @@ * likely woken up by a non-timer wakeup source). * * 2. If the second sum computed in step 1 is greater than a half of the sum of - * both metrics for the candidate state bin and all subsequent bins(if any), + * both metrics for the candidate state bin and all subsequent bins (if any), * a shallower idle state is likely to be more suitable, so look for it. * * - Traverse the enabled idle states shallower than the candidate one in the @@ -133,21 +133,33 @@ struct teo_bin { * @sleep_length_ns: Time till the closest timer event (at the selection time). * @state_bins: Idle state data bins for this CPU. * @total: Grand total of the "intercepts" and "hits" metrics for all bins. + * @total_tick: Wakeups by the scheduler tick. * @tick_intercepts: "Intercepts" before TICK_NSEC. * @short_idles: Wakeups after short idle periods. - * @artificial_wakeup: Set if the wakeup has been triggered by a safety net. + * @tick_wakeup: Set if the last wakeup was by the scheduler tick. */ struct teo_cpu { s64 sleep_length_ns; struct teo_bin state_bins[CPUIDLE_STATE_MAX]; unsigned int total; + unsigned int total_tick; unsigned int tick_intercepts; unsigned int short_idles; - bool artificial_wakeup; + bool tick_wakeup; }; static DEFINE_PER_CPU(struct teo_cpu, teo_cpus); +static void teo_decay(unsigned int *metric) +{ + unsigned int delta = *metric >> DECAY_SHIFT; + + if (delta) + *metric -= delta; + else + *metric = 0; +} + /** * teo_update - Update CPU metrics after wakeup. * @drv: cpuidle driver containing state data. @@ -155,21 +167,22 @@ static DEFINE_PER_CPU(struct teo_cpu, teo_cpus); */ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) { - struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + struct teo_cpu *cpu_data = this_cpu_ptr(&teo_cpus); int i, idx_timer = 0, idx_duration = 0; - s64 target_residency_ns; - u64 measured_ns; + s64 target_residency_ns, measured_ns; + unsigned int total = 0; - cpu_data->short_idles -= cpu_data->short_idles >> DECAY_SHIFT; + teo_decay(&cpu_data->short_idles); - if (cpu_data->artificial_wakeup) { + if (dev->poll_time_limit) { + dev->poll_time_limit = false; /* - * If one of the safety nets has triggered, assume that this + * Polling state timeout has triggered, so assume that this * might have been a long sleep. */ - measured_ns = U64_MAX; + measured_ns = S64_MAX; } else { - u64 lat_ns = drv->states[dev->last_state_idx].exit_latency_ns; + s64 lat_ns = drv->states[dev->last_state_idx].exit_latency_ns; measured_ns = dev->last_residency_ns; /* @@ -196,8 +209,10 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) for (i = 0; i < drv->state_count; i++) { struct teo_bin *bin = &cpu_data->state_bins[i]; - bin->hits -= bin->hits >> DECAY_SHIFT; - bin->intercepts -= bin->intercepts >> DECAY_SHIFT; + teo_decay(&bin->hits); + total += bin->hits; + teo_decay(&bin->intercepts); + total += bin->intercepts; target_residency_ns = drv->states[i].target_residency_ns; @@ -208,7 +223,24 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) } } - cpu_data->tick_intercepts -= cpu_data->tick_intercepts >> DECAY_SHIFT; + cpu_data->total = total + PULSE; + + teo_decay(&cpu_data->tick_intercepts); + + teo_decay(&cpu_data->total_tick); + if (cpu_data->tick_wakeup) { + cpu_data->total_tick += PULSE; + /* + * If tick wakeups dominate the wakeup pattern, count this one + * as a hit on the deepest available idle state to increase the + * likelihood of stopping the tick. + */ + if (3 * cpu_data->total_tick > 2 * cpu_data->total) { + cpu_data->state_bins[drv->state_count-1].hits += PULSE; + return; + } + } + /* * If the measured idle duration falls into the same bin as the sleep * length, this is a "hit", so update the "hits" metric for that bin. @@ -219,18 +251,9 @@ static void teo_update(struct cpuidle_driver *drv, struct cpuidle_device *dev) cpu_data->state_bins[idx_timer].hits += PULSE; } else { cpu_data->state_bins[idx_duration].intercepts += PULSE; - if (TICK_NSEC <= measured_ns) + if (measured_ns <= TICK_NSEC) cpu_data->tick_intercepts += PULSE; } - - cpu_data->total -= cpu_data->total >> DECAY_SHIFT; - cpu_data->total += PULSE; -} - -static bool teo_state_ok(int i, struct cpuidle_driver *drv) -{ - return !tick_nohz_tick_stopped() || - drv->states[i].target_residency_ns >= TICK_NSEC; } /** @@ -239,17 +262,15 @@ static bool teo_state_ok(int i, struct cpuidle_driver *drv) * @dev: Target CPU. * @state_idx: Index of the capping idle state. * @duration_ns: Idle duration value to match. - * @no_poll: Don't consider polling states. */ static int teo_find_shallower_state(struct cpuidle_driver *drv, struct cpuidle_device *dev, int state_idx, - s64 duration_ns, bool no_poll) + s64 duration_ns) { int i; for (i = state_idx - 1; i >= 0; i--) { - if (dev->states_usage[i].disable || - (no_poll && drv->states[i].flags & CPUIDLE_FLAG_POLLING)) + if (dev->states_usage[i].disable) continue; state_idx = i; @@ -268,7 +289,7 @@ static int teo_find_shallower_state(struct cpuidle_driver *drv, static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, bool *stop_tick) { - struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + struct teo_cpu *cpu_data = this_cpu_ptr(&teo_cpus); s64 latency_req = cpuidle_governor_latency_req(dev->cpu); ktime_t delta_tick = TICK_NSEC / 2; unsigned int idx_intercept_sum = 0; @@ -356,7 +377,18 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, * better choice. */ if (2 * idx_intercept_sum > cpu_data->total - idx_hit_sum) { - int first_suitable_idx = idx; + int min_idx = idx0; + + if (tick_nohz_tick_stopped()) { + /* + * Look for the shallowest idle state below the current + * candidate one whose target residency is at least + * equal to the tick period length. + */ + while (min_idx < idx && + drv->states[min_idx].target_residency_ns < TICK_NSEC) + min_idx++; + } /* * Look for the deepest idle state whose target residency had @@ -366,49 +398,14 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, * Take the possible duration limitation present if the tick * has been stopped already into account. */ - intercept_sum = 0; - - for (i = idx - 1; i >= 0; i--) { - struct teo_bin *bin = &cpu_data->state_bins[i]; - - intercept_sum += bin->intercepts; - - if (2 * intercept_sum > idx_intercept_sum) { - /* - * Use the current state unless it is too - * shallow or disabled, in which case take the - * first enabled state that is deep enough. - */ - if (teo_state_ok(i, drv) && - !dev->states_usage[i].disable) { - idx = i; - break; - } - idx = first_suitable_idx; - break; - } + for (i = idx - 1, intercept_sum = 0; i >= min_idx; i--) { + intercept_sum += cpu_data->state_bins[i].intercepts; if (dev->states_usage[i].disable) continue; - if (teo_state_ok(i, drv)) { - /* - * The current state is deep enough, but still - * there may be a better one. - */ - first_suitable_idx = i; - continue; - } - - /* - * The current state is too shallow, so if no suitable - * states other than the initial candidate have been - * found, give up (the remaining states to check are - * shallower still), but otherwise the first suitable - * state other than the initial candidate may turn out - * to be preferable. - */ - if (first_suitable_idx == idx) + idx = i; + if (2 * intercept_sum > idx_intercept_sum) break; } } @@ -458,11 +455,8 @@ static int teo_select(struct cpuidle_driver *drv, struct cpuidle_device *dev, * If the closest expected timer is before the target residency of the * candidate state, a shallower one needs to be found. */ - if (drv->states[idx].target_residency_ns > duration_ns) { - i = teo_find_shallower_state(drv, dev, idx, duration_ns, false); - if (teo_state_ok(i, drv)) - idx = i; - } + if (drv->states[idx].target_residency_ns > duration_ns) + idx = teo_find_shallower_state(drv, dev, idx, duration_ns); /* * If the selected state's target residency is below the tick length @@ -490,7 +484,7 @@ end: */ if (idx > idx0 && drv->states[idx].target_residency_ns > delta_tick) - idx = teo_find_shallower_state(drv, dev, idx, delta_tick, false); + idx = teo_find_shallower_state(drv, dev, idx, delta_tick); out_tick: *stop_tick = false; @@ -504,20 +498,11 @@ out_tick: */ static void teo_reflect(struct cpuidle_device *dev, int state) { - struct teo_cpu *cpu_data = per_cpu_ptr(&teo_cpus, dev->cpu); + struct teo_cpu *cpu_data = this_cpu_ptr(&teo_cpus); + + cpu_data->tick_wakeup = tick_nohz_idle_got_tick(); dev->last_state_idx = state; - if (dev->poll_time_limit || - (tick_nohz_idle_got_tick() && cpu_data->sleep_length_ns > TICK_NSEC)) { - /* - * The wakeup was not "genuine", but triggered by one of the - * safety nets. - */ - dev->poll_time_limit = false; - cpu_data->artificial_wakeup = true; - } else { - cpu_data->artificial_wakeup = false; - } } /** diff --git a/drivers/cpuidle/poll_state.c b/drivers/cpuidle/poll_state.c index 9b6d90a72601..c7524e4c522a 100644 --- a/drivers/cpuidle/poll_state.c +++ b/drivers/cpuidle/poll_state.c @@ -4,9 +4,13 @@ */ #include <linux/cpuidle.h> +#include <linux/export.h> +#include <linux/irqflags.h> #include <linux/sched.h> #include <linux/sched/clock.h> #include <linux/sched/idle.h> +#include <linux/sprintf.h> +#include <linux/types.h> #define POLL_IDLE_RELAX_COUNT 200 diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index a6688d54984c..8d3b5d2890f8 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -728,6 +728,7 @@ config CRYPTO_DEV_TEGRA config CRYPTO_DEV_XILINX_TRNG tristate "Support for Xilinx True Random Generator" depends on ZYNQMP_FIRMWARE || COMPILE_TEST + select CRYPTO_DF80090A select CRYPTO_RNG select HW_RANDOM help diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c index 8bc08089f044..36a1ebca2e70 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-hash.c @@ -502,6 +502,7 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) algt = container_of(alg, struct sun8i_ss_alg_template, alg.hash.base); ss = algt->ss; + j = 0; digestsize = crypto_ahash_digestsize(tfm); if (digestsize == SHA224_DIGEST_SIZE) @@ -536,7 +537,6 @@ int sun8i_ss_hash_run(struct crypto_engine *engine, void *breq) goto err_dma_result; } - j = 0; len = areq->nbytes; sg = areq->src; i = 0; diff --git a/drivers/crypto/atmel-i2c.c b/drivers/crypto/atmel-i2c.c index a895e4289efa..9688d116d07e 100644 --- a/drivers/crypto/atmel-i2c.c +++ b/drivers/crypto/atmel-i2c.c @@ -402,7 +402,7 @@ EXPORT_SYMBOL(atmel_i2c_probe); static int __init atmel_i2c_init(void) { - atmel_wq = alloc_workqueue("atmel_wq", 0, 0); + atmel_wq = alloc_workqueue("atmel_wq", WQ_PERCPU, 0); return atmel_wq ? 0 : -ENOMEM; } diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 75ee065da1ec..b04d6379244a 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -252,7 +252,7 @@ struct artpec6_crypto_dma_descriptors { }; enum artpec6_crypto_variant { - ARTPEC6_CRYPTO, + ARTPEC6_CRYPTO = 1, ARTPEC7_CRYPTO, }; @@ -2842,7 +2842,6 @@ MODULE_DEVICE_TABLE(of, artpec6_crypto_of_match); static int artpec6_crypto_probe(struct platform_device *pdev) { - const struct of_device_id *match; enum artpec6_crypto_variant variant; struct artpec6_crypto *ac; struct device *dev = &pdev->dev; @@ -2853,12 +2852,10 @@ static int artpec6_crypto_probe(struct platform_device *pdev) if (artpec6_crypto_dev) return -ENODEV; - match = of_match_node(artpec6_crypto_of_match, dev->of_node); - if (!match) + variant = (enum artpec6_crypto_variant)of_device_get_match_data(dev); + if (!variant) return -EINVAL; - variant = (enum artpec6_crypto_variant)match->data; - base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) return PTR_ERR(base); diff --git a/drivers/crypto/caam/blob_gen.c b/drivers/crypto/caam/blob_gen.c index 079a22cc9f02..c18dbac56493 100644 --- a/drivers/crypto/caam/blob_gen.c +++ b/drivers/crypto/caam/blob_gen.c @@ -2,13 +2,14 @@ /* * Copyright (C) 2015 Pengutronix, Steffen Trumtrar <kernel@pengutronix.de> * Copyright (C) 2021 Pengutronix, Ahmad Fatoum <kernel@pengutronix.de> - * Copyright 2024 NXP + * Copyright 2024-2025 NXP */ #define pr_fmt(fmt) "caam blob_gen: " fmt #include <linux/bitfield.h> #include <linux/device.h> +#include <keys/trusted-type.h> #include <soc/fsl/caam-blob.h> #include "compat.h" @@ -60,18 +61,27 @@ static void caam_blob_job_done(struct device *dev, u32 *desc, u32 err, void *con complete(&res->completion); } +static u32 check_caam_state(struct device *jrdev) +{ + const struct caam_drv_private *ctrlpriv; + + ctrlpriv = dev_get_drvdata(jrdev->parent); + return FIELD_GET(CSTA_MOO, rd_reg32(&ctrlpriv->jr[0]->perfmon.status)); +} + int caam_process_blob(struct caam_blob_priv *priv, struct caam_blob_info *info, bool encap) { - const struct caam_drv_private *ctrlpriv; struct caam_blob_job_result testres; struct device *jrdev = &priv->jrdev; dma_addr_t dma_in, dma_out; int op = OP_PCLID_BLOB; + int hwbk_caam_ovhd = 0; size_t output_len; u32 *desc; u32 moo; int ret; + int len; if (info->key_mod_len > CAAM_BLOB_KEYMOD_LENGTH) return -EINVAL; @@ -82,14 +92,29 @@ int caam_process_blob(struct caam_blob_priv *priv, } else { op |= OP_TYPE_DECAP_PROTOCOL; output_len = info->input_len - CAAM_BLOB_OVERHEAD; + info->output_len = output_len; + } + + if (encap && info->pkey_info.is_pkey) { + op |= OP_PCL_BLOB_BLACK; + if (info->pkey_info.key_enc_algo == CAAM_ENC_ALGO_CCM) { + op |= OP_PCL_BLOB_EKT; + hwbk_caam_ovhd = CAAM_CCM_OVERHEAD; + } + if ((info->input_len + hwbk_caam_ovhd) > MAX_KEY_SIZE) + return -EINVAL; + + len = info->input_len + hwbk_caam_ovhd; + } else { + len = info->input_len; } desc = kzalloc(CAAM_BLOB_DESC_BYTES_MAX, GFP_KERNEL); if (!desc) return -ENOMEM; - dma_in = dma_map_single(jrdev, info->input, info->input_len, - DMA_TO_DEVICE); + dma_in = dma_map_single(jrdev, info->input, len, + encap ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); if (dma_mapping_error(jrdev, dma_in)) { dev_err(jrdev, "unable to map input DMA buffer\n"); ret = -ENOMEM; @@ -104,8 +129,7 @@ int caam_process_blob(struct caam_blob_priv *priv, goto out_unmap_in; } - ctrlpriv = dev_get_drvdata(jrdev->parent); - moo = FIELD_GET(CSTA_MOO, rd_reg32(&ctrlpriv->jr[0]->perfmon.status)); + moo = check_caam_state(jrdev); if (moo != CSTA_MOO_SECURE && moo != CSTA_MOO_TRUSTED) dev_warn(jrdev, "using insecure test key, enable HAB to use unique device key!\n"); @@ -117,18 +141,48 @@ int caam_process_blob(struct caam_blob_priv *priv, * Class 1 Context DWords 0+1+2+3. The random BK is stored in the * Class 1 Key Register. Operation Mode is set to AES-CCM. */ - init_job_desc(desc, 0); + + if (encap && info->pkey_info.is_pkey) { + /*!1. key command used to load class 1 key register + * from input plain key. + */ + append_key(desc, dma_in, info->input_len, + CLASS_1 | KEY_DEST_CLASS_REG); + /*!2. Fifostore to store protected key from class 1 key register. */ + if (info->pkey_info.key_enc_algo == CAAM_ENC_ALGO_CCM) { + append_fifo_store(desc, dma_in, info->input_len, + LDST_CLASS_1_CCB | + FIFOST_TYPE_KEY_CCM_JKEK); + } else { + append_fifo_store(desc, dma_in, info->input_len, + LDST_CLASS_1_CCB | + FIFOST_TYPE_KEY_KEK); + } + /* + * JUMP_OFFSET specifies the offset of the JUMP target from + * the JUMP command's address in the descriptor buffer. + */ + append_jump(desc, JUMP_COND_NOP | BIT(0) << JUMP_OFFSET_SHIFT); + } + + /*!3. Load class 2 key with key modifier. */ append_key_as_imm(desc, info->key_mod, info->key_mod_len, - info->key_mod_len, CLASS_2 | KEY_DEST_CLASS_REG); - append_seq_in_ptr_intlen(desc, dma_in, info->input_len, 0); - append_seq_out_ptr_intlen(desc, dma_out, output_len, 0); + info->key_mod_len, CLASS_2 | KEY_DEST_CLASS_REG); + + /*!4. SEQ IN PTR Command. */ + append_seq_in_ptr(desc, dma_in, info->input_len, 0); + + /*!5. SEQ OUT PTR Command. */ + append_seq_out_ptr(desc, dma_out, output_len, 0); + + /*!6. Blob encapsulation/decapsulation PROTOCOL Command. */ append_operation(desc, op); - print_hex_dump_debug("data@"__stringify(__LINE__)": ", + print_hex_dump_debug("data@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 1, info->input, - info->input_len, false); - print_hex_dump_debug("jobdesc@"__stringify(__LINE__)": ", + len, false); + print_hex_dump_debug("jobdesc@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 1, desc, desc_bytes(desc), false); @@ -139,7 +193,7 @@ int caam_process_blob(struct caam_blob_priv *priv, if (ret == -EINPROGRESS) { wait_for_completion(&testres.completion); ret = testres.err; - print_hex_dump_debug("output@"__stringify(__LINE__)": ", + print_hex_dump_debug("output@" __stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 1, info->output, output_len, false); } @@ -149,10 +203,10 @@ int caam_process_blob(struct caam_blob_priv *priv, dma_unmap_single(jrdev, dma_out, output_len, DMA_FROM_DEVICE); out_unmap_in: - dma_unmap_single(jrdev, dma_in, info->input_len, DMA_TO_DEVICE); + dma_unmap_single(jrdev, dma_in, len, + encap ? DMA_BIDIRECTIONAL : DMA_TO_DEVICE); out_free: kfree(desc); - return ret; } EXPORT_SYMBOL(caam_process_blob); diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 2cfb1b8d8c7c..32a6e6e15ee2 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -3,7 +3,7 @@ * caam - Freescale FSL CAAM support for crypto API * * Copyright 2008-2011 Freescale Semiconductor, Inc. - * Copyright 2016-2019, 2023 NXP + * Copyright 2016-2019, 2023, 2025 NXP * * Based on talitos crypto API driver. * @@ -61,13 +61,16 @@ #include <crypto/internal/engine.h> #include <crypto/internal/skcipher.h> #include <crypto/xts.h> +#include <keys/trusted-type.h> #include <linux/dma-mapping.h> #include <linux/device.h> #include <linux/err.h> #include <linux/module.h> #include <linux/kernel.h> +#include <linux/key-type.h> #include <linux/slab.h> #include <linux/string.h> +#include <soc/fsl/caam-blob.h> /* * crypto alg @@ -119,12 +122,15 @@ struct caam_ctx { dma_addr_t sh_desc_enc_dma; dma_addr_t sh_desc_dec_dma; dma_addr_t key_dma; + u8 protected_key[CAAM_MAX_KEY_SIZE]; + dma_addr_t protected_key_dma; enum dma_data_direction dir; struct device *jrdev; struct alginfo adata; struct alginfo cdata; unsigned int authsize; bool xts_key_fallback; + bool is_blob; struct crypto_skcipher *fallback; }; @@ -751,9 +757,14 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, print_hex_dump_debug("key in @"__stringify(__LINE__)": ", DUMP_PREFIX_ADDRESS, 16, 4, key, keylen, 1); + /* Here keylen is actual key length */ ctx->cdata.keylen = keylen; ctx->cdata.key_virt = key; ctx->cdata.key_inline = true; + /* Here protected key len is plain key length */ + ctx->cdata.plain_keylen = keylen; + ctx->cdata.key_cmd_opt = 0; + /* skcipher_encrypt shared descriptor */ desc = ctx->sh_desc_enc; @@ -772,6 +783,62 @@ static int skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, return 0; } +static int paes_skcipher_setkey(struct crypto_skcipher *skcipher, + const u8 *key, + unsigned int keylen) +{ + struct caam_pkey_info *pkey_info = (struct caam_pkey_info *)key; + struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); + struct device *jrdev = ctx->jrdev; + int err; + + ctx->cdata.key_inline = false; + + keylen = keylen - CAAM_PKEY_HEADER; + + /* Retrieve the length of key */ + ctx->cdata.plain_keylen = pkey_info->plain_key_sz; + + /* Retrieve the length of blob*/ + ctx->cdata.keylen = keylen; + + /* Retrieve the address of the blob */ + ctx->cdata.key_virt = pkey_info->key_buf; + + /* Validate key length for AES algorithms */ + err = aes_check_keylen(ctx->cdata.plain_keylen); + if (err) { + dev_err(jrdev, "bad key length\n"); + return err; + } + + /* set command option */ + ctx->cdata.key_cmd_opt |= KEY_ENC; + + /* check if the Protected-Key is CCM key */ + if (pkey_info->key_enc_algo == CAAM_ENC_ALGO_CCM) + ctx->cdata.key_cmd_opt |= KEY_EKT; + + memcpy(ctx->key, ctx->cdata.key_virt, keylen); + dma_sync_single_for_device(jrdev, ctx->key_dma, keylen, DMA_TO_DEVICE); + ctx->cdata.key_dma = ctx->key_dma; + + if (pkey_info->key_enc_algo == CAAM_ENC_ALGO_CCM) + ctx->protected_key_dma = dma_map_single(jrdev, ctx->protected_key, + ctx->cdata.plain_keylen + + CAAM_CCM_OVERHEAD, + DMA_FROM_DEVICE); + else + ctx->protected_key_dma = dma_map_single(jrdev, ctx->protected_key, + ctx->cdata.plain_keylen, + DMA_FROM_DEVICE); + + ctx->cdata.protected_key_dma = ctx->protected_key_dma; + ctx->is_blob = true; + + return 0; +} + static int aes_skcipher_setkey(struct crypto_skcipher *skcipher, const u8 *key, unsigned int keylen) { @@ -1254,7 +1321,9 @@ static void init_skcipher_job(struct skcipher_request *req, struct caam_ctx *ctx = crypto_skcipher_ctx_dma(skcipher); struct device *jrdev = ctx->jrdev; int ivsize = crypto_skcipher_ivsize(skcipher); - u32 *desc = edesc->hw_desc; + u32 *desc = !ctx->is_blob ? edesc->hw_desc : + (u32 *)((u8 *)edesc->hw_desc + CAAM_DESC_BYTES_MAX); + dma_addr_t desc_dma; u32 *sh_desc; u32 in_options = 0, out_options = 0; dma_addr_t src_dma, dst_dma, ptr; @@ -1269,11 +1338,6 @@ static void init_skcipher_job(struct skcipher_request *req, DUMP_PREFIX_ADDRESS, 16, 4, req->src, edesc->src_nents > 1 ? 100 : req->cryptlen, 1); - sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; - ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; - - len = desc_len(sh_desc); - init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); if (ivsize || edesc->mapped_src_nents > 1) { src_dma = edesc->sec4_sg_dma; @@ -1283,8 +1347,6 @@ static void init_skcipher_job(struct skcipher_request *req, src_dma = sg_dma_address(req->src); } - append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); - if (likely(req->src == req->dst)) { dst_dma = src_dma + !!ivsize * sizeof(struct sec4_sg_entry); out_options = in_options; @@ -1296,7 +1358,25 @@ static void init_skcipher_job(struct skcipher_request *req, out_options = LDST_SGF; } - append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); + if (ctx->is_blob) { + cnstr_desc_skcipher_enc_dec(desc, &ctx->cdata, + src_dma, dst_dma, req->cryptlen + ivsize, + in_options, out_options, + ivsize, encrypt); + + desc_dma = dma_map_single(jrdev, desc, desc_bytes(desc), DMA_TO_DEVICE); + + cnstr_desc_protected_blob_decap(edesc->hw_desc, &ctx->cdata, desc_dma); + } else { + sh_desc = encrypt ? ctx->sh_desc_enc : ctx->sh_desc_dec; + ptr = encrypt ? ctx->sh_desc_enc_dma : ctx->sh_desc_dec_dma; + + len = desc_len(sh_desc); + init_job_desc_shared(desc, ptr, len, HDR_SHARE_DEFER | HDR_REVERSE); + append_seq_in_ptr(desc, src_dma, req->cryptlen + ivsize, in_options); + + append_seq_out_ptr(desc, dst_dma, req->cryptlen + ivsize, out_options); + } } /* @@ -1817,6 +1897,7 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) struct caam_drv_private *ctrlpriv = dev_get_drvdata(jrdev->parent); u32 *desc; int ret = 0; + int len; /* * XTS is expected to return an error even for input length = 0 @@ -1842,8 +1923,12 @@ static inline int skcipher_crypt(struct skcipher_request *req, bool encrypt) crypto_skcipher_decrypt(&rctx->fallback_req); } + len = DESC_JOB_IO_LEN * CAAM_CMD_SZ; + if (ctx->is_blob) + len += CAAM_DESC_BYTES_MAX; + /* allocate extended descriptor */ - edesc = skcipher_edesc_alloc(req, DESC_JOB_IO_LEN * CAAM_CMD_SZ); + edesc = skcipher_edesc_alloc(req, len); if (IS_ERR(edesc)) return PTR_ERR(edesc); @@ -1888,6 +1973,27 @@ static struct caam_skcipher_alg driver_algs[] = { { .skcipher.base = { .base = { + .cra_name = "cbc(paes)", + .cra_driver_name = "cbc-paes-caam", + .cra_blocksize = AES_BLOCK_SIZE, + }, + .setkey = paes_skcipher_setkey, + .encrypt = skcipher_encrypt, + .decrypt = skcipher_decrypt, + .min_keysize = AES_MIN_KEY_SIZE + CAAM_BLOB_OVERHEAD + + CAAM_PKEY_HEADER, + .max_keysize = AES_MAX_KEY_SIZE + CAAM_BLOB_OVERHEAD + + CAAM_PKEY_HEADER, + .ivsize = AES_BLOCK_SIZE, + }, + .skcipher.op = { + .do_one_request = skcipher_do_one_req, + }, + .caam.class1_alg_type = OP_ALG_ALGSEL_AES | OP_ALG_AAI_CBC, + }, + { + .skcipher.base = { + .base = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-caam", .cra_blocksize = AES_BLOCK_SIZE, diff --git a/drivers/crypto/caam/caamalg_desc.c b/drivers/crypto/caam/caamalg_desc.c index 7571e1ac913b..04c1105eb1f5 100644 --- a/drivers/crypto/caam/caamalg_desc.c +++ b/drivers/crypto/caam/caamalg_desc.c @@ -2,12 +2,13 @@ /* * Shared descriptors for aead, skcipher algorithms * - * Copyright 2016-2019 NXP + * Copyright 2016-2019, 2025 NXP */ #include "compat.h" #include "desc_constr.h" #include "caamalg_desc.h" +#include <soc/fsl/caam-blob.h> /* * For aead functions, read payload and write payload, @@ -1364,6 +1365,84 @@ static inline void skcipher_append_src_dst(u32 *desc) append_seq_fifo_store(desc, 0, FIFOST_TYPE_MESSAGE_DATA | KEY_VLF); } +void cnstr_desc_skcipher_enc_dec(u32 * const desc, struct alginfo *cdata, + dma_addr_t src, dma_addr_t dst, unsigned int data_sz, + unsigned int in_options, unsigned int out_options, + unsigned int ivsize, const bool encrypt) +{ + u32 options = cdata->algtype | OP_ALG_AS_INIT; + + if (encrypt) + options |= OP_ALG_ENCRYPT; + else + options |= OP_ALG_DECRYPT; + + init_job_desc(desc, 0); + + append_jump(desc, JUMP_JSL | JUMP_TYPE_LOCAL | + JUMP_COND_NOP | JUMP_TEST_ALL | 1); + + append_key(desc, cdata->protected_key_dma, cdata->plain_keylen, + CLASS_1 | KEY_DEST_CLASS_REG | cdata->key_cmd_opt); + + append_seq_in_ptr(desc, src, data_sz, in_options); + + append_seq_out_ptr(desc, dst, data_sz, out_options); + + /* Load IV, if there is one */ + if (ivsize) + append_seq_load(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB); + + append_operation(desc, options); + + skcipher_append_src_dst(desc); + + /* Store IV */ + if (ivsize) + append_seq_store(desc, ivsize, LDST_SRCDST_BYTE_CONTEXT | + LDST_CLASS_1_CCB); + + print_hex_dump_debug("skcipher_enc_dec job desc@" __stringify(__LINE__)": ", + DUMP_PREFIX_ADDRESS, 16, 4, desc, desc_bytes(desc), + 1); +} +EXPORT_SYMBOL(cnstr_desc_skcipher_enc_dec); + +void cnstr_desc_protected_blob_decap(u32 * const desc, struct alginfo *cdata, + dma_addr_t next_desc_addr) +{ + u32 protected_store; + + init_job_desc(desc, 0); + + /* Load key modifier */ + append_load_as_imm(desc, KEYMOD, sizeof(KEYMOD) - 1, + LDST_CLASS_2_CCB | LDST_SRCDST_BYTE_KEY); + + append_seq_in_ptr_intlen(desc, cdata->key_dma, + cdata->plain_keylen + CAAM_BLOB_OVERHEAD, 0); + + append_seq_out_ptr_intlen(desc, cdata->protected_key_dma, + cdata->plain_keylen, 0); + + protected_store = OP_PCLID_BLOB | OP_PCL_BLOB_BLACK; + if ((cdata->key_cmd_opt >> KEY_EKT_OFFSET) & 1) + protected_store |= OP_PCL_BLOB_EKT; + + append_operation(desc, OP_TYPE_DECAP_PROTOCOL | protected_store); + + if (next_desc_addr) { + append_jump(desc, JUMP_TYPE_NONLOCAL | JUMP_TEST_ALL); + append_ptr(desc, next_desc_addr); + } + + print_hex_dump_debug("protected blob decap job desc@" __stringify(__LINE__) ":", + DUMP_PREFIX_ADDRESS, 16, 4, desc, + desc_bytes(desc), 1); +} +EXPORT_SYMBOL(cnstr_desc_protected_blob_decap); + /** * cnstr_shdsc_skcipher_encap - skcipher encapsulation shared descriptor * @desc: pointer to buffer used for descriptor construction @@ -1391,7 +1470,8 @@ void cnstr_shdsc_skcipher_encap(u32 * const desc, struct alginfo *cdata, /* Load class1 key only */ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, - cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG + | cdata->key_cmd_opt); /* Load nonce into CONTEXT1 reg */ if (is_rfc3686) { @@ -1466,7 +1546,8 @@ void cnstr_shdsc_skcipher_decap(u32 * const desc, struct alginfo *cdata, /* Load class1 key only */ append_key_as_imm(desc, cdata->key_virt, cdata->keylen, - cdata->keylen, CLASS_1 | KEY_DEST_CLASS_REG); + cdata->plain_keylen, CLASS_1 | KEY_DEST_CLASS_REG + | cdata->key_cmd_opt); /* Load nonce into CONTEXT1 reg */ if (is_rfc3686) { diff --git a/drivers/crypto/caam/caamalg_desc.h b/drivers/crypto/caam/caamalg_desc.h index f2893393ba5e..323490a4a756 100644 --- a/drivers/crypto/caam/caamalg_desc.h +++ b/drivers/crypto/caam/caamalg_desc.h @@ -2,7 +2,7 @@ /* * Shared descriptors for aead, skcipher algorithms * - * Copyright 2016 NXP + * Copyright 2016, 2025 NXP */ #ifndef _CAAMALG_DESC_H_ @@ -48,6 +48,9 @@ #define DESC_SKCIPHER_DEC_LEN (DESC_SKCIPHER_BASE + \ 16 * CAAM_CMD_SZ) +/* Key modifier for CAAM Protected blobs */ +#define KEYMOD "SECURE_KEY" + void cnstr_shdsc_aead_null_encap(u32 * const desc, struct alginfo *adata, unsigned int icvsize, int era); @@ -113,4 +116,12 @@ void cnstr_shdsc_xts_skcipher_encap(u32 * const desc, struct alginfo *cdata); void cnstr_shdsc_xts_skcipher_decap(u32 * const desc, struct alginfo *cdata); +void cnstr_desc_protected_blob_decap(u32 * const desc, struct alginfo *cdata, + dma_addr_t next_desc); + +void cnstr_desc_skcipher_enc_dec(u32 * const desc, struct alginfo *cdata, + dma_addr_t src, dma_addr_t dst, unsigned int data_sz, + unsigned int in_options, unsigned int out_options, + unsigned int ivsize, const bool encrypt); + #endif /* _CAAMALG_DESC_H_ */ diff --git a/drivers/crypto/caam/caamrng.c b/drivers/crypto/caam/caamrng.c index b3d14a7f4dd1..0eb43c862516 100644 --- a/drivers/crypto/caam/caamrng.c +++ b/drivers/crypto/caam/caamrng.c @@ -181,7 +181,9 @@ static inline void test_len(struct hwrng *rng, size_t len, bool wait) struct device *dev = ctx->ctrldev; buf = kcalloc(CAAM_RNG_MAX_FIFO_STORE_SIZE, sizeof(u8), GFP_KERNEL); - + if (!buf) { + return; + } while (len > 0) { read_len = rng->read(rng, buf, len, wait); diff --git a/drivers/crypto/caam/desc.h b/drivers/crypto/caam/desc.h index e13470901586..c28e94fcb8c7 100644 --- a/drivers/crypto/caam/desc.h +++ b/drivers/crypto/caam/desc.h @@ -4,7 +4,7 @@ * Definitions to support CAAM descriptor instruction generation * * Copyright 2008-2011 Freescale Semiconductor, Inc. - * Copyright 2018 NXP + * Copyright 2018, 2025 NXP */ #ifndef DESC_H @@ -162,6 +162,7 @@ * Enhanced Encryption of Key */ #define KEY_EKT 0x00100000 +#define KEY_EKT_OFFSET 20 /* * Encrypted with Trusted Key @@ -403,6 +404,7 @@ #define FIFOST_TYPE_PKHA_N (0x08 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_A (0x0c << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_B (0x0d << FIFOST_TYPE_SHIFT) +#define FIFOST_TYPE_KEY_CCM_JKEK (0x14 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_AF_SBOX_JKEK (0x20 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_AF_SBOX_TKEK (0x21 << FIFOST_TYPE_SHIFT) #define FIFOST_TYPE_PKHA_E_JKEK (0x22 << FIFOST_TYPE_SHIFT) @@ -1001,6 +1003,11 @@ #define OP_PCL_TLS12_AES_256_CBC_SHA384 0xff63 #define OP_PCL_TLS12_AES_256_CBC_SHA512 0xff65 +/* Blob protocol protinfo bits */ + +#define OP_PCL_BLOB_BLACK 0x0004 +#define OP_PCL_BLOB_EKT 0x0100 + /* For DTLS - OP_PCLID_DTLS */ #define OP_PCL_DTLS_AES_128_CBC_SHA 0x002f diff --git a/drivers/crypto/caam/desc_constr.h b/drivers/crypto/caam/desc_constr.h index 824c94d44f94..2a29dd2c9c8a 100644 --- a/drivers/crypto/caam/desc_constr.h +++ b/drivers/crypto/caam/desc_constr.h @@ -3,7 +3,7 @@ * caam descriptor construction helper functions * * Copyright 2008-2012 Freescale Semiconductor, Inc. - * Copyright 2019 NXP + * Copyright 2019, 2025 NXP */ #ifndef DESC_CONSTR_H @@ -498,17 +498,23 @@ do { \ * @keylen: length of the provided algorithm key, in bytes * @keylen_pad: padded length of the provided algorithm key, in bytes * @key_dma: dma (bus) address where algorithm key resides + * @protected_key_dma: dma (bus) address where protected key resides * @key_virt: virtual address where algorithm key resides * @key_inline: true - key can be inlined in the descriptor; false - key is * referenced by the descriptor + * @plain_keylen: size of the key to be loaded by the CAAM + * @key_cmd_opt: optional parameters for KEY command */ struct alginfo { u32 algtype; unsigned int keylen; unsigned int keylen_pad; dma_addr_t key_dma; + dma_addr_t protected_key_dma; const void *key_virt; bool key_inline; + u32 plain_keylen; + u32 key_cmd_opt; }; /** diff --git a/drivers/crypto/cavium/nitrox/nitrox_mbx.c b/drivers/crypto/cavium/nitrox/nitrox_mbx.c index d4e06999af9b..a6a76e50ba84 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_mbx.c +++ b/drivers/crypto/cavium/nitrox/nitrox_mbx.c @@ -192,7 +192,7 @@ int nitrox_mbox_init(struct nitrox_device *ndev) } /* allocate pf2vf response workqueue */ - ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", 0, 0); + ndev->iov.pf2vf_wq = alloc_workqueue("nitrox_pf2vf", WQ_PERCPU, 0); if (!ndev->iov.pf2vf_wq) { kfree(ndev->iov.vfdev); ndev->iov.vfdev = NULL; diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index c531d13d971f..246801912e1a 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -507,7 +507,7 @@ int ccp_trng_read(struct hwrng *rng, void *data, size_t max, bool wait) { struct ccp_device *ccp = container_of(rng, struct ccp_device, hwrng); u32 trng_value; - int len = min_t(int, sizeof(trng_value), max); + int len = min(sizeof(trng_value), max); /* Locking is provided by the caller so we can update device * hwrng-related fields safely diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 0d13d47c164b..b28a6f50daaa 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -259,27 +259,20 @@ static int sev_cmd_buffer_len(int cmd) static struct file *open_file_as_root(const char *filename, int flags, umode_t mode) { - struct file *fp; - struct path root; - struct cred *cred; - const struct cred *old_cred; + struct path root __free(path_put) = {}; task_lock(&init_task); get_fs_root(init_task.fs, &root); task_unlock(&init_task); - cred = prepare_creds(); + CLASS(prepare_creds, cred)(); if (!cred) return ERR_PTR(-ENOMEM); - cred->fsuid = GLOBAL_ROOT_UID; - old_cred = override_creds(cred); - fp = file_open_root(&root, filename, flags, mode); - path_put(&root); - - put_cred(revert_creds(old_cred)); + cred->fsuid = GLOBAL_ROOT_UID; - return fp; + scoped_with_creds(cred) + return file_open_root(&root, filename, flags, mode); } static int sev_read_init_ex_file(void) diff --git a/drivers/crypto/ccp/sp-dev.h b/drivers/crypto/ccp/sp-dev.h index 6f9d7063257d..1335a83fe052 100644 --- a/drivers/crypto/ccp/sp-dev.h +++ b/drivers/crypto/ccp/sp-dev.h @@ -95,7 +95,7 @@ struct sp_device { struct device *dev; - struct sp_dev_vdata *dev_vdata; + const struct sp_dev_vdata *dev_vdata; unsigned int ord; char name[SP_MAX_NAME_LEN]; diff --git a/drivers/crypto/ccp/sp-pci.c b/drivers/crypto/ccp/sp-pci.c index e7bb803912a6..8891ceee1d7d 100644 --- a/drivers/crypto/ccp/sp-pci.c +++ b/drivers/crypto/ccp/sp-pci.c @@ -459,6 +459,17 @@ static const struct psp_vdata pspv6 = { .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ }; +static const struct psp_vdata pspv7 = { + .tee = &teev2, + .cmdresp_reg = 0x10944, /* C2PMSG_17 */ + .cmdbuff_addr_lo_reg = 0x10948, /* C2PMSG_18 */ + .cmdbuff_addr_hi_reg = 0x1094c, /* C2PMSG_19 */ + .bootloader_info_reg = 0x109ec, /* C2PMSG_59 */ + .feature_reg = 0x109fc, /* C2PMSG_63 */ + .inten_reg = 0x10510, /* P2CMSG_INTEN */ + .intsts_reg = 0x10514, /* P2CMSG_INTSTS */ +}; + #endif static const struct sp_dev_vdata dev_vdata[] = { @@ -525,6 +536,13 @@ static const struct sp_dev_vdata dev_vdata[] = { .psp_vdata = &pspv6, #endif }, + { /* 9 */ + .bar = 2, +#ifdef CONFIG_CRYPTO_DEV_SP_PSP + .psp_vdata = &pspv7, +#endif + }, + }; static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x1537), (kernel_ulong_t)&dev_vdata[0] }, @@ -539,6 +557,7 @@ static const struct pci_device_id sp_pci_table[] = { { PCI_VDEVICE(AMD, 0x17E0), (kernel_ulong_t)&dev_vdata[7] }, { PCI_VDEVICE(AMD, 0x156E), (kernel_ulong_t)&dev_vdata[8] }, { PCI_VDEVICE(AMD, 0x17D8), (kernel_ulong_t)&dev_vdata[8] }, + { PCI_VDEVICE(AMD, 0x115A), (kernel_ulong_t)&dev_vdata[9] }, /* Last entry must be zero */ { 0, } }; diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c index 3933cac1694d..3f9843fa7782 100644 --- a/drivers/crypto/ccp/sp-platform.c +++ b/drivers/crypto/ccp/sp-platform.c @@ -52,24 +52,13 @@ static const struct of_device_id sp_of_match[] = { }; MODULE_DEVICE_TABLE(of, sp_of_match); -static struct sp_dev_vdata *sp_get_of_version(struct platform_device *pdev) -{ - const struct of_device_id *match; - - match = of_match_node(sp_of_match, pdev->dev.of_node); - if (match && match->data) - return (struct sp_dev_vdata *)match->data; - - return NULL; -} - -static struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev) +static const struct sp_dev_vdata *sp_get_acpi_version(struct platform_device *pdev) { const struct acpi_device_id *match; match = acpi_match_device(sp_acpi_match, &pdev->dev); if (match && match->driver_data) - return (struct sp_dev_vdata *)match->driver_data; + return (const struct sp_dev_vdata *)match->driver_data; return NULL; } @@ -123,7 +112,7 @@ static int sp_platform_probe(struct platform_device *pdev) goto e_err; sp->dev_specific = sp_platform; - sp->dev_vdata = pdev->dev.of_node ? sp_get_of_version(pdev) + sp->dev_vdata = pdev->dev.of_node ? of_device_get_match_data(&pdev->dev) : sp_get_acpi_version(pdev); if (!sp->dev_vdata) { ret = -ENODEV; diff --git a/drivers/crypto/ccree/cc_buffer_mgr.c b/drivers/crypto/ccree/cc_buffer_mgr.c index 3963bb91321f..dc7e0cd51c25 100644 --- a/drivers/crypto/ccree/cc_buffer_mgr.c +++ b/drivers/crypto/ccree/cc_buffer_mgr.c @@ -1235,6 +1235,7 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, int rc = 0; u32 dummy = 0; u32 mapped_nents = 0; + int sg_nents; dev_dbg(dev, " update params : curr_buff=%p curr_buff_cnt=0x%X nbytes=0x%X src=%p curr_index=%u\n", curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index); @@ -1248,7 +1249,10 @@ int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, if (total_in_len < block_size) { dev_dbg(dev, " less than one block: curr_buff=%p *curr_buff_cnt=0x%X copy_to=%p\n", curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]); - areq_ctx->in_nents = sg_nents_for_len(src, nbytes); + sg_nents = sg_nents_for_len(src, nbytes); + if (sg_nents < 0) + return sg_nents; + areq_ctx->in_nents = sg_nents; sg_copy_to_buffer(src, areq_ctx->in_nents, &curr_buff[*curr_buff_cnt], nbytes); *curr_buff_cnt += nbytes; diff --git a/drivers/crypto/hifn_795x.c b/drivers/crypto/hifn_795x.c index 925991526745..edf36f6add52 100644 --- a/drivers/crypto/hifn_795x.c +++ b/drivers/crypto/hifn_795x.c @@ -913,11 +913,10 @@ static void hifn_init_pll(struct hifn_device *dev) else pllcfg |= HIFN_PLL_REF_CLK_HBI; - if (hifn_pll_ref[3] != '\0') - freq = simple_strtoul(hifn_pll_ref + 3, NULL, 10); - else { + if (hifn_pll_ref[3] == '\0' || + kstrtouint(hifn_pll_ref + 3, 10, &freq)) { freq = 66; - dev_info(&dev->pdev->dev, "assuming %uMHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", + dev_info(&dev->pdev->dev, "assuming %u MHz clock speed, override with hifn_pll_ref=%.3s<frequency>\n", freq, hifn_pll_ref); } diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 3b391a146635..be25ecbdba69 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -64,10 +64,10 @@ #define QM_EQE_AEQE_SIZE (2UL << 12) #define QM_EQC_PHASE_SHIFT 16 -#define QM_EQE_PHASE(eqe) ((le32_to_cpu((eqe)->dw0) >> 16) & 0x1) +#define QM_EQE_PHASE(dw0) (((dw0) >> 16) & 0x1) #define QM_EQE_CQN_MASK GENMASK(15, 0) -#define QM_AEQE_PHASE(aeqe) ((le32_to_cpu((aeqe)->dw0) >> 16) & 0x1) +#define QM_AEQE_PHASE(dw0) (((dw0) >> 16) & 0x1) #define QM_AEQE_TYPE_SHIFT 17 #define QM_AEQE_TYPE_MASK 0xf #define QM_AEQE_CQN_MASK GENMASK(15, 0) @@ -976,23 +976,23 @@ static void qm_get_complete_eqe_num(struct hisi_qm *qm) { struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; struct hisi_qm_poll_data *poll_data = NULL; + u32 dw0 = le32_to_cpu(eqe->dw0); u16 eq_depth = qm->eq_depth; u16 cqn, eqe_num = 0; - if (QM_EQE_PHASE(eqe) != qm->status.eqc_phase) { + if (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) { atomic64_inc(&qm->debug.dfx.err_irq_cnt); qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); return; } - cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; + cqn = dw0 & QM_EQE_CQN_MASK; if (unlikely(cqn >= qm->qp_num)) return; poll_data = &qm->poll_data[cqn]; - while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { - cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; - poll_data->qp_finish_id[eqe_num] = cqn; + while (QM_EQE_PHASE(dw0) != qm->status.eqc_phase) { + poll_data->qp_finish_id[eqe_num] = dw0 & QM_EQE_CQN_MASK; eqe_num++; if (qm->status.eq_head == eq_depth - 1) { @@ -1006,6 +1006,8 @@ static void qm_get_complete_eqe_num(struct hisi_qm *qm) if (eqe_num == (eq_depth >> 1) - 1) break; + + dw0 = le32_to_cpu(eqe->dw0); } poll_data->eqe_num = eqe_num; @@ -1098,15 +1100,15 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) { struct hisi_qm *qm = data; struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; + u32 dw0 = le32_to_cpu(aeqe->dw0); u16 aeq_depth = qm->aeq_depth; u32 type, qp_id; atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); - while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { - type = (le32_to_cpu(aeqe->dw0) >> QM_AEQE_TYPE_SHIFT) & - QM_AEQE_TYPE_MASK; - qp_id = le32_to_cpu(aeqe->dw0) & QM_AEQE_CQN_MASK; + while (QM_AEQE_PHASE(dw0) == qm->status.aeqc_phase) { + type = (dw0 >> QM_AEQE_TYPE_SHIFT) & QM_AEQE_TYPE_MASK; + qp_id = dw0 & QM_AEQE_CQN_MASK; switch (type) { case QM_EQ_OVERFLOW: @@ -1134,6 +1136,7 @@ static irqreturn_t qm_aeq_thread(int irq, void *data) aeqe++; qm->status.aeq_head++; } + dw0 = le32_to_cpu(aeqe->dw0); } qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); @@ -1283,6 +1286,13 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT); } break; + /* + * Note: The current logic only needs to handle the above three types + * If new types are added, they need to be supplemented here, + * otherwise undefined behavior may occur. + */ + default: + break; } } @@ -2652,10 +2662,10 @@ static int qm_hw_err_isolate(struct hisi_qm *qm) } } list_add(&hw_err->list, &isolate->qm_hw_errs); - mutex_unlock(&isolate->isolate_lock); if (count >= isolate->err_threshold) isolate->is_isolate = true; + mutex_unlock(&isolate->isolate_lock); return 0; } @@ -2664,12 +2674,10 @@ static void qm_hw_err_destroy(struct hisi_qm *qm) { struct qm_hw_err *err, *tmp; - mutex_lock(&qm->isolate_data.isolate_lock); list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) { list_del(&err->list); kfree(err); } - mutex_unlock(&qm->isolate_data.isolate_lock); } static enum uacce_dev_state hisi_qm_get_isolate_state(struct uacce_device *uacce) @@ -2697,10 +2705,12 @@ static int hisi_qm_isolate_threshold_write(struct uacce_device *uacce, u32 num) if (qm->isolate_data.is_isolate) return -EPERM; + mutex_lock(&qm->isolate_data.isolate_lock); qm->isolate_data.err_threshold = num; /* After the policy is updated, need to reset the hardware err list */ qm_hw_err_destroy(qm); + mutex_unlock(&qm->isolate_data.isolate_lock); return 0; } @@ -2737,7 +2747,10 @@ static void qm_remove_uacce(struct hisi_qm *qm) struct uacce_device *uacce = qm->uacce; if (qm->use_sva) { + mutex_lock(&qm->isolate_data.isolate_lock); qm_hw_err_destroy(qm); + mutex_unlock(&qm->isolate_data.isolate_lock); + uacce_remove(uacce); qm->uacce = NULL; } @@ -3678,6 +3691,7 @@ static void qm_clear_vft_config(struct hisi_qm *qm) static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) { struct device *dev = &qm->pdev->dev; + struct qm_shaper_factor t_factor; u32 ir = qos * QM_QOS_RATE; int ret, total_vfs, i; @@ -3685,6 +3699,7 @@ static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) if (fun_index > total_vfs) return -EINVAL; + memcpy(&t_factor, &qm->factor[fun_index], sizeof(t_factor)); qm->factor[fun_index].func_qos = qos; ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); @@ -3698,11 +3713,21 @@ static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); if (ret) { dev_err(dev, "type: %d, failed to set shaper vft!\n", i); - return -EINVAL; + goto back_func_qos; } } return 0; + +back_func_qos: + memcpy(&qm->factor[fun_index], &t_factor, sizeof(t_factor)); + for (i--; i >= ALG_TYPE_0; i--) { + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); + if (ret) + dev_err(dev, "failed to restore shaper vft during rollback!\n"); + } + + return -EINVAL; } static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) diff --git a/drivers/crypto/hisilicon/sgl.c b/drivers/crypto/hisilicon/sgl.c index 7a9ef2a9972a..24c7b6ab285b 100644 --- a/drivers/crypto/hisilicon/sgl.c +++ b/drivers/crypto/hisilicon/sgl.c @@ -245,11 +245,6 @@ hisi_acc_sg_buf_map_to_hw_sgl(struct device *dev, struct scatterlist *sgl, } curr_hw_sgl = acc_get_sgl(pool, index, &curr_sgl_dma); - if (IS_ERR(curr_hw_sgl)) { - dev_err(dev, "Get SGL error!\n"); - ret = -ENOMEM; - goto err_unmap; - } curr_hw_sgl->entry_length_in_sgl = cpu_to_le16(pool->sge_nr); curr_hw_sge = curr_hw_sgl->sge_entries; diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 23f585219fb4..d0058757b000 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -805,7 +805,7 @@ static int save_iaa_wq(struct idxd_wq *wq) if (!cpus_per_iaa) cpus_per_iaa = 1; out: - return 0; + return ret; } static void remove_iaa_wq(struct idxd_wq *wq) diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index 35679b21ff63..667d5e320f50 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -276,11 +276,11 @@ int adf_notify_fatal_error(struct adf_accel_dev *accel_dev) int adf_init_aer(void) { device_reset_wq = alloc_workqueue("qat_device_reset_wq", - WQ_MEM_RECLAIM, 0); + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!device_reset_wq) return -EFAULT; - device_sriov_wq = alloc_workqueue("qat_device_sriov_wq", 0, 0); + device_sriov_wq = alloc_workqueue("qat_device_sriov_wq", WQ_PERCPU, 0); if (!device_sriov_wq) { destroy_workqueue(device_reset_wq); device_reset_wq = NULL; diff --git a/drivers/crypto/intel/qat/qat_common/adf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_isr.c index 12e565613661..4639d7fd93e6 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_isr.c @@ -384,7 +384,8 @@ EXPORT_SYMBOL_GPL(adf_isr_resource_alloc); */ int __init adf_init_misc_wq(void) { - adf_misc_wq = alloc_workqueue("qat_misc_wq", WQ_MEM_RECLAIM, 0); + adf_misc_wq = alloc_workqueue("qat_misc_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); return !adf_misc_wq ? -ENOMEM : 0; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_sriov.c b/drivers/crypto/intel/qat/qat_common/adf_sriov.c index 31d1ef0cb1f5..bb904ba4bf84 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_sriov.c +++ b/drivers/crypto/intel/qat/qat_common/adf_sriov.c @@ -299,7 +299,8 @@ EXPORT_SYMBOL_GPL(adf_sriov_configure); int __init adf_init_pf_wq(void) { /* Workqueue for PF2VF responses */ - pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", WQ_MEM_RECLAIM, 0); + pf2vf_resp_wq = alloc_workqueue("qat_pf2vf_resp_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); return !pf2vf_resp_wq ? -ENOMEM : 0; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c index a4636ec9f9ca..d0fef20a3df4 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_vf_isr.c @@ -299,7 +299,8 @@ EXPORT_SYMBOL_GPL(adf_flush_vf_wq); */ int __init adf_init_vf_wq(void) { - adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", WQ_MEM_RECLAIM, 0); + adf_vf_stop_wq = alloc_workqueue("adf_vf_stop_wq", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); return !adf_vf_stop_wq ? -EFAULT : 0; } diff --git a/drivers/crypto/intel/qat/qat_common/qat_uclo.c b/drivers/crypto/intel/qat/qat_common/qat_uclo.c index 18c3e4416dc5..06d49cb781ae 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_uclo.c +++ b/drivers/crypto/intel/qat/qat_common/qat_uclo.c @@ -200,20 +200,12 @@ qat_uclo_cleanup_batch_init_list(struct icp_qat_fw_loader_handle *handle, static int qat_uclo_parse_num(char *str, unsigned int *num) { - char buf[16] = {0}; - unsigned long ae = 0; - int i; - - strscpy(buf, str, sizeof(buf)); - for (i = 0; i < 16; i++) { - if (!isdigit(buf[i])) { - buf[i] = '\0'; - break; - } - } - if ((kstrtoul(buf, 10, &ae))) - return -EFAULT; + unsigned long long ae; + char *end; + ae = simple_strtoull(str, &end, 10); + if (ae > UINT_MAX || str == end || (end - str) > 19) + return -EINVAL; *num = (unsigned int)ae; return 0; } diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index 9c21f5d835d2..301bdf239e7d 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -420,7 +420,6 @@ static int mv_cesa_probe(struct platform_device *pdev) { const struct mv_cesa_caps *caps = &orion_caps; const struct mbus_dram_target_info *dram; - const struct of_device_id *match; struct device *dev = &pdev->dev; struct mv_cesa_dev *cesa; struct mv_cesa_engine *engines; @@ -433,11 +432,9 @@ static int mv_cesa_probe(struct platform_device *pdev) } if (dev->of_node) { - match = of_match_node(mv_cesa_of_match_table, dev->of_node); - if (!match || !match->data) + caps = of_device_get_match_data(dev); + if (!caps) return -ENOTSUPP; - - caps = match->data; } cesa = devm_kzalloc(dev, sizeof(*cesa), GFP_KERNEL); diff --git a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c index ebdf4efa09d4..b5cc5401f704 100644 --- a/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c +++ b/drivers/crypto/marvell/octeontx2/otx2_cptpf_ucode.c @@ -3,6 +3,7 @@ #include <linux/ctype.h> #include <linux/firmware.h> +#include <linux/string.h> #include <linux/string_choices.h> #include "otx2_cptpf_ucode.h" #include "otx2_cpt_common.h" @@ -458,13 +459,13 @@ static int cpt_ucode_load_fw(struct pci_dev *pdev, struct fw_info_t *fw_info, u16 rid) { char filename[OTX2_CPT_NAME_LENGTH]; - char eng_type[8] = {0}; + char eng_type[8]; int ret, e, i; INIT_LIST_HEAD(&fw_info->ucodes); for (e = 1; e < OTX2_CPT_MAX_ENG_TYPES; e++) { - strcpy(eng_type, get_eng_type_str(e)); + strscpy(eng_type, get_eng_type_str(e)); for (i = 0; i < strlen(eng_type); i++) eng_type[i] = tolower(eng_type[i]); diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index e95e84486d9a..b966f3365b7d 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -21,7 +21,6 @@ #include "sha.h" #include "aead.h" -#define QCE_MAJOR_VERSION5 0x05 #define QCE_QUEUE_LENGTH 1 #define QCE_DEFAULT_MEM_BANDWIDTH 393600 @@ -161,7 +160,7 @@ static int qce_check_version(struct qce_device *qce) * the driver does not support v5 with minor 0 because it has special * alignment requirements. */ - if (major != QCE_MAJOR_VERSION5 || minor == 0) + if (major == 5 && minor == 0) return -ENODEV; qce->burst_size = QCE_BAM_BURST_SIZE; diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c index 1dec7aea852d..68cafd4741ad 100644 --- a/drivers/crypto/qce/dma.c +++ b/drivers/crypto/qce/dma.c @@ -24,11 +24,13 @@ int devm_qce_dma_request(struct device *dev, struct qce_dma_data *dma) dma->txchan = dma_request_chan(dev, "tx"); if (IS_ERR(dma->txchan)) - return PTR_ERR(dma->txchan); + return dev_err_probe(dev, PTR_ERR(dma->txchan), + "Failed to get TX DMA channel\n"); dma->rxchan = dma_request_chan(dev, "rx"); if (IS_ERR(dma->rxchan)) { - ret = PTR_ERR(dma->rxchan); + ret = dev_err_probe(dev, PTR_ERR(dma->rxchan), + "Failed to get RX DMA channel\n"); goto error_rx; } diff --git a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c index 9393e10671c2..e80f9148c012 100644 --- a/drivers/crypto/rockchip/rk3288_crypto_skcipher.c +++ b/drivers/crypto/rockchip/rk3288_crypto_skcipher.c @@ -321,8 +321,7 @@ static int rk_cipher_run(struct crypto_engine *engine, void *async_req) algt->stat_req++; rkc->nreq++; - ivsize = crypto_skcipher_ivsize(tfm); - if (areq->iv && crypto_skcipher_ivsize(tfm) > 0) { + if (areq->iv && ivsize > 0) { if (rctx->mode & RK_CRYPTO_DEC) { offset = areq->cryptlen - ivsize; scatterwalk_map_and_copy(rctx->backup_iv, areq->src, diff --git a/drivers/crypto/starfive/jh7110-hash.c b/drivers/crypto/starfive/jh7110-hash.c index e6839c7bfb73..54b7af4a7aee 100644 --- a/drivers/crypto/starfive/jh7110-hash.c +++ b/drivers/crypto/starfive/jh7110-hash.c @@ -325,6 +325,7 @@ static int starfive_hash_digest(struct ahash_request *req) struct starfive_cryp_ctx *ctx = crypto_ahash_ctx(tfm); struct starfive_cryp_request_ctx *rctx = ahash_request_ctx(req); struct starfive_cryp_dev *cryp = ctx->cryp; + int sg_len; memset(rctx, 0, sizeof(struct starfive_cryp_request_ctx)); @@ -333,7 +334,10 @@ static int starfive_hash_digest(struct ahash_request *req) rctx->in_sg = req->src; rctx->blksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(tfm)); rctx->digsize = crypto_ahash_digestsize(tfm); - rctx->in_sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + sg_len = sg_nents_for_len(rctx->in_sg, rctx->total); + if (sg_len < 0) + return sg_len; + rctx->in_sg_len = sg_len; ctx->rctx = rctx; return crypto_transfer_hash_request_to_engine(cryp->engine, req); diff --git a/drivers/crypto/ti/Kconfig b/drivers/crypto/ti/Kconfig index d4f91c1e0cb5..a3692ceec49b 100644 --- a/drivers/crypto/ti/Kconfig +++ b/drivers/crypto/ti/Kconfig @@ -6,6 +6,7 @@ config CRYPTO_DEV_TI_DTHEV2 select CRYPTO_SKCIPHER select CRYPTO_ECB select CRYPTO_CBC + select CRYPTO_XTS help This enables support for the TI DTHE V2 hw cryptography engine which can be found on TI K3 SOCs. Selecting this enables use diff --git a/drivers/crypto/ti/dthev2-aes.c b/drivers/crypto/ti/dthev2-aes.c index 3547c41fa4ed..156729ccc50e 100644 --- a/drivers/crypto/ti/dthev2-aes.c +++ b/drivers/crypto/ti/dthev2-aes.c @@ -25,6 +25,7 @@ // AES Engine #define DTHE_P_AES_BASE 0x7000 + #define DTHE_P_AES_KEY1_0 0x0038 #define DTHE_P_AES_KEY1_1 0x003C #define DTHE_P_AES_KEY1_2 0x0030 @@ -33,6 +34,16 @@ #define DTHE_P_AES_KEY1_5 0x002C #define DTHE_P_AES_KEY1_6 0x0020 #define DTHE_P_AES_KEY1_7 0x0024 + +#define DTHE_P_AES_KEY2_0 0x0018 +#define DTHE_P_AES_KEY2_1 0x001C +#define DTHE_P_AES_KEY2_2 0x0010 +#define DTHE_P_AES_KEY2_3 0x0014 +#define DTHE_P_AES_KEY2_4 0x0008 +#define DTHE_P_AES_KEY2_5 0x000C +#define DTHE_P_AES_KEY2_6 0x0000 +#define DTHE_P_AES_KEY2_7 0x0004 + #define DTHE_P_AES_IV_IN_0 0x0040 #define DTHE_P_AES_IV_IN_1 0x0044 #define DTHE_P_AES_IV_IN_2 0x0048 @@ -52,6 +63,7 @@ enum aes_ctrl_mode_masks { AES_CTRL_ECB_MASK = 0x00, AES_CTRL_CBC_MASK = BIT(5), + AES_CTRL_XTS_MASK = BIT(12) | BIT(11), }; #define DTHE_AES_CTRL_MODE_CLEAR_MASK ~GENMASK(28, 5) @@ -88,6 +100,31 @@ static int dthe_cipher_init_tfm(struct crypto_skcipher *tfm) return 0; } +static int dthe_cipher_xts_init_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + struct dthe_data *dev_data = dthe_get_dev(ctx); + + ctx->dev_data = dev_data; + ctx->keylen = 0; + + ctx->skcipher_fb = crypto_alloc_sync_skcipher("xts(aes)", 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(ctx->skcipher_fb)) { + dev_err(dev_data->dev, "fallback driver xts(aes) couldn't be loaded\n"); + return PTR_ERR(ctx->skcipher_fb); + } + + return 0; +} + +static void dthe_cipher_xts_exit_tfm(struct crypto_skcipher *tfm) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + crypto_free_sync_skcipher(ctx->skcipher_fb); +} + static int dthe_aes_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) { struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -119,6 +156,27 @@ static int dthe_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, unsig return dthe_aes_setkey(tfm, key, keylen); } +static int dthe_aes_xts_setkey(struct crypto_skcipher *tfm, const u8 *key, unsigned int keylen) +{ + struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); + + if (keylen != 2 * AES_KEYSIZE_128 && + keylen != 2 * AES_KEYSIZE_192 && + keylen != 2 * AES_KEYSIZE_256) + return -EINVAL; + + ctx->aes_mode = DTHE_AES_XTS; + ctx->keylen = keylen / 2; + memcpy(ctx->key, key, keylen); + + crypto_sync_skcipher_clear_flags(ctx->skcipher_fb, CRYPTO_TFM_REQ_MASK); + crypto_sync_skcipher_set_flags(ctx->skcipher_fb, + crypto_skcipher_get_flags(tfm) & + CRYPTO_TFM_REQ_MASK); + + return crypto_sync_skcipher_setkey(ctx->skcipher_fb, key, keylen); +} + static void dthe_aes_set_ctrl_key(struct dthe_tfm_ctx *ctx, struct dthe_aes_req_ctx *rctx, u32 *iv_in) @@ -141,6 +199,24 @@ static void dthe_aes_set_ctrl_key(struct dthe_tfm_ctx *ctx, writel_relaxed(ctx->key[7], aes_base_reg + DTHE_P_AES_KEY1_7); } + if (ctx->aes_mode == DTHE_AES_XTS) { + size_t key2_offset = ctx->keylen / sizeof(u32); + + writel_relaxed(ctx->key[key2_offset + 0], aes_base_reg + DTHE_P_AES_KEY2_0); + writel_relaxed(ctx->key[key2_offset + 1], aes_base_reg + DTHE_P_AES_KEY2_1); + writel_relaxed(ctx->key[key2_offset + 2], aes_base_reg + DTHE_P_AES_KEY2_2); + writel_relaxed(ctx->key[key2_offset + 3], aes_base_reg + DTHE_P_AES_KEY2_3); + + if (ctx->keylen > AES_KEYSIZE_128) { + writel_relaxed(ctx->key[key2_offset + 4], aes_base_reg + DTHE_P_AES_KEY2_4); + writel_relaxed(ctx->key[key2_offset + 5], aes_base_reg + DTHE_P_AES_KEY2_5); + } + if (ctx->keylen == AES_KEYSIZE_256) { + writel_relaxed(ctx->key[key2_offset + 6], aes_base_reg + DTHE_P_AES_KEY2_6); + writel_relaxed(ctx->key[key2_offset + 7], aes_base_reg + DTHE_P_AES_KEY2_7); + } + } + if (rctx->enc) ctrl_val |= DTHE_AES_CTRL_DIR_ENC; @@ -160,6 +236,9 @@ static void dthe_aes_set_ctrl_key(struct dthe_tfm_ctx *ctx, case DTHE_AES_CBC: ctrl_val |= AES_CTRL_CBC_MASK; break; + case DTHE_AES_XTS: + ctrl_val |= AES_CTRL_XTS_MASK; + break; } if (iv_in) { @@ -315,24 +394,45 @@ aes_err: local_bh_disable(); crypto_finalize_skcipher_request(dev_data->engine, req, ret); local_bh_enable(); - return ret; + return 0; } static int dthe_aes_crypt(struct skcipher_request *req) { struct dthe_tfm_ctx *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); + struct dthe_aes_req_ctx *rctx = skcipher_request_ctx(req); struct dthe_data *dev_data = dthe_get_dev(ctx); struct crypto_engine *engine; /* - * If data is not a multiple of AES_BLOCK_SIZE, need to return -EINVAL - * If data length input is zero, no need to do any operation. + * If data is not a multiple of AES_BLOCK_SIZE: + * - need to return -EINVAL for ECB, CBC as they are block ciphers + * - need to fallback to software as H/W doesn't support Ciphertext Stealing for XTS */ - if (req->cryptlen % AES_BLOCK_SIZE) + if (req->cryptlen % AES_BLOCK_SIZE) { + if (ctx->aes_mode == DTHE_AES_XTS) { + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, ctx->skcipher_fb); + + skcipher_request_set_callback(subreq, skcipher_request_flags(req), + req->base.complete, req->base.data); + skcipher_request_set_crypt(subreq, req->src, req->dst, + req->cryptlen, req->iv); + + return rctx->enc ? crypto_skcipher_encrypt(subreq) : + crypto_skcipher_decrypt(subreq); + } return -EINVAL; + } - if (req->cryptlen == 0) + /* + * If data length input is zero, no need to do any operation. + * Except for XTS mode, where data length should be non-zero. + */ + if (req->cryptlen == 0) { + if (ctx->aes_mode == DTHE_AES_XTS) + return -EINVAL; return 0; + } engine = dev_data->engine; return crypto_transfer_skcipher_request_to_engine(engine, req); @@ -399,7 +499,32 @@ static struct skcipher_engine_alg cipher_algs[] = { .cra_module = THIS_MODULE, }, .op.do_one_request = dthe_aes_run, - } /* CBC AES */ + }, /* CBC AES */ + { + .base.init = dthe_cipher_xts_init_tfm, + .base.exit = dthe_cipher_xts_exit_tfm, + .base.setkey = dthe_aes_xts_setkey, + .base.encrypt = dthe_aes_encrypt, + .base.decrypt = dthe_aes_decrypt, + .base.min_keysize = AES_MIN_KEY_SIZE * 2, + .base.max_keysize = AES_MAX_KEY_SIZE * 2, + .base.ivsize = AES_IV_SIZE, + .base.base = { + .cra_name = "xts(aes)", + .cra_driver_name = "xts-aes-dthev2", + .cra_priority = 299, + .cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | + CRYPTO_ALG_ASYNC | + CRYPTO_ALG_KERN_DRIVER_ONLY | + CRYPTO_ALG_NEED_FALLBACK, + .cra_alignmask = AES_BLOCK_SIZE - 1, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct dthe_tfm_ctx), + .cra_reqsize = sizeof(struct dthe_aes_req_ctx), + .cra_module = THIS_MODULE, + }, + .op.do_one_request = dthe_aes_run, + }, /* XTS AES */ }; int dthe_register_aes_algs(void) diff --git a/drivers/crypto/ti/dthev2-common.h b/drivers/crypto/ti/dthev2-common.h index 68c94acda8aa..c7a06a4c353f 100644 --- a/drivers/crypto/ti/dthev2-common.h +++ b/drivers/crypto/ti/dthev2-common.h @@ -27,10 +27,16 @@ #define DTHE_REG_SIZE 4 #define DTHE_DMA_TIMEOUT_MS 2000 +/* + * Size of largest possible key (of all algorithms) to be stored in dthe_tfm_ctx + * This is currently the keysize of XTS-AES-256 which is 512 bits (64 bytes) + */ +#define DTHE_MAX_KEYSIZE (AES_MAX_KEY_SIZE * 2) enum dthe_aes_mode { DTHE_AES_ECB = 0, DTHE_AES_CBC, + DTHE_AES_XTS, }; /* Driver specific struct definitions */ @@ -73,12 +79,14 @@ struct dthe_list { * @keylen: AES key length * @key: AES key * @aes_mode: AES mode + * @skcipher_fb: Fallback crypto skcipher handle for AES-XTS mode */ struct dthe_tfm_ctx { struct dthe_data *dev_data; unsigned int keylen; - u32 key[AES_KEYSIZE_256 / sizeof(u32)]; + u32 key[DTHE_MAX_KEYSIZE / sizeof(u32)]; enum dthe_aes_mode aes_mode; + struct crypto_sync_skcipher *skcipher_fb; }; /** diff --git a/drivers/crypto/xilinx/xilinx-trng.c b/drivers/crypto/xilinx/xilinx-trng.c index 4e4700d68127..db0fbb28ff32 100644 --- a/drivers/crypto/xilinx/xilinx-trng.c +++ b/drivers/crypto/xilinx/xilinx-trng.c @@ -8,7 +8,6 @@ #include <linux/clk.h> #include <linux/crypto.h> #include <linux/delay.h> -#include <linux/errno.h> #include <linux/firmware/xlnx-zynqmp.h> #include <linux/hw_random.h> #include <linux/io.h> @@ -18,10 +17,11 @@ #include <linux/mutex.h> #include <linux/mod_devicetable.h> #include <linux/platform_device.h> -#include <linux/string.h> +#include <crypto/aes.h> +#include <crypto/df_sp80090a.h> +#include <crypto/internal/drbg.h> #include <crypto/internal/cipher.h> #include <crypto/internal/rng.h> -#include <crypto/aes.h> /* TRNG Registers Offsets */ #define TRNG_STATUS_OFFSET 0x4U @@ -59,6 +59,8 @@ struct xilinx_rng { void __iomem *rng_base; struct device *dev; + unsigned char *scratchpadbuf; + struct crypto_aes_ctx *aesctx; struct mutex lock; /* Protect access to TRNG device */ struct hwrng trng; }; @@ -182,9 +184,13 @@ static void xtrng_enable_entropy(struct xilinx_rng *rng) static int xtrng_reseed_internal(struct xilinx_rng *rng) { u8 entropy[TRNG_ENTROPY_SEED_LEN_BYTES]; + struct drbg_string data; + LIST_HEAD(seedlist); u32 val; int ret; + drbg_string_fill(&data, entropy, TRNG_SEED_LEN_BYTES); + list_add_tail(&data.list, &seedlist); memset(entropy, 0, sizeof(entropy)); xtrng_enable_entropy(rng); @@ -192,9 +198,14 @@ static int xtrng_reseed_internal(struct xilinx_rng *rng) ret = xtrng_collect_random_data(rng, entropy, TRNG_SEED_LEN_BYTES, true); if (ret != TRNG_SEED_LEN_BYTES) return -EINVAL; + ret = crypto_drbg_ctr_df(rng->aesctx, rng->scratchpadbuf, + TRNG_SEED_LEN_BYTES, &seedlist, AES_BLOCK_SIZE, + TRNG_SEED_LEN_BYTES); + if (ret) + return ret; xtrng_write_multiple_registers(rng->rng_base + TRNG_EXT_SEED_OFFSET, - (u32 *)entropy, TRNG_NUM_INIT_REGS); + (u32 *)rng->scratchpadbuf, TRNG_NUM_INIT_REGS); /* select reseed operation */ iowrite32(TRNG_CTRL_PRNGXS_MASK, rng->rng_base + TRNG_CTRL_OFFSET); @@ -324,6 +335,7 @@ static void xtrng_hwrng_unregister(struct hwrng *trng) static int xtrng_probe(struct platform_device *pdev) { struct xilinx_rng *rng; + size_t sb_size; int ret; rng = devm_kzalloc(&pdev->dev, sizeof(*rng), GFP_KERNEL); @@ -333,15 +345,26 @@ static int xtrng_probe(struct platform_device *pdev) rng->dev = &pdev->dev; rng->rng_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(rng->rng_base)) { - dev_err(&pdev->dev, "Failed to map resource %ld\n", PTR_ERR(rng->rng_base)); + dev_err(&pdev->dev, "Failed to map resource %pe\n", rng->rng_base); return PTR_ERR(rng->rng_base); } + rng->aesctx = devm_kzalloc(&pdev->dev, sizeof(*rng->aesctx), GFP_KERNEL); + if (!rng->aesctx) + return -ENOMEM; + + sb_size = crypto_drbg_ctr_df_datalen(TRNG_SEED_LEN_BYTES, AES_BLOCK_SIZE); + rng->scratchpadbuf = devm_kzalloc(&pdev->dev, sb_size, GFP_KERNEL); + if (!rng->scratchpadbuf) { + ret = -ENOMEM; + goto end; + } + xtrng_trng_reset(rng->rng_base); ret = xtrng_reseed_internal(rng); if (ret) { dev_err(&pdev->dev, "TRNG Seed fail\n"); - return ret; + goto end; } xilinx_rng_dev = rng; @@ -349,8 +372,9 @@ static int xtrng_probe(struct platform_device *pdev) ret = crypto_register_rng(&xtrng_trng_alg); if (ret) { dev_err(&pdev->dev, "Crypto Random device registration failed: %d\n", ret); - return ret; + goto end; } + ret = xtrng_hwrng_register(&rng->trng); if (ret) { dev_err(&pdev->dev, "HWRNG device registration failed: %d\n", ret); @@ -363,6 +387,7 @@ static int xtrng_probe(struct platform_device *pdev) crypto_rng_free: crypto_unregister_rng(&xtrng_trng_alg); +end: return ret; } diff --git a/drivers/dax/super.c b/drivers/dax/super.c index d7714d8afb0f..c00b9dff4a06 100644 --- a/drivers/dax/super.c +++ b/drivers/dax/super.c @@ -433,7 +433,7 @@ static struct dax_device *dax_dev_get(dev_t devt) return NULL; dax_dev = to_dax_dev(inode); - if (inode->i_state & I_NEW) { + if (inode_state_read_once(inode) & I_NEW) { set_bit(DAXDEV_ALIVE, &dax_dev->flags); inode->i_cdev = &dax_dev->cdev; inode->i_mode = S_IFCHR; diff --git a/drivers/devfreq/devfreq.c b/drivers/devfreq/devfreq.c index 2e8d01d47f69..00979f2e0e27 100644 --- a/drivers/devfreq/devfreq.c +++ b/drivers/devfreq/devfreq.c @@ -20,6 +20,7 @@ #include <linux/stat.h> #include <linux/pm_opp.h> #include <linux/devfreq.h> +#include <linux/devfreq-governor.h> #include <linux/workqueue.h> #include <linux/platform_device.h> #include <linux/list.h> @@ -28,7 +29,6 @@ #include <linux/of.h> #include <linux/pm_qos.h> #include <linux/units.h> -#include "governor.h" #define CREATE_TRACE_POINTS #include <trace/events/devfreq.h> diff --git a/drivers/devfreq/governor.h b/drivers/devfreq/governor.h deleted file mode 100644 index 0adfebc0467a..000000000000 --- a/drivers/devfreq/governor.h +++ /dev/null @@ -1,127 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* - * governor.h - internal header for devfreq governors. - * - * Copyright (C) 2011 Samsung Electronics - * MyungJoo Ham <myungjoo.ham@samsung.com> - * - * This header is for devfreq governors in drivers/devfreq/ - */ - -#ifndef _GOVERNOR_H -#define _GOVERNOR_H - -#include <linux/devfreq.h> - -#define DEVFREQ_NAME_LEN 16 - -#define to_devfreq(DEV) container_of((DEV), struct devfreq, dev) - -/* Devfreq events */ -#define DEVFREQ_GOV_START 0x1 -#define DEVFREQ_GOV_STOP 0x2 -#define DEVFREQ_GOV_UPDATE_INTERVAL 0x3 -#define DEVFREQ_GOV_SUSPEND 0x4 -#define DEVFREQ_GOV_RESUME 0x5 - -#define DEVFREQ_MIN_FREQ 0 -#define DEVFREQ_MAX_FREQ ULONG_MAX - -/* - * Definition of the governor feature flags - * - DEVFREQ_GOV_FLAG_IMMUTABLE - * : This governor is never changeable to other governors. - * - DEVFREQ_GOV_FLAG_IRQ_DRIVEN - * : The devfreq won't schedule the work for this governor. - */ -#define DEVFREQ_GOV_FLAG_IMMUTABLE BIT(0) -#define DEVFREQ_GOV_FLAG_IRQ_DRIVEN BIT(1) - -/* - * Definition of governor attribute flags except for common sysfs attributes - * - DEVFREQ_GOV_ATTR_POLLING_INTERVAL - * : Indicate polling_interval sysfs attribute - * - DEVFREQ_GOV_ATTR_TIMER - * : Indicate timer sysfs attribute - */ -#define DEVFREQ_GOV_ATTR_POLLING_INTERVAL BIT(0) -#define DEVFREQ_GOV_ATTR_TIMER BIT(1) - -/** - * struct devfreq_cpu_data - Hold the per-cpu data - * @node: list node - * @dev: reference to cpu device. - * @first_cpu: the cpumask of the first cpu of a policy. - * @opp_table: reference to cpu opp table. - * @cur_freq: the current frequency of the cpu. - * @min_freq: the min frequency of the cpu. - * @max_freq: the max frequency of the cpu. - * - * This structure stores the required cpu_data of a cpu. - * This is auto-populated by the governor. - */ -struct devfreq_cpu_data { - struct list_head node; - - struct device *dev; - unsigned int first_cpu; - - struct opp_table *opp_table; - unsigned int cur_freq; - unsigned int min_freq; - unsigned int max_freq; -}; - -/** - * struct devfreq_governor - Devfreq policy governor - * @node: list node - contains registered devfreq governors - * @name: Governor's name - * @attrs: Governor's sysfs attribute flags - * @flags: Governor's feature flags - * @get_target_freq: Returns desired operating frequency for the device. - * Basically, get_target_freq will run - * devfreq_dev_profile.get_dev_status() to get the - * status of the device (load = busy_time / total_time). - * @event_handler: Callback for devfreq core framework to notify events - * to governors. Events include per device governor - * init and exit, opp changes out of devfreq, suspend - * and resume of per device devfreq during device idle. - * - * Note that the callbacks are called with devfreq->lock locked by devfreq. - */ -struct devfreq_governor { - struct list_head node; - - const char name[DEVFREQ_NAME_LEN]; - const u64 attrs; - const u64 flags; - int (*get_target_freq)(struct devfreq *this, unsigned long *freq); - int (*event_handler)(struct devfreq *devfreq, - unsigned int event, void *data); -}; - -void devfreq_monitor_start(struct devfreq *devfreq); -void devfreq_monitor_stop(struct devfreq *devfreq); -void devfreq_monitor_suspend(struct devfreq *devfreq); -void devfreq_monitor_resume(struct devfreq *devfreq); -void devfreq_update_interval(struct devfreq *devfreq, unsigned int *delay); - -int devfreq_add_governor(struct devfreq_governor *governor); -int devfreq_remove_governor(struct devfreq_governor *governor); - -int devm_devfreq_add_governor(struct device *dev, - struct devfreq_governor *governor); - -int devfreq_update_status(struct devfreq *devfreq, unsigned long freq); -int devfreq_update_target(struct devfreq *devfreq, unsigned long freq); -void devfreq_get_freq_range(struct devfreq *devfreq, unsigned long *min_freq, - unsigned long *max_freq); - -static inline int devfreq_update_stats(struct devfreq *df) -{ - if (!df->profile->get_dev_status) - return -EINVAL; - - return df->profile->get_dev_status(df->dev.parent, &df->last_status); -} -#endif /* _GOVERNOR_H */ diff --git a/drivers/devfreq/governor_passive.c b/drivers/devfreq/governor_passive.c index 953cf9a1e9f7..8cd6f9a59f64 100644 --- a/drivers/devfreq/governor_passive.c +++ b/drivers/devfreq/governor_passive.c @@ -14,8 +14,33 @@ #include <linux/slab.h> #include <linux/device.h> #include <linux/devfreq.h> +#include <linux/devfreq-governor.h> #include <linux/units.h> -#include "governor.h" + +/** + * struct devfreq_cpu_data - Hold the per-cpu data + * @node: list node + * @dev: reference to cpu device. + * @first_cpu: the cpumask of the first cpu of a policy. + * @opp_table: reference to cpu opp table. + * @cur_freq: the current frequency of the cpu. + * @min_freq: the min frequency of the cpu. + * @max_freq: the max frequency of the cpu. + * + * This structure stores the required cpu_data of a cpu. + * This is auto-populated by the governor. + */ +struct devfreq_cpu_data { + struct list_head node; + + struct device *dev; + unsigned int first_cpu; + + struct opp_table *opp_table; + unsigned int cur_freq; + unsigned int min_freq; + unsigned int max_freq; +}; static struct devfreq_cpu_data * get_parent_cpu_data(struct devfreq_passive_data *p_data, diff --git a/drivers/devfreq/governor_performance.c b/drivers/devfreq/governor_performance.c index 2e4e981446fa..fdb22bf512cf 100644 --- a/drivers/devfreq/governor_performance.c +++ b/drivers/devfreq/governor_performance.c @@ -7,8 +7,8 @@ */ #include <linux/devfreq.h> +#include <linux/devfreq-governor.h> #include <linux/module.h> -#include "governor.h" static int devfreq_performance_func(struct devfreq *df, unsigned long *freq) diff --git a/drivers/devfreq/governor_powersave.c b/drivers/devfreq/governor_powersave.c index f059e8814804..ee2d6ec8a512 100644 --- a/drivers/devfreq/governor_powersave.c +++ b/drivers/devfreq/governor_powersave.c @@ -7,8 +7,8 @@ */ #include <linux/devfreq.h> +#include <linux/devfreq-governor.h> #include <linux/module.h> -#include "governor.h" static int devfreq_powersave_func(struct devfreq *df, unsigned long *freq) diff --git a/drivers/devfreq/governor_simpleondemand.c b/drivers/devfreq/governor_simpleondemand.c index c23435736367..ac9c5e9e51a4 100644 --- a/drivers/devfreq/governor_simpleondemand.c +++ b/drivers/devfreq/governor_simpleondemand.c @@ -9,12 +9,12 @@ #include <linux/errno.h> #include <linux/module.h> #include <linux/devfreq.h> +#include <linux/devfreq-governor.h> #include <linux/math64.h> -#include "governor.h" /* Default constants for DevFreq-Simple-Ondemand (DFSO) */ #define DFSO_UPTHRESHOLD (90) -#define DFSO_DOWNDIFFERENCTIAL (5) +#define DFSO_DOWNDIFFERENTIAL (5) static int devfreq_simple_ondemand_func(struct devfreq *df, unsigned long *freq) { @@ -22,7 +22,7 @@ static int devfreq_simple_ondemand_func(struct devfreq *df, struct devfreq_dev_status *stat; unsigned long long a, b; unsigned int dfso_upthreshold = DFSO_UPTHRESHOLD; - unsigned int dfso_downdifferential = DFSO_DOWNDIFFERENCTIAL; + unsigned int dfso_downdifferential = DFSO_DOWNDIFFERENTIAL; struct devfreq_simple_ondemand_data *data = df->data; err = devfreq_update_stats(df); diff --git a/drivers/devfreq/governor_userspace.c b/drivers/devfreq/governor_userspace.c index 175de0c0b50e..395174f93960 100644 --- a/drivers/devfreq/governor_userspace.c +++ b/drivers/devfreq/governor_userspace.c @@ -9,11 +9,11 @@ #include <linux/slab.h> #include <linux/device.h> #include <linux/devfreq.h> +#include <linux/devfreq-governor.h> #include <linux/kstrtox.h> #include <linux/pm.h> #include <linux/mutex.h> #include <linux/module.h> -#include "governor.h" struct userspace_data { unsigned long user_frequency; diff --git a/drivers/devfreq/hisi_uncore_freq.c b/drivers/devfreq/hisi_uncore_freq.c index 96d1815059e3..4d00d813c8ac 100644 --- a/drivers/devfreq/hisi_uncore_freq.c +++ b/drivers/devfreq/hisi_uncore_freq.c @@ -9,6 +9,7 @@ #include <linux/bits.h> #include <linux/cleanup.h> #include <linux/devfreq.h> +#include <linux/devfreq-governor.h> #include <linux/device.h> #include <linux/dev_printk.h> #include <linux/errno.h> @@ -26,8 +27,6 @@ #include <linux/units.h> #include <acpi/pcc.h> -#include "governor.h" - struct hisi_uncore_pcc_data { u16 status; u16 resv; @@ -265,10 +264,11 @@ static int hisi_uncore_target(struct device *dev, unsigned long *freq, dev_err(dev, "Failed to get opp for freq %lu hz\n", *freq); return PTR_ERR(opp); } - dev_pm_opp_put(opp); data = (u32)(dev_pm_opp_get_freq(opp) / HZ_PER_MHZ); + dev_pm_opp_put(opp); + return hisi_uncore_cmd_send(uncore, HUCF_PCC_CMD_SET_FREQ, &data); } diff --git a/drivers/devfreq/tegra30-devfreq.c b/drivers/devfreq/tegra30-devfreq.c index 4a4f0106ab9d..8b57194ac698 100644 --- a/drivers/devfreq/tegra30-devfreq.c +++ b/drivers/devfreq/tegra30-devfreq.c @@ -9,9 +9,11 @@ #include <linux/clk.h> #include <linux/cpufreq.h> #include <linux/devfreq.h> +#include <linux/devfreq-governor.h> #include <linux/interrupt.h> #include <linux/io.h> #include <linux/irq.h> +#include <linux/minmax.h> #include <linux/module.h> #include <linux/of.h> #include <linux/platform_device.h> @@ -21,8 +23,6 @@ #include <soc/tegra/fuse.h> -#include "governor.h" - #define ACTMON_GLB_STATUS 0x0 #define ACTMON_GLB_PERIOD_CTRL 0x4 @@ -326,14 +326,9 @@ static unsigned long actmon_cpu_to_emc_rate(struct tegra_devfreq *tegra, unsigned int i; const struct tegra_actmon_emc_ratio *ratio = actmon_emc_ratios; - for (i = 0; i < ARRAY_SIZE(actmon_emc_ratios); i++, ratio++) { - if (cpu_freq >= ratio->cpu_freq) { - if (ratio->emc_freq >= tegra->max_freq) - return tegra->max_freq; - else - return ratio->emc_freq; - } - } + for (i = 0; i < ARRAY_SIZE(actmon_emc_ratios); i++, ratio++) + if (cpu_freq >= ratio->cpu_freq) + return min(ratio->emc_freq, tegra->max_freq); return 0; } diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c index 2bcf9ceca997..edaa9e4ee4ae 100644 --- a/drivers/dma-buf/dma-buf.c +++ b/drivers/dma-buf/dma-buf.c @@ -768,18 +768,10 @@ EXPORT_SYMBOL_NS_GPL(dma_buf_export, "DMA_BUF"); */ int dma_buf_fd(struct dma_buf *dmabuf, int flags) { - int fd; - if (!dmabuf || !dmabuf->file) return -EINVAL; - fd = get_unused_fd_flags(flags); - if (fd < 0) - return fd; - - fd_install(fd, dmabuf->file); - - return fd; + return FD_ADD(flags, dmabuf->file); } EXPORT_SYMBOL_NS_GPL(dma_buf_fd, "DMA_BUF"); diff --git a/drivers/dma-buf/sync_debug.c b/drivers/dma-buf/sync_debug.c index 67cd69551e42..9e5d662cd4e8 100644 --- a/drivers/dma-buf/sync_debug.c +++ b/drivers/dma-buf/sync_debug.c @@ -59,7 +59,7 @@ static void sync_print_fence(struct seq_file *s, struct timespec64 ts64 = ktime_to_timespec64(fence->timestamp); - seq_printf(s, "@%lld.%09ld", (s64)ts64.tv_sec, ts64.tv_nsec); + seq_printf(s, "@%ptSp", &ts64); } seq_printf(s, ": %lld", fence->seqno); diff --git a/drivers/edac/Kconfig b/drivers/edac/Kconfig index 39352b9b7a7e..81e40543ffd8 100644 --- a/drivers/edac/Kconfig +++ b/drivers/edac/Kconfig @@ -23,14 +23,6 @@ menuconfig EDAC if EDAC -config EDAC_LEGACY_SYSFS - bool "EDAC legacy sysfs" - default y - help - Enable the compatibility sysfs nodes. - Use 'Y' if your edac utilities aren't ported to work with the newer - structures. - config EDAC_DEBUG bool "Debugging" select DEBUG_FS @@ -291,6 +283,18 @@ config EDAC_I10NM system has non-volatile DIMMs you should also manually select CONFIG_ACPI_NFIT. +config EDAC_IMH + tristate "Intel Integrated Memory/IO Hub MC" + depends on X86_64 && X86_MCE_INTEL && ACPI + depends on ACPI_NFIT || !ACPI_NFIT # if ACPI_NFIT=m, EDAC_IMH can't be y + select DMI + select ACPI_ADXL + help + Support for error detection and correction the Intel + Integrated Memory/IO Hub Memory Controller. This MC IP is + first used on the Diamond Rapids servers but may appear on + others in the future. + config EDAC_PND2 tristate "Intel Pondicherry2" depends on PCI && X86_64 && X86_MCE_INTEL diff --git a/drivers/edac/Makefile b/drivers/edac/Makefile index 1c14796410a3..8429b1e856bc 100644 --- a/drivers/edac/Makefile +++ b/drivers/edac/Makefile @@ -65,6 +65,9 @@ obj-$(CONFIG_EDAC_SKX) += skx_edac.o skx_edac_common.o i10nm_edac-y := i10nm_base.o obj-$(CONFIG_EDAC_I10NM) += i10nm_edac.o skx_edac_common.o +imh_edac-y := imh_base.o +obj-$(CONFIG_EDAC_IMH) += imh_edac.o skx_edac_common.o + obj-$(CONFIG_EDAC_HIGHBANK_MC) += highbank_mc_edac.o obj-$(CONFIG_EDAC_HIGHBANK_L2) += highbank_l2_edac.o diff --git a/drivers/edac/amd64_edac.c b/drivers/edac/amd64_edac.c index 2f6ab783bf20..2391f3469961 100644 --- a/drivers/edac/amd64_edac.c +++ b/drivers/edac/amd64_edac.c @@ -3732,6 +3732,7 @@ static void hw_info_put(struct amd64_pvt *pvt) pci_dev_put(pvt->F1); pci_dev_put(pvt->F2); kfree(pvt->umc); + kfree(pvt->csels); } static struct low_ops umc_ops = { @@ -3766,6 +3767,7 @@ static int per_family_init(struct amd64_pvt *pvt) pvt->stepping = boot_cpu_data.x86_stepping; pvt->model = boot_cpu_data.x86_model; pvt->fam = boot_cpu_data.x86; + char *tmp_name = NULL; pvt->max_mcs = 2; /* @@ -3779,7 +3781,7 @@ static int per_family_init(struct amd64_pvt *pvt) switch (pvt->fam) { case 0xf: - pvt->ctl_name = (pvt->ext_model >= K8_REV_F) ? + tmp_name = (pvt->ext_model >= K8_REV_F) ? "K8 revF or later" : "K8 revE or earlier"; pvt->f1_id = PCI_DEVICE_ID_AMD_K8_NB_ADDRMAP; pvt->f2_id = PCI_DEVICE_ID_AMD_K8_NB_MEMCTL; @@ -3788,7 +3790,6 @@ static int per_family_init(struct amd64_pvt *pvt) break; case 0x10: - pvt->ctl_name = "F10h"; pvt->f1_id = PCI_DEVICE_ID_AMD_10H_NB_MAP; pvt->f2_id = PCI_DEVICE_ID_AMD_10H_NB_DRAM; pvt->ops->dbam_to_cs = f10_dbam_to_chip_select; @@ -3797,12 +3798,10 @@ static int per_family_init(struct amd64_pvt *pvt) case 0x15: switch (pvt->model) { case 0x30: - pvt->ctl_name = "F15h_M30h"; pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F1; pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M30H_NB_F2; break; case 0x60: - pvt->ctl_name = "F15h_M60h"; pvt->f1_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F1; pvt->f2_id = PCI_DEVICE_ID_AMD_15H_M60H_NB_F2; pvt->ops->dbam_to_cs = f15_m60h_dbam_to_chip_select; @@ -3811,7 +3810,6 @@ static int per_family_init(struct amd64_pvt *pvt) /* Richland is only client */ return -ENODEV; default: - pvt->ctl_name = "F15h"; pvt->f1_id = PCI_DEVICE_ID_AMD_15H_NB_F1; pvt->f2_id = PCI_DEVICE_ID_AMD_15H_NB_F2; pvt->ops->dbam_to_cs = f15_dbam_to_chip_select; @@ -3822,12 +3820,10 @@ static int per_family_init(struct amd64_pvt *pvt) case 0x16: switch (pvt->model) { case 0x30: - pvt->ctl_name = "F16h_M30h"; pvt->f1_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F1; pvt->f2_id = PCI_DEVICE_ID_AMD_16H_M30H_NB_F2; break; default: - pvt->ctl_name = "F16h"; pvt->f1_id = PCI_DEVICE_ID_AMD_16H_NB_F1; pvt->f2_id = PCI_DEVICE_ID_AMD_16H_NB_F2; break; @@ -3836,76 +3832,51 @@ static int per_family_init(struct amd64_pvt *pvt) case 0x17: switch (pvt->model) { - case 0x10 ... 0x2f: - pvt->ctl_name = "F17h_M10h"; - break; case 0x30 ... 0x3f: - pvt->ctl_name = "F17h_M30h"; pvt->max_mcs = 8; break; - case 0x60 ... 0x6f: - pvt->ctl_name = "F17h_M60h"; - break; - case 0x70 ... 0x7f: - pvt->ctl_name = "F17h_M70h"; - break; default: - pvt->ctl_name = "F17h"; break; } break; case 0x18: - pvt->ctl_name = "F18h"; break; case 0x19: switch (pvt->model) { case 0x00 ... 0x0f: - pvt->ctl_name = "F19h"; pvt->max_mcs = 8; break; case 0x10 ... 0x1f: - pvt->ctl_name = "F19h_M10h"; pvt->max_mcs = 12; pvt->flags.zn_regs_v2 = 1; break; - case 0x20 ... 0x2f: - pvt->ctl_name = "F19h_M20h"; - break; case 0x30 ... 0x3f: if (pvt->F3->device == PCI_DEVICE_ID_AMD_MI200_DF_F3) { - pvt->ctl_name = "MI200"; + tmp_name = "MI200"; pvt->max_mcs = 4; pvt->dram_type = MEM_HBM2; pvt->gpu_umc_base = 0x50000; pvt->ops = &gpu_ops; } else { - pvt->ctl_name = "F19h_M30h"; pvt->max_mcs = 8; } break; - case 0x50 ... 0x5f: - pvt->ctl_name = "F19h_M50h"; - break; case 0x60 ... 0x6f: - pvt->ctl_name = "F19h_M60h"; pvt->flags.zn_regs_v2 = 1; break; case 0x70 ... 0x7f: - pvt->ctl_name = "F19h_M70h"; pvt->max_mcs = 4; pvt->flags.zn_regs_v2 = 1; break; case 0x90 ... 0x9f: - pvt->ctl_name = "F19h_M90h"; pvt->max_mcs = 4; pvt->dram_type = MEM_HBM3; pvt->gpu_umc_base = 0x90000; pvt->ops = &gpu_ops; break; case 0xa0 ... 0xaf: - pvt->ctl_name = "F19h_MA0h"; pvt->max_mcs = 12; pvt->flags.zn_regs_v2 = 1; break; @@ -3915,34 +3886,22 @@ static int per_family_init(struct amd64_pvt *pvt) case 0x1A: switch (pvt->model) { case 0x00 ... 0x1f: - pvt->ctl_name = "F1Ah"; pvt->max_mcs = 12; pvt->flags.zn_regs_v2 = 1; break; case 0x40 ... 0x4f: - pvt->ctl_name = "F1Ah_M40h"; pvt->flags.zn_regs_v2 = 1; break; case 0x50 ... 0x57: - pvt->ctl_name = "F1Ah_M50h"; + case 0xc0 ... 0xc7: pvt->max_mcs = 16; pvt->flags.zn_regs_v2 = 1; break; case 0x90 ... 0x9f: - pvt->ctl_name = "F1Ah_M90h"; - pvt->max_mcs = 8; - pvt->flags.zn_regs_v2 = 1; - break; case 0xa0 ... 0xaf: - pvt->ctl_name = "F1Ah_MA0h"; pvt->max_mcs = 8; pvt->flags.zn_regs_v2 = 1; break; - case 0xc0 ... 0xc7: - pvt->ctl_name = "F1Ah_MC0h"; - pvt->max_mcs = 16; - pvt->flags.zn_regs_v2 = 1; - break; } break; @@ -3951,6 +3910,16 @@ static int per_family_init(struct amd64_pvt *pvt) return -ENODEV; } + if (tmp_name) + scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), tmp_name); + else + scnprintf(pvt->ctl_name, sizeof(pvt->ctl_name), "F%02Xh_M%02Xh", + pvt->fam, pvt->model); + + pvt->csels = kcalloc(pvt->max_mcs, sizeof(*pvt->csels), GFP_KERNEL); + if (!pvt->csels) + return -ENOMEM; + return 0; } diff --git a/drivers/edac/amd64_edac.h b/drivers/edac/amd64_edac.h index d70b8a8d0b09..1757c1b99fc8 100644 --- a/drivers/edac/amd64_edac.h +++ b/drivers/edac/amd64_edac.h @@ -96,11 +96,12 @@ /* Hardware limit on ChipSelect rows per MC and processors per system */ #define NUM_CHIPSELECTS 8 #define DRAM_RANGES 8 -#define NUM_CONTROLLERS 16 #define ON true #define OFF false +#define MAX_CTL_NAMELEN 19 + /* * PCI-defined configuration space registers */ @@ -346,7 +347,7 @@ struct amd64_pvt { u32 dbam1; /* DRAM Base Address Mapping reg for DCT1 */ /* one for each DCT/UMC */ - struct chip_select csels[NUM_CONTROLLERS]; + struct chip_select *csels; /* DRAM base and limit pairs F1x[78,70,68,60,58,50,48,40] */ struct dram_range ranges[DRAM_RANGES]; @@ -362,7 +363,7 @@ struct amd64_pvt { /* x4, x8, or x16 syndromes in use */ u8 ecc_sym_sz; - const char *ctl_name; + char ctl_name[MAX_CTL_NAMELEN]; u16 f1_id, f2_id; /* Maximum number of memory controllers per die/node. */ u8 max_mcs; diff --git a/drivers/edac/edac_mc_sysfs.c b/drivers/edac/edac_mc_sysfs.c index 8689631f1905..091cc6aae8a9 100644 --- a/drivers/edac/edac_mc_sysfs.c +++ b/drivers/edac/edac_mc_sysfs.c @@ -115,401 +115,6 @@ static const char * const edac_caps[] = { [EDAC_S16ECD16ED] = "S16ECD16ED" }; -#ifdef CONFIG_EDAC_LEGACY_SYSFS -/* - * EDAC sysfs CSROW data structures and methods - */ - -#define to_csrow(k) container_of(k, struct csrow_info, dev) - -/* - * We need it to avoid namespace conflicts between the legacy API - * and the per-dimm/per-rank one - */ -#define DEVICE_ATTR_LEGACY(_name, _mode, _show, _store) \ - static struct device_attribute dev_attr_legacy_##_name = __ATTR(_name, _mode, _show, _store) - -struct dev_ch_attribute { - struct device_attribute attr; - unsigned int channel; -}; - -#define DEVICE_CHANNEL(_name, _mode, _show, _store, _var) \ - static struct dev_ch_attribute dev_attr_legacy_##_name = \ - { __ATTR(_name, _mode, _show, _store), (_var) } - -#define to_channel(k) (container_of(k, struct dev_ch_attribute, attr)->channel) - -/* Set of more default csrow<id> attribute show/store functions */ -static ssize_t csrow_ue_count_show(struct device *dev, - struct device_attribute *mattr, char *data) -{ - struct csrow_info *csrow = to_csrow(dev); - - return sysfs_emit(data, "%u\n", csrow->ue_count); -} - -static ssize_t csrow_ce_count_show(struct device *dev, - struct device_attribute *mattr, char *data) -{ - struct csrow_info *csrow = to_csrow(dev); - - return sysfs_emit(data, "%u\n", csrow->ce_count); -} - -static ssize_t csrow_size_show(struct device *dev, - struct device_attribute *mattr, char *data) -{ - struct csrow_info *csrow = to_csrow(dev); - int i; - u32 nr_pages = 0; - - for (i = 0; i < csrow->nr_channels; i++) - nr_pages += csrow->channels[i]->dimm->nr_pages; - return sysfs_emit(data, "%u\n", PAGES_TO_MiB(nr_pages)); -} - -static ssize_t csrow_mem_type_show(struct device *dev, - struct device_attribute *mattr, char *data) -{ - struct csrow_info *csrow = to_csrow(dev); - - return sysfs_emit(data, "%s\n", edac_mem_types[csrow->channels[0]->dimm->mtype]); -} - -static ssize_t csrow_dev_type_show(struct device *dev, - struct device_attribute *mattr, char *data) -{ - struct csrow_info *csrow = to_csrow(dev); - - return sysfs_emit(data, "%s\n", dev_types[csrow->channels[0]->dimm->dtype]); -} - -static ssize_t csrow_edac_mode_show(struct device *dev, - struct device_attribute *mattr, - char *data) -{ - struct csrow_info *csrow = to_csrow(dev); - - return sysfs_emit(data, "%s\n", edac_caps[csrow->channels[0]->dimm->edac_mode]); -} - -/* show/store functions for DIMM Label attributes */ -static ssize_t channel_dimm_label_show(struct device *dev, - struct device_attribute *mattr, - char *data) -{ - struct csrow_info *csrow = to_csrow(dev); - unsigned int chan = to_channel(mattr); - struct rank_info *rank = csrow->channels[chan]; - - /* if field has not been initialized, there is nothing to send */ - if (!rank->dimm->label[0]) - return 0; - - return sysfs_emit(data, "%s\n", rank->dimm->label); -} - -static ssize_t channel_dimm_label_store(struct device *dev, - struct device_attribute *mattr, - const char *data, size_t count) -{ - struct csrow_info *csrow = to_csrow(dev); - unsigned int chan = to_channel(mattr); - struct rank_info *rank = csrow->channels[chan]; - size_t copy_count = count; - - if (count == 0) - return -EINVAL; - - if (data[count - 1] == '\0' || data[count - 1] == '\n') - copy_count -= 1; - - if (copy_count == 0 || copy_count >= sizeof(rank->dimm->label)) - return -EINVAL; - - memcpy(rank->dimm->label, data, copy_count); - rank->dimm->label[copy_count] = '\0'; - - return count; -} - -/* show function for dynamic chX_ce_count attribute */ -static ssize_t channel_ce_count_show(struct device *dev, - struct device_attribute *mattr, char *data) -{ - struct csrow_info *csrow = to_csrow(dev); - unsigned int chan = to_channel(mattr); - struct rank_info *rank = csrow->channels[chan]; - - return sysfs_emit(data, "%u\n", rank->ce_count); -} - -/* cwrow<id>/attribute files */ -DEVICE_ATTR_LEGACY(size_mb, S_IRUGO, csrow_size_show, NULL); -DEVICE_ATTR_LEGACY(dev_type, S_IRUGO, csrow_dev_type_show, NULL); -DEVICE_ATTR_LEGACY(mem_type, S_IRUGO, csrow_mem_type_show, NULL); -DEVICE_ATTR_LEGACY(edac_mode, S_IRUGO, csrow_edac_mode_show, NULL); -DEVICE_ATTR_LEGACY(ue_count, S_IRUGO, csrow_ue_count_show, NULL); -DEVICE_ATTR_LEGACY(ce_count, S_IRUGO, csrow_ce_count_show, NULL); - -/* default attributes of the CSROW<id> object */ -static struct attribute *csrow_attrs[] = { - &dev_attr_legacy_dev_type.attr, - &dev_attr_legacy_mem_type.attr, - &dev_attr_legacy_edac_mode.attr, - &dev_attr_legacy_size_mb.attr, - &dev_attr_legacy_ue_count.attr, - &dev_attr_legacy_ce_count.attr, - NULL, -}; - -static const struct attribute_group csrow_attr_grp = { - .attrs = csrow_attrs, -}; - -static const struct attribute_group *csrow_attr_groups[] = { - &csrow_attr_grp, - NULL -}; - -static const struct device_type csrow_attr_type = { - .groups = csrow_attr_groups, -}; - -/* - * possible dynamic channel DIMM Label attribute files - * - */ -DEVICE_CHANNEL(ch0_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 0); -DEVICE_CHANNEL(ch1_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 1); -DEVICE_CHANNEL(ch2_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 2); -DEVICE_CHANNEL(ch3_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 3); -DEVICE_CHANNEL(ch4_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 4); -DEVICE_CHANNEL(ch5_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 5); -DEVICE_CHANNEL(ch6_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 6); -DEVICE_CHANNEL(ch7_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 7); -DEVICE_CHANNEL(ch8_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 8); -DEVICE_CHANNEL(ch9_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 9); -DEVICE_CHANNEL(ch10_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 10); -DEVICE_CHANNEL(ch11_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 11); -DEVICE_CHANNEL(ch12_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 12); -DEVICE_CHANNEL(ch13_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 13); -DEVICE_CHANNEL(ch14_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 14); -DEVICE_CHANNEL(ch15_dimm_label, S_IRUGO | S_IWUSR, - channel_dimm_label_show, channel_dimm_label_store, 15); - -/* Total possible dynamic DIMM Label attribute file table */ -static struct attribute *dynamic_csrow_dimm_attr[] = { - &dev_attr_legacy_ch0_dimm_label.attr.attr, - &dev_attr_legacy_ch1_dimm_label.attr.attr, - &dev_attr_legacy_ch2_dimm_label.attr.attr, - &dev_attr_legacy_ch3_dimm_label.attr.attr, - &dev_attr_legacy_ch4_dimm_label.attr.attr, - &dev_attr_legacy_ch5_dimm_label.attr.attr, - &dev_attr_legacy_ch6_dimm_label.attr.attr, - &dev_attr_legacy_ch7_dimm_label.attr.attr, - &dev_attr_legacy_ch8_dimm_label.attr.attr, - &dev_attr_legacy_ch9_dimm_label.attr.attr, - &dev_attr_legacy_ch10_dimm_label.attr.attr, - &dev_attr_legacy_ch11_dimm_label.attr.attr, - &dev_attr_legacy_ch12_dimm_label.attr.attr, - &dev_attr_legacy_ch13_dimm_label.attr.attr, - &dev_attr_legacy_ch14_dimm_label.attr.attr, - &dev_attr_legacy_ch15_dimm_label.attr.attr, - NULL -}; - -/* possible dynamic channel ce_count attribute files */ -DEVICE_CHANNEL(ch0_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 0); -DEVICE_CHANNEL(ch1_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 1); -DEVICE_CHANNEL(ch2_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 2); -DEVICE_CHANNEL(ch3_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 3); -DEVICE_CHANNEL(ch4_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 4); -DEVICE_CHANNEL(ch5_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 5); -DEVICE_CHANNEL(ch6_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 6); -DEVICE_CHANNEL(ch7_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 7); -DEVICE_CHANNEL(ch8_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 8); -DEVICE_CHANNEL(ch9_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 9); -DEVICE_CHANNEL(ch10_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 10); -DEVICE_CHANNEL(ch11_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 11); -DEVICE_CHANNEL(ch12_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 12); -DEVICE_CHANNEL(ch13_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 13); -DEVICE_CHANNEL(ch14_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 14); -DEVICE_CHANNEL(ch15_ce_count, S_IRUGO, - channel_ce_count_show, NULL, 15); - -/* Total possible dynamic ce_count attribute file table */ -static struct attribute *dynamic_csrow_ce_count_attr[] = { - &dev_attr_legacy_ch0_ce_count.attr.attr, - &dev_attr_legacy_ch1_ce_count.attr.attr, - &dev_attr_legacy_ch2_ce_count.attr.attr, - &dev_attr_legacy_ch3_ce_count.attr.attr, - &dev_attr_legacy_ch4_ce_count.attr.attr, - &dev_attr_legacy_ch5_ce_count.attr.attr, - &dev_attr_legacy_ch6_ce_count.attr.attr, - &dev_attr_legacy_ch7_ce_count.attr.attr, - &dev_attr_legacy_ch8_ce_count.attr.attr, - &dev_attr_legacy_ch9_ce_count.attr.attr, - &dev_attr_legacy_ch10_ce_count.attr.attr, - &dev_attr_legacy_ch11_ce_count.attr.attr, - &dev_attr_legacy_ch12_ce_count.attr.attr, - &dev_attr_legacy_ch13_ce_count.attr.attr, - &dev_attr_legacy_ch14_ce_count.attr.attr, - &dev_attr_legacy_ch15_ce_count.attr.attr, - NULL -}; - -static umode_t csrow_dev_is_visible(struct kobject *kobj, - struct attribute *attr, int idx) -{ - struct device *dev = kobj_to_dev(kobj); - struct csrow_info *csrow = container_of(dev, struct csrow_info, dev); - - if (idx >= csrow->nr_channels) - return 0; - - if (idx >= ARRAY_SIZE(dynamic_csrow_ce_count_attr) - 1) { - WARN_ONCE(1, "idx: %d\n", idx); - return 0; - } - - /* Only expose populated DIMMs */ - if (!csrow->channels[idx]->dimm->nr_pages) - return 0; - - return attr->mode; -} - - -static const struct attribute_group csrow_dev_dimm_group = { - .attrs = dynamic_csrow_dimm_attr, - .is_visible = csrow_dev_is_visible, -}; - -static const struct attribute_group csrow_dev_ce_count_group = { - .attrs = dynamic_csrow_ce_count_attr, - .is_visible = csrow_dev_is_visible, -}; - -static const struct attribute_group *csrow_dev_groups[] = { - &csrow_dev_dimm_group, - &csrow_dev_ce_count_group, - NULL -}; - -static void csrow_release(struct device *dev) -{ - /* - * Nothing to do, just unregister sysfs here. The mci - * device owns the data and will also release it. - */ -} - -static inline int nr_pages_per_csrow(struct csrow_info *csrow) -{ - int chan, nr_pages = 0; - - for (chan = 0; chan < csrow->nr_channels; chan++) - nr_pages += csrow->channels[chan]->dimm->nr_pages; - - return nr_pages; -} - -/* Create a CSROW object under specified edac_mc_device */ -static int edac_create_csrow_object(struct mem_ctl_info *mci, - struct csrow_info *csrow, int index) -{ - int err; - - csrow->dev.type = &csrow_attr_type; - csrow->dev.groups = csrow_dev_groups; - csrow->dev.release = csrow_release; - device_initialize(&csrow->dev); - csrow->dev.parent = &mci->dev; - csrow->mci = mci; - dev_set_name(&csrow->dev, "csrow%d", index); - dev_set_drvdata(&csrow->dev, csrow); - - err = device_add(&csrow->dev); - if (err) { - edac_dbg(1, "failure: create device %s\n", dev_name(&csrow->dev)); - put_device(&csrow->dev); - return err; - } - - edac_dbg(0, "device %s created\n", dev_name(&csrow->dev)); - - return 0; -} - -/* Create a CSROW object under specified edac_mc_device */ -static int edac_create_csrow_objects(struct mem_ctl_info *mci) -{ - int err, i; - struct csrow_info *csrow; - - for (i = 0; i < mci->nr_csrows; i++) { - csrow = mci->csrows[i]; - if (!nr_pages_per_csrow(csrow)) - continue; - err = edac_create_csrow_object(mci, mci->csrows[i], i); - if (err < 0) - goto error; - } - return 0; - -error: - for (--i; i >= 0; i--) { - if (device_is_registered(&mci->csrows[i]->dev)) - device_unregister(&mci->csrows[i]->dev); - } - - return err; -} - -static void edac_delete_csrow_objects(struct mem_ctl_info *mci) -{ - int i; - - for (i = 0; i < mci->nr_csrows; i++) { - if (device_is_registered(&mci->csrows[i]->dev)) - device_unregister(&mci->csrows[i]->dev); - } -} - -#endif - /* * Per-dimm (or per-rank) devices */ @@ -989,12 +594,6 @@ int edac_create_sysfs_mci_device(struct mem_ctl_info *mci, goto fail; } -#ifdef CONFIG_EDAC_LEGACY_SYSFS - err = edac_create_csrow_objects(mci); - if (err < 0) - goto fail; -#endif - edac_create_debugfs_nodes(mci); return 0; @@ -1019,9 +618,6 @@ void edac_remove_sysfs_mci_device(struct mem_ctl_info *mci) #ifdef CONFIG_EDAC_DEBUG edac_debugfs_remove_recursive(mci->debugfs); #endif -#ifdef CONFIG_EDAC_LEGACY_SYSFS - edac_delete_csrow_objects(mci); -#endif mci_for_each_dimm(mci, dimm) { if (!device_is_registered(&dimm->dev)) diff --git a/drivers/edac/ghes_edac.c b/drivers/edac/ghes_edac.c index 1eb0136c6fbd..d80c88818691 100644 --- a/drivers/edac/ghes_edac.c +++ b/drivers/edac/ghes_edac.c @@ -15,6 +15,7 @@ #include "edac_module.h" #include <ras/ras_event.h> #include <linux/notifier.h> +#include <linux/string.h> #define OTHER_DETAIL_LEN 400 @@ -332,7 +333,7 @@ static int ghes_edac_report_mem_error(struct notifier_block *nb, p = pvt->msg; p += snprintf(p, sizeof(pvt->msg), "%s", cper_mem_err_type_str(etype)); } else { - strcpy(pvt->msg, "unknown error"); + strscpy(pvt->msg, "unknown error"); } /* Error address */ @@ -357,14 +358,14 @@ static int ghes_edac_report_mem_error(struct notifier_block *nb, dimm = find_dimm_by_handle(mci, mem_err->mem_dev_handle); if (dimm) { e->top_layer = dimm->idx; - strcpy(e->label, dimm->label); + strscpy(e->label, dimm->label); } } if (p > e->location) *(p - 1) = '\0'; if (!*e->label) - strcpy(e->label, "unknown memory"); + strscpy(e->label, "unknown memory"); /* All other fields are mapped on e->other_detail */ p = pvt->other_detail; diff --git a/drivers/edac/i10nm_base.c b/drivers/edac/i10nm_base.c index 2010a47149f4..89b3e8cc38b1 100644 --- a/drivers/edac/i10nm_base.c +++ b/drivers/edac/i10nm_base.c @@ -1198,7 +1198,8 @@ static int __init i10nm_init(void) d->imc[i].num_dimms = cfg->ddr_dimm_num; } - rc = skx_register_mci(&d->imc[i], d->imc[i].mdev, + rc = skx_register_mci(&d->imc[i], &d->imc[i].mdev->dev, + pci_name(d->imc[i].mdev), "Intel_10nm Socket", EDAC_MOD_STR, i10nm_get_dimm_config, cfg); if (rc < 0) diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index 5a080ab65476..8d4ddaa85ae8 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -526,6 +526,7 @@ static int ie31200_register_mci(struct pci_dev *pdev, struct res_config *cfg, in ie31200_pvt.priv[mc] = priv; return 0; fail_unmap: + put_device(&priv->dev); iounmap(window); fail_free: edac_mc_free(mci); @@ -598,6 +599,7 @@ static void ie31200_unregister_mcis(void) mci = priv->mci; edac_mc_del_mc(mci->pdev); iounmap(priv->window); + put_device(&priv->dev); edac_mc_free(mci); } } diff --git a/drivers/edac/igen6_edac.c b/drivers/edac/igen6_edac.c index 2fc59f9eed69..553c31a2d922 100644 --- a/drivers/edac/igen6_edac.c +++ b/drivers/edac/igen6_edac.c @@ -1300,6 +1300,7 @@ static int igen6_register_mci(int mc, void __iomem *window, struct pci_dev *pdev imc->mci = mci; return 0; fail3: + put_device(&imc->dev); mci->pvt_info = NULL; kfree(mci->ctl_name); fail2: @@ -1326,6 +1327,7 @@ static void igen6_unregister_mcis(void) kfree(mci->ctl_name); mci->pvt_info = NULL; edac_mc_free(mci); + put_device(&imc->dev); iounmap(imc->window); } } diff --git a/drivers/edac/imh_base.c b/drivers/edac/imh_base.c new file mode 100644 index 000000000000..4348b3883b45 --- /dev/null +++ b/drivers/edac/imh_base.c @@ -0,0 +1,602 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for Intel(R) servers with Integrated Memory/IO Hub-based memory controller. + * Copyright (c) 2025, Intel Corporation. + */ + +#include <linux/kernel.h> +#include <linux/io.h> +#include <asm/cpu_device_id.h> +#include <asm/intel-family.h> +#include <asm/mce.h> +#include <asm/cpu.h> +#include "edac_module.h" +#include "skx_common.h" + +#define IMH_REVISION "v0.0.1" +#define EDAC_MOD_STR "imh_edac" + +/* Debug macros */ +#define imh_printk(level, fmt, arg...) \ + edac_printk(level, "imh", fmt, ##arg) + +/* Configuration Agent(Ubox) */ +#define MMIO_BASE_H(reg) (((u64)GET_BITFIELD(reg, 0, 29)) << 23) +#define SOCKET_ID(reg) GET_BITFIELD(reg, 0, 3) + +/* PUNIT */ +#define DDR_IMC_BITMAP(reg) GET_BITFIELD(reg, 23, 30) + +/* Memory Controller */ +#define ECC_ENABLED(reg) GET_BITFIELD(reg, 2, 2) +#define DIMM_POPULATED(reg) GET_BITFIELD(reg, 15, 15) + +/* System Cache Agent(SCA) */ +#define TOLM(reg) (((u64)GET_BITFIELD(reg, 16, 31)) << 16) +#define TOHM(reg) (((u64)GET_BITFIELD(reg, 16, 51)) << 16) + +/* Home Agent (HA) */ +#define NMCACHING(reg) GET_BITFIELD(reg, 8, 8) + +/** + * struct local_reg - A register as described in the local package view. + * + * @pkg: (input) The package where the register is located. + * @pbase: (input) The IP MMIO base physical address in the local package view. + * @size: (input) The IP MMIO size. + * @offset: (input) The register offset from the IP MMIO base @pbase. + * @width: (input) The register width in byte. + * @vbase: (internal) The IP MMIO base virtual address. + * @val: (output) The register value. + */ +struct local_reg { + int pkg; + u64 pbase; + u32 size; + u32 offset; + u8 width; + void __iomem *vbase; + u64 val; +}; + +#define DEFINE_LOCAL_REG(name, cfg, package, north, ip_name, ip_idx, reg_name) \ + struct local_reg name = { \ + .pkg = package, \ + .pbase = (north ? (cfg)->mmio_base_l_north : \ + (cfg)->mmio_base_l_south) + \ + (cfg)->ip_name##_base + \ + (cfg)->ip_name##_size * (ip_idx), \ + .size = (cfg)->ip_name##_size, \ + .offset = (cfg)->ip_name##_reg_##reg_name##_offset, \ + .width = (cfg)->ip_name##_reg_##reg_name##_width, \ + } + +static u64 readx(void __iomem *addr, u8 width) +{ + switch (width) { + case 1: + return readb(addr); + case 2: + return readw(addr); + case 4: + return readl(addr); + case 8: + return readq(addr); + default: + imh_printk(KERN_ERR, "Invalid reg 0x%p width %d\n", addr, width); + return 0; + } +} + +static void __read_local_reg(void *reg) +{ + struct local_reg *r = (struct local_reg *)reg; + + r->val = readx(r->vbase + r->offset, r->width); +} + +/* Read a local-view register. */ +static bool read_local_reg(struct local_reg *reg) +{ + int cpu; + + /* Get the target CPU in the package @reg->pkg. */ + for_each_online_cpu(cpu) { + if (reg->pkg == topology_physical_package_id(cpu)) + break; + } + + if (cpu >= nr_cpu_ids) + return false; + + reg->vbase = ioremap(reg->pbase, reg->size); + if (!reg->vbase) { + imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", reg->pbase); + return false; + } + + /* Get the target CPU to read the register. */ + smp_call_function_single(cpu, __read_local_reg, reg, 1); + iounmap(reg->vbase); + + return true; +} + +/* Get the bitmap of memory controller instances in package @pkg. */ +static u32 get_imc_bitmap(struct res_config *cfg, int pkg, bool north) +{ + DEFINE_LOCAL_REG(reg, cfg, pkg, north, pcu, 0, capid3); + + if (!read_local_reg(®)) + return 0; + + edac_dbg(2, "Pkg%d %s mc instances bitmap 0x%llx (reg 0x%llx)\n", + pkg, north ? "north" : "south", + DDR_IMC_BITMAP(reg.val), reg.val); + + return DDR_IMC_BITMAP(reg.val); +} + +static void imc_release(struct device *dev) +{ + edac_dbg(2, "imc device %s released\n", dev_name(dev)); + kfree(dev); +} + +static int __get_ddr_munits(struct res_config *cfg, struct skx_dev *d, + bool north, int lmc) +{ + unsigned long size = cfg->ddr_chan_mmio_sz * cfg->ddr_chan_num; + unsigned long bitmap = get_imc_bitmap(cfg, d->pkg, north); + void __iomem *mbase; + struct device *dev; + int i, rc, pmc; + u64 base; + + for_each_set_bit(i, &bitmap, sizeof(bitmap) * 8) { + base = north ? d->mmio_base_h_north : d->mmio_base_h_south; + base += cfg->ddr_imc_base + size * i; + + edac_dbg(2, "Pkg%d mc%d mmio base 0x%llx size 0x%lx\n", + d->pkg, lmc, base, size); + + /* Set up the imc MMIO. */ + mbase = ioremap(base, size); + if (!mbase) { + imh_printk(KERN_ERR, "Failed to ioremap 0x%llx\n", base); + return -ENOMEM; + } + + d->imc[lmc].mbase = mbase; + d->imc[lmc].lmc = lmc; + + /* Create the imc device instance. */ + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return -ENOMEM; + + dev->release = imc_release; + device_initialize(dev); + rc = dev_set_name(dev, "0x%llx", base); + if (rc) { + imh_printk(KERN_ERR, "Failed to set dev name\n"); + put_device(dev); + return rc; + } + + d->imc[lmc].dev = dev; + + /* Set up the imc index mapping. */ + pmc = north ? i : 8 + i; + skx_set_mc_mapping(d, pmc, lmc); + + lmc++; + } + + return lmc; +} + +static bool get_ddr_munits(struct res_config *cfg, struct skx_dev *d) +{ + int lmc = __get_ddr_munits(cfg, d, true, 0); + + if (lmc < 0) + return false; + + lmc = __get_ddr_munits(cfg, d, false, lmc); + if (lmc <= 0) + return false; + + return true; +} + +static bool get_socket_id(struct res_config *cfg, struct skx_dev *d) +{ + DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ubox, 0, socket_id); + u8 src_id; + int i; + + if (!read_local_reg(®)) + return false; + + src_id = SOCKET_ID(reg.val); + edac_dbg(2, "socket id 0x%x (reg 0x%llx)\n", src_id, reg.val); + + for (i = 0; i < cfg->ddr_imc_num; i++) + d->imc[i].src_id = src_id; + + return true; +} + +/* Get TOLM (Top Of Low Memory) and TOHM (Top Of High Memory) parameters. */ +static bool imh_get_tolm_tohm(struct res_config *cfg, u64 *tolm, u64 *tohm) +{ + DEFINE_LOCAL_REG(reg, cfg, 0, true, sca, 0, tolm); + + if (!read_local_reg(®)) + return false; + + *tolm = TOLM(reg.val); + edac_dbg(2, "tolm 0x%llx (reg 0x%llx)\n", *tolm, reg.val); + + DEFINE_LOCAL_REG(reg2, cfg, 0, true, sca, 0, tohm); + + if (!read_local_reg(®2)) + return false; + + *tohm = TOHM(reg2.val); + edac_dbg(2, "tohm 0x%llx (reg 0x%llx)\n", *tohm, reg2.val); + + return true; +} + +/* Get the system-view MMIO_BASE_H for {north,south}-IMH. */ +static int imh_get_all_mmio_base_h(struct res_config *cfg, struct list_head *edac_list) +{ + int i, n = topology_max_packages(), imc_num = cfg->ddr_imc_num + cfg->hbm_imc_num; + struct skx_dev *d; + + for (i = 0; i < n; i++) { + d = kzalloc(struct_size(d, imc, imc_num), GFP_KERNEL); + if (!d) + return -ENOMEM; + + DEFINE_LOCAL_REG(reg, cfg, i, true, ubox, 0, mmio_base); + + /* Get MMIO_BASE_H for the north-IMH. */ + if (!read_local_reg(®) || !reg.val) { + kfree(d); + imh_printk(KERN_ERR, "Pkg%d has no north mmio_base_h\n", i); + return -ENODEV; + } + + d->mmio_base_h_north = MMIO_BASE_H(reg.val); + edac_dbg(2, "Pkg%d north mmio_base_h 0x%llx (reg 0x%llx)\n", + i, d->mmio_base_h_north, reg.val); + + /* Get MMIO_BASE_H for the south-IMH (optional). */ + DEFINE_LOCAL_REG(reg2, cfg, i, false, ubox, 0, mmio_base); + + if (read_local_reg(®2)) { + d->mmio_base_h_south = MMIO_BASE_H(reg2.val); + edac_dbg(2, "Pkg%d south mmio_base_h 0x%llx (reg 0x%llx)\n", + i, d->mmio_base_h_south, reg2.val); + } + + d->pkg = i; + d->num_imc = imc_num; + skx_init_mc_mapping(d); + list_add_tail(&d->list, edac_list); + } + + return 0; +} + +/* Get the number of per-package memory controllers. */ +static int imh_get_imc_num(struct res_config *cfg) +{ + int imc_num = hweight32(get_imc_bitmap(cfg, 0, true)) + + hweight32(get_imc_bitmap(cfg, 0, false)); + + if (!imc_num) { + imh_printk(KERN_ERR, "Invalid mc number\n"); + return -ENODEV; + } + + if (cfg->ddr_imc_num != imc_num) { + /* + * Update the configuration data to reflect the number of + * present DDR memory controllers. + */ + cfg->ddr_imc_num = imc_num; + edac_dbg(2, "Set ddr mc number %d\n", imc_num); + } + + return 0; +} + +/* Get all memory controllers' parameters. */ +static int imh_get_munits(struct res_config *cfg, struct list_head *edac_list) +{ + struct skx_imc *imc; + struct skx_dev *d; + u8 mc = 0; + int i; + + list_for_each_entry(d, edac_list, list) { + if (!get_ddr_munits(cfg, d)) { + imh_printk(KERN_ERR, "No mc found\n"); + return -ENODEV; + } + + if (!get_socket_id(cfg, d)) { + imh_printk(KERN_ERR, "Failed to get socket id\n"); + return -ENODEV; + } + + for (i = 0; i < cfg->ddr_imc_num; i++) { + imc = &d->imc[i]; + if (!imc->mbase) + continue; + + imc->chan_mmio_sz = cfg->ddr_chan_mmio_sz; + imc->num_channels = cfg->ddr_chan_num; + imc->num_dimms = cfg->ddr_dimm_num; + imc->mc = mc++; + } + } + + return 0; +} + +static bool check_2lm_enabled(struct res_config *cfg, struct skx_dev *d, int ha_idx) +{ + DEFINE_LOCAL_REG(reg, cfg, d->pkg, true, ha, ha_idx, mode); + + if (!read_local_reg(®)) + return false; + + if (!NMCACHING(reg.val)) + return false; + + edac_dbg(2, "2-level memory configuration (reg 0x%llx, ha idx %d)\n", reg.val, ha_idx); + return true; +} + +/* Check whether the system has a 2-level memory configuration. */ +static bool imh_2lm_enabled(struct res_config *cfg, struct list_head *head) +{ + struct skx_dev *d; + int i; + + list_for_each_entry(d, head, list) { + for (i = 0; i < cfg->ddr_imc_num; i++) + if (check_2lm_enabled(cfg, d, i)) + return true; + } + + return false; +} + +/* Helpers to read memory controller registers */ +static u64 read_imc_reg(struct skx_imc *imc, int chan, u32 offset, u8 width) +{ + return readx(imc->mbase + imc->chan_mmio_sz * chan + offset, width); +} + +static u32 read_imc_mcmtr(struct res_config *cfg, struct skx_imc *imc, int chan) +{ + return (u32)read_imc_reg(imc, chan, cfg->ddr_reg_mcmtr_offset, cfg->ddr_reg_mcmtr_width); +} + +static u32 read_imc_dimmmtr(struct res_config *cfg, struct skx_imc *imc, int chan, int dimm) +{ + return (u32)read_imc_reg(imc, chan, cfg->ddr_reg_dimmmtr_offset + + cfg->ddr_reg_dimmmtr_width * dimm, + cfg->ddr_reg_dimmmtr_width); +} + +static bool ecc_enabled(u32 mcmtr) +{ + return (bool)ECC_ENABLED(mcmtr); +} + +static bool dimm_populated(u32 dimmmtr) +{ + return (bool)DIMM_POPULATED(dimmmtr); +} + +/* Get each DIMM's configurations of the memory controller @mci. */ +static int imh_get_dimm_config(struct mem_ctl_info *mci, struct res_config *cfg) +{ + struct skx_pvt *pvt = mci->pvt_info; + struct skx_imc *imc = pvt->imc; + struct dimm_info *dimm; + u32 mcmtr, dimmmtr; + int i, j, ndimms; + + for (i = 0; i < imc->num_channels; i++) { + if (!imc->mbase) + continue; + + mcmtr = read_imc_mcmtr(cfg, imc, i); + + for (ndimms = 0, j = 0; j < imc->num_dimms; j++) { + dimmmtr = read_imc_dimmmtr(cfg, imc, i, j); + edac_dbg(1, "mcmtr 0x%x dimmmtr 0x%x (mc%d ch%d dimm%d)\n", + mcmtr, dimmmtr, imc->mc, i, j); + + if (!dimm_populated(dimmmtr)) + continue; + + dimm = edac_get_dimm(mci, i, j, 0); + ndimms += skx_get_dimm_info(dimmmtr, 0, 0, dimm, + imc, i, j, cfg); + } + + if (ndimms && !ecc_enabled(mcmtr)) { + imh_printk(KERN_ERR, "ECC is disabled on mc%d ch%d\n", + imc->mc, i); + return -ENODEV; + } + } + + return 0; +} + +/* Register all memory controllers to the EDAC core. */ +static int imh_register_mci(struct res_config *cfg, struct list_head *edac_list) +{ + struct skx_imc *imc; + struct skx_dev *d; + int i, rc; + + list_for_each_entry(d, edac_list, list) { + for (i = 0; i < cfg->ddr_imc_num; i++) { + imc = &d->imc[i]; + if (!imc->mbase) + continue; + + rc = skx_register_mci(imc, imc->dev, + dev_name(imc->dev), + "Intel IMH-based Socket", + EDAC_MOD_STR, + imh_get_dimm_config, cfg); + if (rc) + return rc; + } + } + + return 0; +} + +static struct res_config dmr_cfg = { + .type = DMR, + .support_ddr5 = true, + .mmio_base_l_north = 0xf6800000, + .mmio_base_l_south = 0xf6000000, + .ddr_chan_num = 1, + .ddr_dimm_num = 2, + .ddr_imc_base = 0x39b000, + .ddr_chan_mmio_sz = 0x8000, + .ddr_reg_mcmtr_offset = 0x360, + .ddr_reg_mcmtr_width = 4, + .ddr_reg_dimmmtr_offset = 0x370, + .ddr_reg_dimmmtr_width = 4, + .ubox_base = 0x0, + .ubox_size = 0x2000, + .ubox_reg_mmio_base_offset = 0x580, + .ubox_reg_mmio_base_width = 4, + .ubox_reg_socket_id_offset = 0x1080, + .ubox_reg_socket_id_width = 4, + .pcu_base = 0x3000, + .pcu_size = 0x10000, + .pcu_reg_capid3_offset = 0x290, + .pcu_reg_capid3_width = 4, + .sca_base = 0x24c000, + .sca_size = 0x2500, + .sca_reg_tolm_offset = 0x2100, + .sca_reg_tolm_width = 8, + .sca_reg_tohm_offset = 0x2108, + .sca_reg_tohm_width = 8, + .ha_base = 0x3eb000, + .ha_size = 0x1000, + .ha_reg_mode_offset = 0x4a0, + .ha_reg_mode_width = 4, +}; + +static const struct x86_cpu_id imh_cpuids[] = { + X86_MATCH_VFM(INTEL_DIAMONDRAPIDS_X, &dmr_cfg), + {} +}; +MODULE_DEVICE_TABLE(x86cpu, imh_cpuids); + +static struct notifier_block imh_mce_dec = { + .notifier_call = skx_mce_check_error, + .priority = MCE_PRIO_EDAC, +}; + +static int __init imh_init(void) +{ + const struct x86_cpu_id *id; + struct list_head *edac_list; + struct res_config *cfg; + const char *owner; + u64 tolm, tohm; + int rc; + + edac_dbg(2, "\n"); + + if (ghes_get_devices()) + return -EBUSY; + + owner = edac_get_owner(); + if (owner && strncmp(owner, EDAC_MOD_STR, sizeof(EDAC_MOD_STR))) + return -EBUSY; + + if (cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) + return -ENODEV; + + id = x86_match_cpu(imh_cpuids); + if (!id) + return -ENODEV; + cfg = (struct res_config *)id->driver_data; + skx_set_res_cfg(cfg); + + if (!imh_get_tolm_tohm(cfg, &tolm, &tohm)) + return -ENODEV; + + skx_set_hi_lo(tolm, tohm); + + rc = imh_get_imc_num(cfg); + if (rc < 0) + goto fail; + + edac_list = skx_get_edac_list(); + + rc = imh_get_all_mmio_base_h(cfg, edac_list); + if (rc) + goto fail; + + rc = imh_get_munits(cfg, edac_list); + if (rc) + goto fail; + + skx_set_mem_cfg(imh_2lm_enabled(cfg, edac_list)); + + rc = imh_register_mci(cfg, edac_list); + if (rc) + goto fail; + + rc = skx_adxl_get(); + if (rc) + goto fail; + + opstate_init(); + mce_register_decode_chain(&imh_mce_dec); + skx_setup_debug("imh_test"); + + imh_printk(KERN_INFO, "%s\n", IMH_REVISION); + + return 0; +fail: + skx_remove(); + return rc; +} + +static void __exit imh_exit(void) +{ + edac_dbg(2, "\n"); + + skx_teardown_debug(); + mce_unregister_decode_chain(&imh_mce_dec); + skx_adxl_put(); + skx_remove(); +} + +module_init(imh_init); +module_exit(imh_exit); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Qiuxu Zhuo"); +MODULE_DESCRIPTION("MC Driver for Intel servers using IMH-based memory controller"); diff --git a/drivers/edac/skx_base.c b/drivers/edac/skx_base.c index 078ddf95cc6e..aa6593ccda2d 100644 --- a/drivers/edac/skx_base.c +++ b/drivers/edac/skx_base.c @@ -662,8 +662,8 @@ static int __init skx_init(void) d->imc[i].src_id = src_id; d->imc[i].num_channels = cfg->ddr_chan_num; d->imc[i].num_dimms = cfg->ddr_dimm_num; - - rc = skx_register_mci(&d->imc[i], d->imc[i].chan[0].cdev, + rc = skx_register_mci(&d->imc[i], &d->imc[i].chan[0].cdev->dev, + pci_name(d->imc[i].chan[0].cdev), "Skylake Socket", EDAC_MOD_STR, skx_get_dimm_config, cfg); if (rc < 0) diff --git a/drivers/edac/skx_common.c b/drivers/edac/skx_common.c index 724842f512ac..3276afe43922 100644 --- a/drivers/edac/skx_common.c +++ b/drivers/edac/skx_common.c @@ -124,7 +124,7 @@ void skx_adxl_put(void) } EXPORT_SYMBOL_GPL(skx_adxl_put); -static void skx_init_mc_mapping(struct skx_dev *d) +void skx_init_mc_mapping(struct skx_dev *d) { /* * By default, the BIOS presents all memory controllers within each @@ -135,6 +135,7 @@ static void skx_init_mc_mapping(struct skx_dev *d) for (int i = 0; i < d->num_imc; i++) d->imc[i].mc_mapping = i; } +EXPORT_SYMBOL_GPL(skx_init_mc_mapping); void skx_set_mc_mapping(struct skx_dev *d, u8 pmc, u8 lmc) { @@ -384,6 +385,12 @@ int skx_get_all_bus_mappings(struct res_config *cfg, struct list_head **list) } EXPORT_SYMBOL_GPL(skx_get_all_bus_mappings); +struct list_head *skx_get_edac_list(void) +{ + return &dev_edac_list; +} +EXPORT_SYMBOL_GPL(skx_get_edac_list); + int skx_get_hi_lo(unsigned int did, int off[], u64 *tolm, u64 *tohm) { struct pci_dev *pdev; @@ -424,6 +431,13 @@ fail: } EXPORT_SYMBOL_GPL(skx_get_hi_lo); +void skx_set_hi_lo(u64 tolm, u64 tohm) +{ + skx_tolm = tolm; + skx_tohm = tohm; +} +EXPORT_SYMBOL_GPL(skx_set_hi_lo); + static int skx_get_dimm_attr(u32 reg, int lobit, int hibit, int add, int minval, int maxval, const char *name) { @@ -437,7 +451,7 @@ static int skx_get_dimm_attr(u32 reg, int lobit, int hibit, int add, } #define numrank(reg) skx_get_dimm_attr(reg, 12, 13, 0, 0, 2, "ranks") -#define numrow(reg) skx_get_dimm_attr(reg, 2, 4, 12, 1, 6, "rows") +#define numrow(reg) skx_get_dimm_attr(reg, 2, 4, 12, 1, 7, "rows") #define numcol(reg) skx_get_dimm_attr(reg, 0, 1, 10, 0, 2, "cols") int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, @@ -545,9 +559,9 @@ unknown_size: } EXPORT_SYMBOL_GPL(skx_get_nvdimm_info); -int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, - const char *ctl_name, const char *mod_str, - get_dimm_config_f get_dimm_config, +int skx_register_mci(struct skx_imc *imc, struct device *dev, + const char *dev_name, const char *ctl_name, + const char *mod_str, get_dimm_config_f get_dimm_config, struct res_config *cfg) { struct mem_ctl_info *mci; @@ -588,7 +602,7 @@ int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, mci->edac_ctl_cap = EDAC_FLAG_NONE; mci->edac_cap = EDAC_FLAG_NONE; mci->mod_name = mod_str; - mci->dev_name = pci_name(pdev); + mci->dev_name = dev_name; mci->ctl_page_to_phys = NULL; rc = get_dimm_config(mci, cfg); @@ -596,7 +610,7 @@ int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, goto fail; /* Record ptr to the generic device */ - mci->pdev = &pdev->dev; + mci->pdev = dev; /* Add this new MC control structure to EDAC's list of MCs */ if (unlikely(edac_mc_add_mc(mci))) { @@ -810,6 +824,9 @@ void skx_remove(void) if (d->imc[i].mbase) iounmap(d->imc[i].mbase); + if (d->imc[i].dev) + put_device(d->imc[i].dev); + for (j = 0; j < d->imc[i].num_channels; j++) { if (d->imc[i].chan[j].cdev) pci_dev_put(d->imc[i].chan[j].cdev); @@ -833,7 +850,7 @@ EXPORT_SYMBOL_GPL(skx_remove); /* * Debug feature. * Exercise the address decode logic by writing an address to - * /sys/kernel/debug/edac/{skx,i10nm}_test/addr. + * /sys/kernel/debug/edac/{skx,i10nm,imh}_test/addr. */ static struct dentry *skx_test; diff --git a/drivers/edac/skx_common.h b/drivers/edac/skx_common.h index 73ba89786cdf..f88038e5b18c 100644 --- a/drivers/edac/skx_common.h +++ b/drivers/edac/skx_common.h @@ -121,20 +121,33 @@ struct reg_rrl { * memory controllers on the die. */ struct skx_dev { - struct list_head list; + /* {skx,i10nm}_edac */ u8 bus[4]; int seg; struct pci_dev *sad_all; struct pci_dev *util_all; - struct pci_dev *uracu; /* for i10nm CPU */ - struct pci_dev *pcu_cr3; /* for HBM memory detection */ + struct pci_dev *uracu; + struct pci_dev *pcu_cr3; u32 mcroute; + + /* imh_edac */ + /* System-view MMIO base physical addresses. */ + u64 mmio_base_h_north; + u64 mmio_base_h_south; + int pkg; + int num_imc; + struct list_head list; struct skx_imc { + /* i10nm_edac */ + struct pci_dev *mdev; + + /* imh_edac */ + struct device *dev; + struct mem_ctl_info *mci; - struct pci_dev *mdev; /* for i10nm CPU */ - void __iomem *mbase; /* for i10nm CPU */ - int chan_mmio_sz; /* for i10nm CPU */ + void __iomem *mbase; + int chan_mmio_sz; int num_channels; /* channels per memory controller */ int num_dimms; /* dimms per channel */ bool hbm_mc; @@ -178,7 +191,8 @@ enum type { SKX, I10NM, SPR, - GNR + GNR, + DMR, }; enum { @@ -237,10 +251,6 @@ struct pci_bdf { struct res_config { enum type type; - /* Configuration agent device ID */ - unsigned int decs_did; - /* Default bus number configuration register offset */ - int busno_cfg_offset; /* DDR memory controllers per socket */ int ddr_imc_num; /* DDR channels per DDR memory controller */ @@ -258,23 +268,57 @@ struct res_config { /* Per HBM channel memory-mapped I/O size */ int hbm_chan_mmio_sz; bool support_ddr5; - /* SAD device BDF */ - struct pci_bdf sad_all_bdf; - /* PCU device BDF */ - struct pci_bdf pcu_cr3_bdf; - /* UTIL device BDF */ - struct pci_bdf util_all_bdf; - /* URACU device BDF */ - struct pci_bdf uracu_bdf; - /* DDR mdev device BDF */ - struct pci_bdf ddr_mdev_bdf; - /* HBM mdev device BDF */ - struct pci_bdf hbm_mdev_bdf; - int sad_all_offset; /* RRL register sets per DDR channel */ struct reg_rrl *reg_rrl_ddr; /* RRL register sets per HBM channel */ struct reg_rrl *reg_rrl_hbm[2]; + union { + /* {skx,i10nm}_edac */ + struct { + /* Configuration agent device ID */ + unsigned int decs_did; + /* Default bus number configuration register offset */ + int busno_cfg_offset; + struct pci_bdf sad_all_bdf; + struct pci_bdf pcu_cr3_bdf; + struct pci_bdf util_all_bdf; + struct pci_bdf uracu_bdf; + struct pci_bdf ddr_mdev_bdf; + struct pci_bdf hbm_mdev_bdf; + int sad_all_offset; + }; + /* imh_edac */ + struct { + /* MMIO base physical address in local package view */ + u64 mmio_base_l_north; + u64 mmio_base_l_south; + u64 ddr_imc_base; + u64 ddr_reg_mcmtr_offset; + u8 ddr_reg_mcmtr_width; + u64 ddr_reg_dimmmtr_offset; + u8 ddr_reg_dimmmtr_width; + u64 ubox_base; + u32 ubox_size; + u32 ubox_reg_mmio_base_offset; + u8 ubox_reg_mmio_base_width; + u32 ubox_reg_socket_id_offset; + u8 ubox_reg_socket_id_width; + u64 pcu_base; + u32 pcu_size; + u32 pcu_reg_capid3_offset; + u8 pcu_reg_capid3_width; + u64 sca_base; + u32 sca_size; + u32 sca_reg_tolm_offset; + u8 sca_reg_tolm_width; + u32 sca_reg_tohm_offset; + u8 sca_reg_tohm_width; + u64 ha_base; + u32 ha_size; + u32 ha_reg_mode_offset; + u8 ha_reg_mode_width; + }; + }; }; typedef int (*get_dimm_config_f)(struct mem_ctl_info *mci, @@ -287,13 +331,17 @@ void skx_adxl_put(void); void skx_set_decode(skx_decode_f decode, skx_show_retry_log_f show_retry_log); void skx_set_mem_cfg(bool mem_cfg_2lm); void skx_set_res_cfg(struct res_config *cfg); +void skx_init_mc_mapping(struct skx_dev *d); void skx_set_mc_mapping(struct skx_dev *d, u8 pmc, u8 lmc); int skx_get_src_id(struct skx_dev *d, int off, u8 *id); int skx_get_all_bus_mappings(struct res_config *cfg, struct list_head **list); +struct list_head *skx_get_edac_list(void); + int skx_get_hi_lo(unsigned int did, int off[], u64 *tolm, u64 *tohm); +void skx_set_hi_lo(u64 tolm, u64 tohm); int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, struct skx_imc *imc, int chan, int dimmno, @@ -302,7 +350,7 @@ int skx_get_dimm_info(u32 mtr, u32 mcmtr, u32 amap, struct dimm_info *dimm, int skx_get_nvdimm_info(struct dimm_info *dimm, struct skx_imc *imc, int chan, int dimmno, const char *mod_str); -int skx_register_mci(struct skx_imc *imc, struct pci_dev *pdev, +int skx_register_mci(struct skx_imc *imc, struct device *dev, const char *dev_name, const char *ctl_name, const char *mod_str, get_dimm_config_f get_dimm_config, struct res_config *cfg); diff --git a/drivers/firmware/efi/efi.c b/drivers/firmware/efi/efi.c index 1ce428e2ac8a..a9070d00b833 100644 --- a/drivers/firmware/efi/efi.c +++ b/drivers/firmware/efi/efi.c @@ -74,6 +74,9 @@ struct mm_struct efi_mm = { .page_table_lock = __SPIN_LOCK_UNLOCKED(efi_mm.page_table_lock), .mmlist = LIST_HEAD_INIT(efi_mm.mmlist), .cpu_bitmap = { [BITS_TO_LONGS(NR_CPUS)] = 0}, +#ifdef CONFIG_SCHED_MM_CID + .mm_cid.lock = __RAW_SPIN_LOCK_UNLOCKED(efi_mm.mm_cid.lock), +#endif }; struct workqueue_struct *efi_rts_wq; diff --git a/drivers/firmware/efi/libstub/Makefile b/drivers/firmware/efi/libstub/Makefile index 94b05e4451dd..7d15a85d579f 100644 --- a/drivers/firmware/efi/libstub/Makefile +++ b/drivers/firmware/efi/libstub/Makefile @@ -11,12 +11,12 @@ cflags-y := $(KBUILD_CFLAGS) cflags-$(CONFIG_X86_32) := -march=i386 cflags-$(CONFIG_X86_64) := -mcmodel=small -cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ -std=gnu11 \ +cflags-$(CONFIG_X86) += -m$(BITS) -D__KERNEL__ -std=gnu11 -fms-extensions \ -fPIC -fno-strict-aliasing -mno-red-zone \ -mno-mmx -mno-sse -fshort-wchar \ -Wno-pointer-sign \ $(call cc-disable-warning, address-of-packed-member) \ - $(call cc-disable-warning, gnu) \ + $(if $(CONFIG_CC_IS_CLANG),-Wno-gnu -Wno-microsoft-anon-tag) \ -fno-asynchronous-unwind-tables \ $(CLANG_FLAGS) diff --git a/drivers/firmware/efi/libstub/x86-5lvl.c b/drivers/firmware/efi/libstub/x86-5lvl.c index f1c5fb45d5f7..c00d0ae7ed5d 100644 --- a/drivers/firmware/efi/libstub/x86-5lvl.c +++ b/drivers/firmware/efi/libstub/x86-5lvl.c @@ -66,7 +66,7 @@ void efi_5level_switch(void) bool have_la57 = native_read_cr4() & X86_CR4_LA57; bool need_toggle = want_la57 ^ have_la57; u64 *pgt = (void *)la57_toggle + PAGE_SIZE; - u64 *cr3 = (u64 *)__native_read_cr3(); + pgd_t *cr3 = (pgd_t *)native_read_cr3_pa(); u64 *new_cr3; if (!la57_toggle || !need_toggle) @@ -82,7 +82,7 @@ void efi_5level_switch(void) new_cr3[0] = (u64)cr3 | _PAGE_TABLE_NOENC; } else { /* take the new root table pointer from the current entry #0 */ - new_cr3 = (u64 *)(cr3[0] & PAGE_MASK); + new_cr3 = (u64 *)(native_pgd_val(cr3[0]) & PTE_PFN_MASK); /* copy the new root table if it is not 32-bit addressable */ if ((u64)new_cr3 > U32_MAX) diff --git a/drivers/firmware/efi/runtime-wrappers.c b/drivers/firmware/efi/runtime-wrappers.c index 708b777857d3..da8d29621644 100644 --- a/drivers/firmware/efi/runtime-wrappers.c +++ b/drivers/firmware/efi/runtime-wrappers.c @@ -202,6 +202,8 @@ void efi_call_virt_check_flags(unsigned long flags, const void *caller) */ static DEFINE_SEMAPHORE(efi_runtime_lock, 1); +static struct task_struct *efi_runtime_lock_owner; + /* * Expose the EFI runtime lock to the UV platform */ @@ -219,6 +221,8 @@ static void __nocfi efi_call_rts(struct work_struct *work) efi_status_t status = EFI_NOT_FOUND; unsigned long flags; + efi_runtime_lock_owner = current; + arch_efi_call_virt_setup(); flags = efi_call_virt_save_flags(); @@ -310,6 +314,7 @@ static void __nocfi efi_call_rts(struct work_struct *work) efi_rts_work.status = status; complete(&efi_rts_work.efi_rts_comp); + efi_runtime_lock_owner = NULL; } static efi_status_t __efi_queue_work(enum efi_rts_ids id, @@ -444,8 +449,10 @@ virt_efi_set_variable_nb(efi_char16_t *name, efi_guid_t *vendor, u32 attr, if (down_trylock(&efi_runtime_lock)) return EFI_NOT_READY; + efi_runtime_lock_owner = current; status = efi_call_virt_pointer(efi.runtime, set_variable, name, vendor, attr, data_size, data); + efi_runtime_lock_owner = NULL; up(&efi_runtime_lock); return status; } @@ -481,9 +488,11 @@ virt_efi_query_variable_info_nb(u32 attr, u64 *storage_space, if (down_trylock(&efi_runtime_lock)) return EFI_NOT_READY; + efi_runtime_lock_owner = current; status = efi_call_virt_pointer(efi.runtime, query_variable_info, attr, storage_space, remaining_space, max_variable_size); + efi_runtime_lock_owner = NULL; up(&efi_runtime_lock); return status; } @@ -509,12 +518,13 @@ virt_efi_reset_system(int reset_type, efi_status_t status, return; } + efi_runtime_lock_owner = current; arch_efi_call_virt_setup(); efi_rts_work.efi_rts_id = EFI_RESET_SYSTEM; arch_efi_call_virt(efi.runtime, reset_system, reset_type, status, data_size, data); arch_efi_call_virt_teardown(); - + efi_runtime_lock_owner = NULL; up(&efi_runtime_lock); } @@ -587,3 +597,8 @@ efi_call_acpi_prm_handler(efi_status_t (__efiapi *handler_addr)(u64, void *), } #endif + +void efi_runtime_assert_lock_held(void) +{ + WARN_ON(efi_runtime_lock_owner != current); +} diff --git a/drivers/firmware/stratix10-svc.c b/drivers/firmware/stratix10-svc.c index e3f990d888d7..00f58e27f6de 100644 --- a/drivers/firmware/stratix10-svc.c +++ b/drivers/firmware/stratix10-svc.c @@ -134,6 +134,7 @@ struct stratix10_svc_data { * @complete_status: state for completion * @svc_fifo_lock: protect access to service message data queue * @invoke_fn: function to issue secure monitor call or hypervisor call + * @svc: manages the list of client svc drivers * * This struct is used to create communication channels for service clients, to * handle secure monitor or hypervisor call. @@ -150,6 +151,7 @@ struct stratix10_svc_controller { struct completion complete_status; spinlock_t svc_fifo_lock; svc_invoke_fn *invoke_fn; + struct stratix10_svc *svc; }; /** @@ -1206,6 +1208,7 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) ret = -ENOMEM; goto err_free_kfifo; } + controller->svc = svc; svc->stratix10_svc_rsu = platform_device_alloc(STRATIX10_RSU, 0); if (!svc->stratix10_svc_rsu) { @@ -1237,8 +1240,6 @@ static int stratix10_svc_drv_probe(struct platform_device *pdev) if (ret) goto err_unregister_fcs_dev; - dev_set_drvdata(dev, svc); - pr_info("Intel Service Layer Driver Initialized\n"); return 0; @@ -1256,8 +1257,8 @@ err_destroy_pool: static void stratix10_svc_drv_remove(struct platform_device *pdev) { - struct stratix10_svc *svc = dev_get_drvdata(&pdev->dev); struct stratix10_svc_controller *ctrl = platform_get_drvdata(pdev); + struct stratix10_svc *svc = ctrl->svc; of_platform_depopulate(ctrl->dev); diff --git a/drivers/gpio/gpiolib-cdev.c b/drivers/gpio/gpiolib-cdev.c index d8d93059ac04..084656564176 100644 --- a/drivers/gpio/gpiolib-cdev.c +++ b/drivers/gpio/gpiolib-cdev.c @@ -298,12 +298,13 @@ static const struct file_operations linehandle_fileops = { #endif }; +DEFINE_FREE(linehandle_free, struct linehandle_state *, if (!IS_ERR_OR_NULL(_T)) linehandle_free(_T)) + static int linehandle_create(struct gpio_device *gdev, void __user *ip) { struct gpiohandle_request handlereq; - struct linehandle_state *lh; - struct file *file; - int fd, i, ret; + struct linehandle_state *lh __free(linehandle_free) = NULL; + int i, ret; u32 lflags; if (copy_from_user(&handlereq, ip, sizeof(handlereq))) @@ -327,10 +328,8 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) lh->label = kstrndup(handlereq.consumer_label, sizeof(handlereq.consumer_label) - 1, GFP_KERNEL); - if (!lh->label) { - ret = -ENOMEM; - goto out_free_lh; - } + if (!lh->label) + return -ENOMEM; } lh->num_descs = handlereq.lines; @@ -340,20 +339,18 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) u32 offset = handlereq.lineoffsets[i]; struct gpio_desc *desc = gpio_device_get_desc(gdev, offset); - if (IS_ERR(desc)) { - ret = PTR_ERR(desc); - goto out_free_lh; - } + if (IS_ERR(desc)) + return PTR_ERR(desc); ret = gpiod_request_user(desc, lh->label); if (ret) - goto out_free_lh; + return ret; lh->descs[i] = desc; linehandle_flags_to_desc_flags(handlereq.flags, &desc->flags); ret = gpiod_set_transitory(desc, false); if (ret < 0) - goto out_free_lh; + return ret; /* * Lines have to be requested explicitly for input @@ -364,11 +361,11 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) ret = gpiod_direction_output_nonotify(desc, val); if (ret) - goto out_free_lh; + return ret; } else if (lflags & GPIOHANDLE_REQUEST_INPUT) { ret = gpiod_direction_input_nonotify(desc); if (ret) - goto out_free_lh; + return ret; } gpiod_line_state_notify(desc, GPIO_V2_LINE_CHANGED_REQUESTED); @@ -377,44 +374,23 @@ static int linehandle_create(struct gpio_device *gdev, void __user *ip) offset); } - fd = get_unused_fd_flags(O_RDONLY | O_CLOEXEC); - if (fd < 0) { - ret = fd; - goto out_free_lh; - } + FD_PREPARE(fdf, O_RDONLY | O_CLOEXEC, + anon_inode_getfile("gpio-linehandle", &linehandle_fileops, + lh, O_RDONLY | O_CLOEXEC)); + if (fdf.err) + return fdf.err; + retain_and_null_ptr(lh); - file = anon_inode_getfile("gpio-linehandle", - &linehandle_fileops, - lh, - O_RDONLY | O_CLOEXEC); - if (IS_ERR(file)) { - ret = PTR_ERR(file); - goto out_put_unused_fd; - } - - handlereq.fd = fd; - if (copy_to_user(ip, &handlereq, sizeof(handlereq))) { - /* - * fput() will trigger the release() callback, so do not go onto - * the regular error cleanup path here. - */ - fput(file); - put_unused_fd(fd); + handlereq.fd = fd_prepare_fd(fdf); + if (copy_to_user(ip, &handlereq, sizeof(handlereq))) return -EFAULT; - } - fd_install(fd, file); + fd_publish(fdf); dev_dbg(&gdev->dev, "registered chardev handle for %d lines\n", lh->num_descs); return 0; - -out_put_unused_fd: - put_unused_fd(fd); -out_free_lh: - linehandle_free(lh); - return ret; } #endif /* CONFIG_GPIO_CDEV_V1 */ diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile index da2565e6de71..1701e5cafd9f 100644 --- a/drivers/gpu/drm/Makefile +++ b/drivers/gpu/drm/Makefile @@ -6,7 +6,7 @@ CFLAGS-$(CONFIG_DRM_USE_DYNAMIC_DEBUG) += -DDYNAMIC_DEBUG_MODULE # Unconditionally enable W=1 warnings locally -# --- begin copy-paste W=1 warnings from scripts/Makefile.extrawarn +# --- begin copy-paste W=1 warnings from scripts/Makefile.warn subdir-ccflags-y += -Wextra -Wunused -Wno-unused-parameter subdir-ccflags-y += $(call cc-option, -Wrestrict) subdir-ccflags-y += -Wmissing-format-attribute diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c index 8a026bc9ea44..4e2fe6674db8 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_dev_coredump.c @@ -217,8 +217,7 @@ amdgpu_devcoredump_read(char *buffer, loff_t offset, size_t count, drm_printf(&p, "version: " AMDGPU_COREDUMP_VERSION "\n"); drm_printf(&p, "kernel: " UTS_RELEASE "\n"); drm_printf(&p, "module: " KBUILD_MODNAME "\n"); - drm_printf(&p, "time: %lld.%09ld\n", coredump->reset_time.tv_sec, - coredump->reset_time.tv_nsec); + drm_printf(&p, "time: %ptSp\n", &coredump->reset_time); if (coredump->reset_task_info.task.pid) drm_printf(&p, "process_name: %s PID: %d\n", diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c index 076bbc09f30c..96b6738e6252 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c @@ -2638,6 +2638,8 @@ static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev) chip_name = "navi12"; break; case CHIP_CYAN_SKILLFISH: + if (adev->mman.discovery_bin) + return 0; chip_name = "cyan_skillfish"; break; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c index 9dcf51991b5b..869bceb0fe2c 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gmc.c @@ -597,6 +597,9 @@ int amdgpu_gmc_allocate_vm_inv_eng(struct amdgpu_device *adev) /* reserve engine 5 for firmware */ if (adev->enable_mes) vm_inv_engs[i] &= ~(1 << 5); + /* reserve engine 6 for uni mes */ + if (adev->enable_uni_mes) + vm_inv_engs[i] &= ~(1 << 6); /* reserve mmhub engine 3 for firmware */ if (adev->enable_umsch_mm) vm_inv_engs[i] &= ~(1 << 3); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c index b1aaef962ad9..676e24fb8864 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c @@ -1066,7 +1066,7 @@ amdgpu_vm_tlb_flush(struct amdgpu_vm_update_params *params, } /* Prepare a TLB flush fence to be attached to PTs */ - if (!params->unlocked && vm->is_compute_context) { + if (!params->unlocked) { amdgpu_vm_tlb_fence_create(params->adev, vm, fence); /* Makes sure no PD/PT is freed before the flush */ diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c index 2a7a491a62e0..7fe40bbba265 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm.c @@ -5052,6 +5052,21 @@ static void amdgpu_dm_backlight_set_level(struct amdgpu_display_manager *dm, struct dc_link *link; u32 brightness; bool rc, reallow_idle = false; + struct drm_connector *connector; + + list_for_each_entry(connector, &dm->ddev->mode_config.connector_list, head) { + struct amdgpu_dm_connector *aconnector = to_amdgpu_dm_connector(connector); + + if (aconnector->bl_idx != bl_idx) + continue; + + /* if connector is off, save the brightness for next time it's on */ + if (!aconnector->base.encoder) { + dm->brightness[bl_idx] = user_brightness; + dm->actual_brightness[bl_idx] = 0; + return; + } + } amdgpu_dm_update_backlight_caps(dm, bl_idx); caps = &dm->backlight_caps[bl_idx]; diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c index cc21337a182f..d0f770dd0a95 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_helpers.c @@ -997,8 +997,8 @@ enum dc_edid_status dm_helpers_read_local_edid( struct amdgpu_dm_connector *aconnector = link->priv; struct drm_connector *connector = &aconnector->base; struct i2c_adapter *ddc; - int retry = 3; - enum dc_edid_status edid_status; + int retry = 25; + enum dc_edid_status edid_status = EDID_NO_RESPONSE; const struct drm_edid *drm_edid; const struct edid *edid; @@ -1028,7 +1028,7 @@ enum dc_edid_status dm_helpers_read_local_edid( } if (!drm_edid) - return EDID_NO_RESPONSE; + continue; edid = drm_edid_raw(drm_edid); // FIXME: Get rid of drm_edid_raw() if (!edid || @@ -1046,7 +1046,7 @@ enum dc_edid_status dm_helpers_read_local_edid( &sink->dc_edid, &sink->edid_caps); - } while (edid_status == EDID_BAD_CHECKSUM && --retry > 0); + } while ((edid_status == EDID_BAD_CHECKSUM || edid_status == EDID_NO_RESPONSE) && --retry > 0); if (edid_status != EDID_OK) DRM_ERROR("EDID err: %d, on connector: %s", diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c index 9ac2d41f8fca..0a46e834357a 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_stream.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_stream.c @@ -705,9 +705,14 @@ bool dc_stream_get_scanoutpos(const struct dc_stream_state *stream, { uint8_t i; bool ret = false; - struct dc *dc = stream->ctx->dc; - struct resource_context *res_ctx = - &dc->current_state->res_ctx; + struct dc *dc; + struct resource_context *res_ctx; + + if (!stream->ctx) + return false; + + dc = stream->ctx->dc; + res_ctx = &dc->current_state->res_ctx; dc_exit_ips_for_hw_access(dc); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c index 24184b4eb352..ebc220b29d14 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dce110/dce110_hwseq.c @@ -671,7 +671,6 @@ void dce110_enable_stream(struct pipe_ctx *pipe_ctx) uint32_t early_control = 0; struct timing_generator *tg = pipe_ctx->stream_res.tg; - link_hwss->setup_stream_attribute(pipe_ctx); link_hwss->setup_stream_encoder(pipe_ctx); dc->hwss.update_info_frame(pipe_ctx); diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c index 59c42db5382e..56c1ab6c7330 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn20/dcn20_hwseq.c @@ -3060,8 +3060,6 @@ void dcn20_enable_stream(struct pipe_ctx *pipe_ctx) link_enc->transmitter - TRANSMITTER_UNIPHY_A); } - link_hwss->setup_stream_attribute(pipe_ctx); - if (dc->res_pool->dccg->funcs->set_pixel_rate_div) dc->res_pool->dccg->funcs->set_pixel_rate_div( dc->res_pool->dccg, diff --git a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c index ce3d0b45fb4c..68e48a2492c9 100644 --- a/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c +++ b/drivers/gpu/drm/amd/display/dc/hwss/dcn401/dcn401_hwseq.c @@ -971,8 +971,6 @@ void dcn401_enable_stream(struct pipe_ctx *pipe_ctx) } } - link_hwss->setup_stream_attribute(pipe_ctx); - if (dc->res_pool->dccg->funcs->set_pixel_rate_div) { dc->res_pool->dccg->funcs->set_pixel_rate_div( dc->res_pool->dccg, diff --git a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c index 83419e1a9036..b66fbcb0040d 100644 --- a/drivers/gpu/drm/amd/display/dc/link/link_dpms.c +++ b/drivers/gpu/drm/amd/display/dc/link/link_dpms.c @@ -2458,6 +2458,7 @@ void link_set_dpms_on( struct link_encoder *link_enc = pipe_ctx->link_res.dio_link_enc; enum otg_out_mux_dest otg_out_dest = OUT_MUX_DIO; struct vpg *vpg = pipe_ctx->stream_res.stream_enc->vpg; + const struct link_hwss *link_hwss = get_link_hwss(link, &pipe_ctx->link_res); bool apply_edp_fast_boot_optimization = pipe_ctx->stream->apply_edp_fast_boot_optimization; @@ -2502,6 +2503,8 @@ void link_set_dpms_on( pipe_ctx->stream_res.tg->funcs->set_out_mux(pipe_ctx->stream_res.tg, otg_out_dest); } + link_hwss->setup_stream_attribute(pipe_ctx); + pipe_ctx->stream->apply_edp_fast_boot_optimization = false; // Enable VPG before building infoframe diff --git a/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c b/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c index 6ffc74fc9dcd..ad088d70e189 100644 --- a/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/virtual/virtual_stream_encoder.c @@ -44,11 +44,6 @@ static void virtual_stream_encoder_dvi_set_stream_attribute( struct dc_crtc_timing *crtc_timing, bool is_dual_link) {} -static void virtual_stream_encoder_lvds_set_stream_attribute( - struct stream_encoder *enc, - struct dc_crtc_timing *crtc_timing) -{} - static void virtual_stream_encoder_set_throttled_vcp_size( struct stream_encoder *enc, struct fixed31_32 avg_time_slots_per_mtp) @@ -120,8 +115,6 @@ static const struct stream_encoder_funcs virtual_str_enc_funcs = { virtual_stream_encoder_hdmi_set_stream_attribute, .dvi_set_stream_attribute = virtual_stream_encoder_dvi_set_stream_attribute, - .lvds_set_stream_attribute = - virtual_stream_encoder_lvds_set_stream_attribute, .set_throttled_vcp_size = virtual_stream_encoder_set_throttled_vcp_size, .update_hdmi_info_packets = diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c index d537b1d036fb..1f0aba28ad1e 100644 --- a/drivers/gpu/drm/bridge/sii902x.c +++ b/drivers/gpu/drm/bridge/sii902x.c @@ -179,7 +179,6 @@ struct sii902x { struct drm_connector connector; struct gpio_desc *reset_gpio; struct i2c_mux_core *i2cmux; - bool sink_is_hdmi; u32 bus_width; /* @@ -315,8 +314,6 @@ static int sii902x_get_modes(struct drm_connector *connector) drm_edid_free(drm_edid); } - sii902x->sink_is_hdmi = connector->display_info.is_hdmi; - return num; } @@ -342,9 +339,17 @@ static void sii902x_bridge_atomic_enable(struct drm_bridge *bridge, struct drm_atomic_state *state) { struct sii902x *sii902x = bridge_to_sii902x(bridge); + struct drm_connector *connector; + u8 output_mode = SII902X_SYS_CTRL_OUTPUT_DVI; + + connector = drm_atomic_get_new_connector_for_encoder(state, bridge->encoder); + if (connector && connector->display_info.is_hdmi) + output_mode = SII902X_SYS_CTRL_OUTPUT_HDMI; mutex_lock(&sii902x->mutex); + regmap_update_bits(sii902x->regmap, SII902X_SYS_CTRL_DATA, + SII902X_SYS_CTRL_OUTPUT_MODE, output_mode); regmap_update_bits(sii902x->regmap, SII902X_PWR_STATE_CTRL, SII902X_AVI_POWER_STATE_MSK, SII902X_AVI_POWER_STATE_D(0)); @@ -359,16 +364,12 @@ static void sii902x_bridge_mode_set(struct drm_bridge *bridge, const struct drm_display_mode *adj) { struct sii902x *sii902x = bridge_to_sii902x(bridge); - u8 output_mode = SII902X_SYS_CTRL_OUTPUT_DVI; struct regmap *regmap = sii902x->regmap; u8 buf[HDMI_INFOFRAME_SIZE(AVI)]; struct hdmi_avi_infoframe frame; u16 pixel_clock_10kHz = adj->clock / 10; int ret; - if (sii902x->sink_is_hdmi) - output_mode = SII902X_SYS_CTRL_OUTPUT_HDMI; - buf[0] = pixel_clock_10kHz & 0xff; buf[1] = pixel_clock_10kHz >> 8; buf[2] = drm_mode_vrefresh(adj); @@ -384,11 +385,6 @@ static void sii902x_bridge_mode_set(struct drm_bridge *bridge, mutex_lock(&sii902x->mutex); - ret = regmap_update_bits(sii902x->regmap, SII902X_SYS_CTRL_DATA, - SII902X_SYS_CTRL_OUTPUT_MODE, output_mode); - if (ret) - goto out; - ret = regmap_bulk_write(regmap, SII902X_TPI_VIDEO_DATA, buf, 10); if (ret) goto out; diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c index 11a5b60cb9ce..0b3ee008523d 100644 --- a/drivers/gpu/drm/drm_fb_helper.c +++ b/drivers/gpu/drm/drm_fb_helper.c @@ -31,9 +31,7 @@ #include <linux/console.h> #include <linux/export.h> -#include <linux/pci.h> #include <linux/sysrq.h> -#include <linux/vga_switcheroo.h> #include <drm/drm_atomic.h> #include <drm/drm_drv.h> @@ -566,11 +564,6 @@ EXPORT_SYMBOL(drm_fb_helper_release_info); */ void drm_fb_helper_unregister_info(struct drm_fb_helper *fb_helper) { - struct fb_info *info = fb_helper->info; - struct device *dev = info->device; - - if (dev_is_pci(dev)) - vga_switcheroo_client_fb_set(to_pci_dev(dev), NULL); unregister_framebuffer(fb_helper->info); } EXPORT_SYMBOL(drm_fb_helper_unregister_info); @@ -1632,7 +1625,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper) struct drm_client_dev *client = &fb_helper->client; struct drm_device *dev = fb_helper->dev; struct drm_fb_helper_surface_size sizes; - struct fb_info *info; int ret; if (drm_WARN_ON(dev, !dev->driver->fbdev_probe)) @@ -1653,12 +1645,6 @@ static int drm_fb_helper_single_fb_probe(struct drm_fb_helper *fb_helper) strcpy(fb_helper->fb->comm, "[fbcon]"); - info = fb_helper->info; - - /* Set the fb info for vgaswitcheroo clients. Does nothing otherwise. */ - if (dev_is_pci(info->device)) - vga_switcheroo_client_fb_set(to_pci_dev(info->device), info); - return 0; } diff --git a/drivers/gpu/drm/drm_vblank.c b/drivers/gpu/drm/drm_vblank.c index 46f59883183d..67eae477cbd7 100644 --- a/drivers/gpu/drm/drm_vblank.c +++ b/drivers/gpu/drm/drm_vblank.c @@ -794,10 +794,8 @@ drm_crtc_vblank_helper_get_vblank_timestamp_internal( ts_vblank_time = ktime_to_timespec64(*vblank_time); drm_dbg_vbl(dev, - "crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n", - pipe, hpos, vpos, - (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000, - (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000, + "crtc %u : v p(%d,%d)@ %ptSp -> %ptSp [e %d us, %d rep]\n", + pipe, hpos, vpos, &ts_etime, &ts_vblank_time, duration_ns / 1000, i); return true; diff --git a/drivers/gpu/drm/i915/display/intel_display.c b/drivers/gpu/drm/i915/display/intel_display.c index 5dca7f96b425..0d527cf22866 100644 --- a/drivers/gpu/drm/i915/display/intel_display.c +++ b/drivers/gpu/drm/i915/display/intel_display.c @@ -5964,6 +5964,14 @@ static int intel_async_flip_check_uapi(struct intel_atomic_state *state, return -EINVAL; } + /* FIXME: selective fetch should be disabled for async flips */ + if (new_crtc_state->enable_psr2_sel_fetch) { + drm_dbg_kms(display->drm, + "[CRTC:%d:%s] async flip disallowed with PSR2 selective fetch\n", + crtc->base.base.id, crtc->base.name); + return -EINVAL; + } + for_each_oldnew_intel_plane_in_state(state, plane, old_plane_state, new_plane_state, i) { if (plane->pipe != crtc->pipe) diff --git a/drivers/gpu/drm/i915/display/intel_psr.c b/drivers/gpu/drm/i915/display/intel_psr.c index 4619237f1346..6d9c95e5c025 100644 --- a/drivers/gpu/drm/i915/display/intel_psr.c +++ b/drivers/gpu/drm/i915/display/intel_psr.c @@ -1256,12 +1256,6 @@ static bool intel_psr2_sel_fetch_config_valid(struct intel_dp *intel_dp, return false; } - if (crtc_state->uapi.async_flip) { - drm_dbg_kms(display->drm, - "PSR2 sel fetch not enabled, async flip enabled\n"); - return false; - } - return crtc_state->enable_psr2_sel_fetch = true; } diff --git a/drivers/gpu/drm/imagination/pvr_device.h b/drivers/gpu/drm/imagination/pvr_device.h index ab8f56ae15df..ec53ff275541 100644 --- a/drivers/gpu/drm/imagination/pvr_device.h +++ b/drivers/gpu/drm/imagination/pvr_device.h @@ -146,6 +146,14 @@ struct pvr_device { */ struct clk *mem_clk; + /** + * @power: Optional power domain devices. + * + * On platforms with more than one power domain for the GPU, they are + * stored here in @domain_devs, along with links between them in + * @domain_links. The size of @domain_devs is given by @domain_count, + * while the size of @domain_links is (2 * @domain_count) - 1. + */ struct pvr_device_power { struct device **domain_devs; struct device_link **domain_links; diff --git a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c index 071bcdea80f7..19b470968f4d 100644 --- a/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c +++ b/drivers/gpu/drm/msm/disp/msm_disp_snapshot_util.c @@ -82,8 +82,7 @@ void msm_disp_state_print(struct msm_disp_state *state, struct drm_printer *p) drm_printf(p, "kernel: " UTS_RELEASE "\n"); drm_printf(p, "module: " KBUILD_MODNAME "\n"); drm_printf(p, "dpu devcoredump\n"); - drm_printf(p, "time: %lld.%09ld\n", - state->time.tv_sec, state->time.tv_nsec); + drm_printf(p, "time: %ptSp\n", &state->time); list_for_each_entry_safe(block, tmp, &state->blocks, node) { drm_printf(p, "====================%s================\n", block->name); diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index 17759abc46d7..a4251afe4541 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -197,8 +197,7 @@ static ssize_t msm_gpu_devcoredump_read(char *buffer, loff_t offset, drm_printf(&p, "---\n"); drm_printf(&p, "kernel: " UTS_RELEASE "\n"); drm_printf(&p, "module: " KBUILD_MODNAME "\n"); - drm_printf(&p, "time: %lld.%09ld\n", - state->time.tv_sec, state->time.tv_nsec); + drm_printf(&p, "time: %ptSp\n", &state->time); if (state->comm) drm_printf(&p, "comm: %s\n", state->comm); if (state->cmd) diff --git a/drivers/gpu/drm/sti/sti_vtg.c b/drivers/gpu/drm/sti/sti_vtg.c index ee81691b3203..ce6bc7e7b135 100644 --- a/drivers/gpu/drm/sti/sti_vtg.c +++ b/drivers/gpu/drm/sti/sti_vtg.c @@ -143,12 +143,17 @@ struct sti_vtg { struct sti_vtg *of_vtg_find(struct device_node *np) { struct platform_device *pdev; + struct sti_vtg *vtg; pdev = of_find_device_by_node(np); if (!pdev) return NULL; - return (struct sti_vtg *)platform_get_drvdata(pdev); + vtg = platform_get_drvdata(pdev); + + put_device(&pdev->dev); + + return vtg; } static void vtg_reset(struct sti_vtg *vtg) diff --git a/drivers/gpu/drm/xe/xe_devcoredump.c b/drivers/gpu/drm/xe/xe_devcoredump.c index 203e3038cc81..d444eda65ca6 100644 --- a/drivers/gpu/drm/xe/xe_devcoredump.c +++ b/drivers/gpu/drm/xe/xe_devcoredump.c @@ -106,9 +106,9 @@ static ssize_t __xe_devcoredump_read(char *buffer, ssize_t count, drm_puts(&p, "module: " KBUILD_MODNAME "\n"); ts = ktime_to_timespec64(ss->snapshot_time); - drm_printf(&p, "Snapshot time: %lld.%09ld\n", ts.tv_sec, ts.tv_nsec); + drm_printf(&p, "Snapshot time: %ptSp\n", &ts); ts = ktime_to_timespec64(ss->boot_time); - drm_printf(&p, "Uptime: %lld.%09ld\n", ts.tv_sec, ts.tv_nsec); + drm_printf(&p, "Uptime: %ptSp\n", &ts); drm_printf(&p, "Process: %s [%d]\n", ss->process_name, ss->pid); xe_device_snapshot_print(xe, &p); diff --git a/drivers/gpu/drm/xe/xe_gt_clock.c b/drivers/gpu/drm/xe/xe_gt_clock.c index 4f011d1573c6..f65d1edd0567 100644 --- a/drivers/gpu/drm/xe/xe_gt_clock.c +++ b/drivers/gpu/drm/xe/xe_gt_clock.c @@ -93,11 +93,6 @@ int xe_gt_clock_init(struct xe_gt *gt) return 0; } -static u64 div_u64_roundup(u64 n, u32 d) -{ - return div_u64(n + d - 1, d); -} - /** * xe_gt_clock_interval_to_ms - Convert sampled GT clock ticks to msec * @@ -108,5 +103,5 @@ static u64 div_u64_roundup(u64 n, u32 d) */ u64 xe_gt_clock_interval_to_ms(struct xe_gt *gt, u64 count) { - return div_u64_roundup(count * MSEC_PER_SEC, gt->info.reference_clock); + return mul_u64_u32_div(count, MSEC_PER_SEC, gt->info.reference_clock); } diff --git a/drivers/gpu/drm/xe/xe_guc_ct.c b/drivers/gpu/drm/xe/xe_guc_ct.c index 283d846c3512..b7afe8e983cb 100644 --- a/drivers/gpu/drm/xe/xe_guc_ct.c +++ b/drivers/gpu/drm/xe/xe_guc_ct.c @@ -226,6 +226,12 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) xe_gt_assert(gt, !(guc_ct_size() % PAGE_SIZE)); + err = drmm_mutex_init(&xe->drm, &ct->lock); + if (err) + return err; + + primelockdep(ct); + ct->g2h_wq = alloc_ordered_workqueue("xe-g2h-wq", WQ_MEM_RECLAIM); if (!ct->g2h_wq) return -ENOMEM; @@ -237,16 +243,13 @@ int xe_guc_ct_init_noalloc(struct xe_guc_ct *ct) #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) spin_lock_init(&ct->dead.lock); INIT_WORK(&ct->dead.worker, ct_dead_worker_func); +#if IS_ENABLED(CONFIG_DRM_XE_DEBUG_GUC) + stack_depot_init(); +#endif #endif init_waitqueue_head(&ct->wq); init_waitqueue_head(&ct->g2h_fence_wq); - err = drmm_mutex_init(&xe->drm, &ct->lock); - if (err) - return err; - - primelockdep(ct); - err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct); if (err) return err; diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c index e3b2bd417c46..1d8d8d00e4e0 100644 --- a/drivers/hv/mshv_root_main.c +++ b/drivers/hv/mshv_root_main.c @@ -29,6 +29,7 @@ #include <linux/crash_dump.h> #include <linux/panic_notifier.h> #include <linux/vmalloc.h> +#include <linux/rseq.h> #include "mshv_eventfd.h" #include "mshv.h" @@ -560,6 +561,8 @@ static long mshv_run_vp_with_root_scheduler(struct mshv_vp *vp) } } while (!vp->run.flags.intercept_suspend); + rseq_virt_userspace_exit(); + return ret; } @@ -1870,8 +1873,6 @@ mshv_ioctl_create_partition(void __user *user_arg, struct device *module_dev) struct hv_partition_creation_properties creation_properties = {}; union hv_partition_isolation_properties isolation_properties = {}; struct mshv_partition *partition; - struct file *file; - int fd; long ret; if (copy_from_user(&args, user_arg, sizeof(args))) @@ -1938,29 +1939,13 @@ mshv_ioctl_create_partition(void __user *user_arg, struct device *module_dev) goto delete_partition; ret = mshv_init_async_handler(partition); - if (ret) - goto remove_partition; - - fd = get_unused_fd_flags(O_CLOEXEC); - if (fd < 0) { - ret = fd; - goto remove_partition; - } - - file = anon_inode_getfile("mshv_partition", &mshv_partition_fops, - partition, O_RDWR); - if (IS_ERR(file)) { - ret = PTR_ERR(file); - goto put_fd; + if (!ret) { + ret = FD_ADD(O_CLOEXEC, anon_inode_getfile("mshv_partition", + &mshv_partition_fops, + partition, O_RDWR)); + if (ret >= 0) + return ret; } - - fd_install(fd, file); - - return fd; - -put_fd: - put_unused_fd(fd); -remove_partition: remove_partition(partition); delete_partition: hv_call_delete_partition(partition->pt_id); diff --git a/drivers/hwtracing/coresight/coresight-trbe.c b/drivers/hwtracing/coresight/coresight-trbe.c index 43643d2c5bdd..9f64f463339d 100644 --- a/drivers/hwtracing/coresight/coresight-trbe.c +++ b/drivers/hwtracing/coresight/coresight-trbe.c @@ -1474,9 +1474,10 @@ static void arm_trbe_remove_cpuhp(struct trbe_drvdata *drvdata) static int arm_trbe_probe_irq(struct platform_device *pdev, struct trbe_drvdata *drvdata) { + const struct cpumask *affinity; int ret; - drvdata->irq = platform_get_irq(pdev, 0); + drvdata->irq = platform_get_irq_affinity(pdev, 0, &affinity); if (drvdata->irq < 0) { pr_err("IRQ not found for the platform device\n"); return drvdata->irq; @@ -1487,14 +1488,14 @@ static int arm_trbe_probe_irq(struct platform_device *pdev, return -EINVAL; } - if (irq_get_percpu_devid_partition(drvdata->irq, &drvdata->supported_cpus)) - return -EINVAL; + cpumask_copy(&drvdata->supported_cpus, affinity); drvdata->handle = alloc_percpu(struct perf_output_handle *); if (!drvdata->handle) return -ENOMEM; - ret = request_percpu_irq(drvdata->irq, arm_trbe_irq_handler, DRVNAME, drvdata->handle); + ret = request_percpu_irq_affinity(drvdata->irq, arm_trbe_irq_handler, DRVNAME, + affinity, drvdata->handle); if (ret) { free_percpu(drvdata->handle); return ret; diff --git a/drivers/iio/accel/adxl355_core.c b/drivers/iio/accel/adxl355_core.c index 2e00fd51b4d5..5fc7f814b907 100644 --- a/drivers/iio/accel/adxl355_core.c +++ b/drivers/iio/accel/adxl355_core.c @@ -56,6 +56,8 @@ #define ADXL355_POWER_CTL_DRDY_MSK BIT(2) #define ADXL355_SELF_TEST_REG 0x2E #define ADXL355_RESET_REG 0x2F +#define ADXL355_BASE_ADDR_SHADOW_REG 0x50 +#define ADXL355_SHADOW_REG_COUNT 5 #define ADXL355_DEVID_AD_VAL 0xAD #define ADXL355_DEVID_MST_VAL 0x1D @@ -294,7 +296,12 @@ static void adxl355_fill_3db_frequency_table(struct adxl355_data *data) static int adxl355_setup(struct adxl355_data *data) { unsigned int regval; + int retries = 5; /* the number is chosen based on empirical reasons */ int ret; + u8 *shadow_regs __free(kfree) = kzalloc(ADXL355_SHADOW_REG_COUNT, GFP_KERNEL); + + if (!shadow_regs) + return -ENOMEM; ret = regmap_read(data->regmap, ADXL355_DEVID_AD_REG, ®val); if (ret) @@ -321,14 +328,41 @@ static int adxl355_setup(struct adxl355_data *data) if (regval != ADXL355_PARTID_VAL) dev_warn(data->dev, "Invalid DEV ID 0x%02x\n", regval); - /* - * Perform a software reset to make sure the device is in a consistent - * state after start-up. - */ - ret = regmap_write(data->regmap, ADXL355_RESET_REG, ADXL355_RESET_CODE); + /* Read shadow registers to be compared after reset */ + ret = regmap_bulk_read(data->regmap, + ADXL355_BASE_ADDR_SHADOW_REG, + shadow_regs, ADXL355_SHADOW_REG_COUNT); if (ret) return ret; + do { + if (--retries == 0) { + dev_err(data->dev, "Shadow registers mismatch\n"); + return -EIO; + } + + /* + * Perform a software reset to make sure the device is in a consistent + * state after start-up. + */ + ret = regmap_write(data->regmap, ADXL355_RESET_REG, + ADXL355_RESET_CODE); + if (ret) + return ret; + + /* Wait at least 5ms after software reset */ + usleep_range(5000, 10000); + + /* Read shadow registers for comparison */ + ret = regmap_bulk_read(data->regmap, + ADXL355_BASE_ADDR_SHADOW_REG, + data->buffer.buf, + ADXL355_SHADOW_REG_COUNT); + if (ret) + return ret; + } while (memcmp(shadow_regs, data->buffer.buf, + ADXL355_SHADOW_REG_COUNT)); + ret = regmap_update_bits(data->regmap, ADXL355_POWER_CTL_REG, ADXL355_POWER_CTL_DRDY_MSK, FIELD_PREP(ADXL355_POWER_CTL_DRDY_MSK, 1)); diff --git a/drivers/iio/accel/bmc150-accel-core.c b/drivers/iio/accel/bmc150-accel-core.c index 3c5d1560b163..42ccf0316ce5 100644 --- a/drivers/iio/accel/bmc150-accel-core.c +++ b/drivers/iio/accel/bmc150-accel-core.c @@ -523,6 +523,10 @@ static int bmc150_accel_set_interrupt(struct bmc150_accel_data *data, int i, const struct bmc150_accel_interrupt_info *info = intr->info; int ret; + /* We do not always have an IRQ */ + if (data->irq <= 0) + return 0; + if (state) { if (atomic_inc_return(&intr->users) > 1) return 0; @@ -1696,6 +1700,7 @@ int bmc150_accel_core_probe(struct device *dev, struct regmap *regmap, int irq, } if (irq > 0) { + data->irq = irq; ret = devm_request_threaded_irq(dev, irq, bmc150_accel_irq_handler, bmc150_accel_irq_thread_handler, diff --git a/drivers/iio/accel/bmc150-accel.h b/drivers/iio/accel/bmc150-accel.h index 7a7baf52e595..e8f26198359f 100644 --- a/drivers/iio/accel/bmc150-accel.h +++ b/drivers/iio/accel/bmc150-accel.h @@ -58,6 +58,7 @@ enum bmc150_accel_trigger_id { struct bmc150_accel_data { struct regmap *regmap; + int irq; struct regulator_bulk_data regulators[2]; struct bmc150_accel_interrupt interrupts[BMC150_ACCEL_INTERRUPTS]; struct bmc150_accel_trigger triggers[BMC150_ACCEL_TRIGGERS]; diff --git a/drivers/iio/adc/ad4030.c b/drivers/iio/adc/ad4030.c index 1bc2f9a22470..d8bee6a4215a 100644 --- a/drivers/iio/adc/ad4030.c +++ b/drivers/iio/adc/ad4030.c @@ -385,7 +385,7 @@ static int ad4030_get_chan_scale(struct iio_dev *indio_dev, struct ad4030_state *st = iio_priv(indio_dev); const struct iio_scan_type *scan_type; - scan_type = iio_get_current_scan_type(indio_dev, st->chip->channels); + scan_type = iio_get_current_scan_type(indio_dev, chan); if (IS_ERR(scan_type)) return PTR_ERR(scan_type); diff --git a/drivers/iio/adc/ad7124.c b/drivers/iio/adc/ad7124.c index 910b40393f77..61623cc6cb25 100644 --- a/drivers/iio/adc/ad7124.c +++ b/drivers/iio/adc/ad7124.c @@ -1525,10 +1525,6 @@ static int __ad7124_calibrate_all(struct ad7124_state *st, struct iio_dev *indio int ret, i; for (i = 0; i < st->num_channels; i++) { - - if (indio_dev->channels[i].type != IIO_VOLTAGE) - continue; - /* * For calibration the OFFSET register should hold its reset default * value. For the GAIN register there is no such requirement but @@ -1539,6 +1535,14 @@ static int __ad7124_calibrate_all(struct ad7124_state *st, struct iio_dev *indio st->channels[i].cfg.calibration_gain = st->gain_default; /* + * Only the main voltage input channels are important enough + * to be automatically calibrated here. For everything else, + * just use the default values set above. + */ + if (indio_dev->channels[i].type != IIO_VOLTAGE) + continue; + + /* * Full-scale calibration isn't supported at gain 1, so skip in * that case. Note that untypically full-scale calibration has * to happen before zero-scale calibration. This only applies to diff --git a/drivers/iio/adc/ad7280a.c b/drivers/iio/adc/ad7280a.c index dda2986ccda0..50a6ff7c8b1c 100644 --- a/drivers/iio/adc/ad7280a.c +++ b/drivers/iio/adc/ad7280a.c @@ -541,7 +541,7 @@ static ssize_t ad7280_store_balance_timer(struct iio_dev *indio_dev, int val, val2; int ret; - ret = iio_str_to_fixpoint(buf, 1000, &val, &val2); + ret = iio_str_to_fixpoint(buf, 100, &val, &val2); if (ret) return ret; diff --git a/drivers/iio/adc/ad7380.c b/drivers/iio/adc/ad7380.c index fa251dc1aae6..bfd908deefc0 100644 --- a/drivers/iio/adc/ad7380.c +++ b/drivers/iio/adc/ad7380.c @@ -1227,6 +1227,14 @@ static int ad7380_offload_buffer_postenable(struct iio_dev *indio_dev) if (ret) return ret; + /* + * When the sequencer is required to read all channels, we need to + * trigger twice per sample period in order to read one complete set + * of samples. + */ + if (st->seq) + config.periodic.frequency_hz *= 2; + ret = spi_offload_trigger_enable(st->offload, st->offload_trigger, &config); if (ret) spi_unoptimize_message(&st->offload_msg); diff --git a/drivers/iio/adc/rtq6056.c b/drivers/iio/adc/rtq6056.c index ad9738228b7f..2bf3a09ac6b0 100644 --- a/drivers/iio/adc/rtq6056.c +++ b/drivers/iio/adc/rtq6056.c @@ -300,7 +300,7 @@ static int rtq6056_adc_read_channel(struct rtq6056_priv *priv, return IIO_VAL_INT; case RTQ6056_REG_SHUNTVOLT: case RTQ6056_REG_CURRENT: - *val = sign_extend32(regval, 16); + *val = sign_extend32(regval, 15); return IIO_VAL_INT; default: return -EINVAL; diff --git a/drivers/iio/adc/stm32-dfsdm-adc.c b/drivers/iio/adc/stm32-dfsdm-adc.c index 74b1b4dc6e81..9664b9bd75d4 100644 --- a/drivers/iio/adc/stm32-dfsdm-adc.c +++ b/drivers/iio/adc/stm32-dfsdm-adc.c @@ -725,9 +725,8 @@ static int stm32_dfsdm_generic_channel_parse_of(struct stm32_dfsdm *dfsdm, } df_ch->src = val; - ret = fwnode_property_read_u32(node, "st,adc-alt-channel", &df_ch->alt_si); - if (ret != -EINVAL) - df_ch->alt_si = 0; + if (fwnode_property_present(node, "st,adc-alt-channel")) + df_ch->alt_si = 1; if (adc->dev_data->type == DFSDM_IIO) { backend = devm_iio_backend_fwnode_get(&indio_dev->dev, NULL, node); diff --git a/drivers/iio/buffer/industrialio-buffer-dma.c b/drivers/iio/buffer/industrialio-buffer-dma.c index ee294a775e8a..7a7a9d37339b 100644 --- a/drivers/iio/buffer/industrialio-buffer-dma.c +++ b/drivers/iio/buffer/industrialio-buffer-dma.c @@ -786,6 +786,12 @@ out_end_signalling: } EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_enqueue_dmabuf, "IIO_DMA_BUFFER"); +struct device *iio_dma_buffer_get_dma_dev(struct iio_buffer *buffer) +{ + return iio_buffer_to_queue(buffer)->dev; +} +EXPORT_SYMBOL_NS_GPL(iio_dma_buffer_get_dma_dev, "IIO_DMA_BUFFER"); + void iio_dma_buffer_lock_queue(struct iio_buffer *buffer) { struct iio_dma_buffer_queue *queue = iio_buffer_to_queue(buffer); diff --git a/drivers/iio/buffer/industrialio-buffer-dmaengine.c b/drivers/iio/buffer/industrialio-buffer-dmaengine.c index e9d9a7d39fe1..27dd56334345 100644 --- a/drivers/iio/buffer/industrialio-buffer-dmaengine.c +++ b/drivers/iio/buffer/industrialio-buffer-dmaengine.c @@ -177,6 +177,8 @@ static const struct iio_buffer_access_funcs iio_dmaengine_buffer_ops = { .lock_queue = iio_dma_buffer_lock_queue, .unlock_queue = iio_dma_buffer_unlock_queue, + .get_dma_dev = iio_dma_buffer_get_dma_dev, + .modes = INDIO_BUFFER_HARDWARE, .flags = INDIO_BUFFER_FLAG_FIXED_WATERMARK, }; diff --git a/drivers/iio/common/ssp_sensors/ssp_dev.c b/drivers/iio/common/ssp_sensors/ssp_dev.c index 1e167dc673ca..da09c9f3ceb6 100644 --- a/drivers/iio/common/ssp_sensors/ssp_dev.c +++ b/drivers/iio/common/ssp_sensors/ssp_dev.c @@ -503,7 +503,7 @@ static int ssp_probe(struct spi_device *spi) ret = spi_setup(spi); if (ret < 0) { dev_err(&spi->dev, "Failed to setup spi\n"); - return ret; + goto err_setup_spi; } data->fw_dl_state = SSP_FW_DL_STATE_NONE; @@ -568,6 +568,8 @@ err_read_reg: err_setup_irq: mutex_destroy(&data->pending_lock); mutex_destroy(&data->comm_lock); +err_setup_spi: + mfd_remove_devices(&spi->dev); dev_err(&spi->dev, "Probe failed!\n"); diff --git a/drivers/iio/humidity/hdc3020.c b/drivers/iio/humidity/hdc3020.c index ffb25596d3a8..78b2c171c8da 100644 --- a/drivers/iio/humidity/hdc3020.c +++ b/drivers/iio/humidity/hdc3020.c @@ -72,6 +72,9 @@ #define HDC3020_MAX_TEMP_HYST_MICRO 164748607 #define HDC3020_MAX_HUM_MICRO 99220264 +/* Divide 65535 from the datasheet by 5 to avoid overflows */ +#define HDC3020_THRESH_FRACTION (65535 / 5) + struct hdc3020_data { struct i2c_client *client; struct gpio_desc *reset_gpio; @@ -301,9 +304,9 @@ static int hdc3020_read_raw(struct iio_dev *indio_dev, case IIO_CHAN_INFO_SCALE: *val2 = 65536; if (chan->type == IIO_TEMP) - *val = 175; + *val = 175 * MILLI; else - *val = 100; + *val = 100 * MILLI; return IIO_VAL_FRACTIONAL; case IIO_CHAN_INFO_OFFSET: @@ -376,15 +379,18 @@ static int hdc3020_thresh_get_temp(u16 thresh) int temp; /* - * Get the temperature threshold from 9 LSBs, shift them to get - * the truncated temperature threshold representation and - * calculate the threshold according to the formula in the - * datasheet. Result is degree celsius scaled by 65535. + * Get the temperature threshold from 9 LSBs, shift them to get the + * truncated temperature threshold representation and calculate the + * threshold according to the explicit formula in the datasheet: + * T(C) = -45 + (175 * temp) / 65535. + * Additionally scale by HDC3020_THRESH_FRACTION to avoid precision loss + * when calculating threshold and hysteresis values. Result is degree + * celsius scaled by HDC3020_THRESH_FRACTION. */ temp = FIELD_GET(HDC3020_THRESH_TEMP_MASK, thresh) << HDC3020_THRESH_TEMP_TRUNC_SHIFT; - return -2949075 + (175 * temp); + return -2949075 / 5 + (175 / 5 * temp); } static int hdc3020_thresh_get_hum(u16 thresh) @@ -394,13 +400,16 @@ static int hdc3020_thresh_get_hum(u16 thresh) /* * Get the humidity threshold from 7 MSBs, shift them to get the * truncated humidity threshold representation and calculate the - * threshold according to the formula in the datasheet. Result is - * percent scaled by 65535. + * threshold according to the explicit formula in the datasheet: + * RH(%) = 100 * hum / 65535. + * Additionally scale by HDC3020_THRESH_FRACTION to avoid precision loss + * when calculating threshold and hysteresis values. Result is percent + * scaled by HDC3020_THRESH_FRACTION. */ hum = FIELD_GET(HDC3020_THRESH_HUM_MASK, thresh) << HDC3020_THRESH_HUM_TRUNC_SHIFT; - return hum * 100; + return hum * 100 / 5; } static u16 hdc3020_thresh_set_temp(int s_temp, u16 curr_thresh) @@ -455,8 +464,8 @@ int hdc3020_thresh_clr(s64 s_thresh, s64 s_hyst, enum iio_event_direction dir) else s_clr = s_thresh + s_hyst; - /* Divide by 65535 to get units of micro */ - return div_s64(s_clr, 65535); + /* Divide by HDC3020_THRESH_FRACTION to get units of micro */ + return div_s64(s_clr, HDC3020_THRESH_FRACTION); } static int _hdc3020_write_thresh(struct hdc3020_data *data, u16 reg, u16 val) @@ -507,7 +516,7 @@ static int hdc3020_write_thresh(struct iio_dev *indio_dev, clr = ret; /* Scale value to include decimal part into calculations */ - s_val = (val < 0) ? (val * 1000000 - val2) : (val * 1000000 + val2); + s_val = (val < 0) ? (val * 1000 - val2) : (val * 1000 + val2); switch (chan->type) { case IIO_TEMP: switch (info) { @@ -523,7 +532,8 @@ static int hdc3020_write_thresh(struct iio_dev *indio_dev, /* Calculate old hysteresis */ s_thresh = (s64)hdc3020_thresh_get_temp(thresh) * 1000000; s_clr = (s64)hdc3020_thresh_get_temp(clr) * 1000000; - s_hyst = div_s64(abs(s_thresh - s_clr), 65535); + s_hyst = div_s64(abs(s_thresh - s_clr), + HDC3020_THRESH_FRACTION); /* Set new threshold */ thresh = reg_val; /* Set old hysteresis */ @@ -532,16 +542,17 @@ static int hdc3020_write_thresh(struct iio_dev *indio_dev, case IIO_EV_INFO_HYSTERESIS: /* * Function hdc3020_thresh_get_temp returns temperature - * in degree celsius scaled by 65535. Scale by 1000000 - * to be able to subtract scaled hysteresis value. + * in degree celsius scaled by HDC3020_THRESH_FRACTION. + * Scale by 1000000 to be able to subtract scaled + * hysteresis value. */ s_thresh = (s64)hdc3020_thresh_get_temp(thresh) * 1000000; /* * Units of s_val are in micro degree celsius, scale by - * 65535 to get same units as s_thresh. + * HDC3020_THRESH_FRACTION to get same units as s_thresh. */ s_val = min(abs(s_val), HDC3020_MAX_TEMP_HYST_MICRO); - s_hyst = (s64)s_val * 65535; + s_hyst = (s64)s_val * HDC3020_THRESH_FRACTION; s_clr = hdc3020_thresh_clr(s_thresh, s_hyst, dir); s_clr = max(s_clr, HDC3020_MIN_TEMP_MICRO); s_clr = min(s_clr, HDC3020_MAX_TEMP_MICRO); @@ -565,7 +576,8 @@ static int hdc3020_write_thresh(struct iio_dev *indio_dev, /* Calculate old hysteresis */ s_thresh = (s64)hdc3020_thresh_get_hum(thresh) * 1000000; s_clr = (s64)hdc3020_thresh_get_hum(clr) * 1000000; - s_hyst = div_s64(abs(s_thresh - s_clr), 65535); + s_hyst = div_s64(abs(s_thresh - s_clr), + HDC3020_THRESH_FRACTION); /* Set new threshold */ thresh = reg_val; /* Try to set old hysteresis */ @@ -574,15 +586,16 @@ static int hdc3020_write_thresh(struct iio_dev *indio_dev, case IIO_EV_INFO_HYSTERESIS: /* * Function hdc3020_thresh_get_hum returns relative - * humidity in percent scaled by 65535. Scale by 1000000 - * to be able to subtract scaled hysteresis value. + * humidity in percent scaled by HDC3020_THRESH_FRACTION. + * Scale by 1000000 to be able to subtract scaled + * hysteresis value. */ s_thresh = (s64)hdc3020_thresh_get_hum(thresh) * 1000000; /* - * Units of s_val are in micro percent, scale by 65535 - * to get same units as s_thresh. + * Units of s_val are in micro percent, scale by + * HDC3020_THRESH_FRACTION to get same units as s_thresh. */ - s_hyst = (s64)s_val * 65535; + s_hyst = (s64)s_val * HDC3020_THRESH_FRACTION; s_clr = hdc3020_thresh_clr(s_thresh, s_hyst, dir); s_clr = max(s_clr, 0); s_clr = min(s_clr, HDC3020_MAX_HUM_MICRO); @@ -630,7 +643,7 @@ static int hdc3020_read_thresh(struct iio_dev *indio_dev, thresh = hdc3020_thresh_get_temp(ret); switch (info) { case IIO_EV_INFO_VALUE: - *val = thresh; + *val = thresh * MILLI; break; case IIO_EV_INFO_HYSTERESIS: ret = hdc3020_read_be16(data, reg_clr); @@ -638,18 +651,18 @@ static int hdc3020_read_thresh(struct iio_dev *indio_dev, return ret; clr = hdc3020_thresh_get_temp(ret); - *val = abs(thresh - clr); + *val = abs(thresh - clr) * MILLI; break; default: return -EOPNOTSUPP; } - *val2 = 65535; + *val2 = HDC3020_THRESH_FRACTION; return IIO_VAL_FRACTIONAL; case IIO_HUMIDITYRELATIVE: thresh = hdc3020_thresh_get_hum(ret); switch (info) { case IIO_EV_INFO_VALUE: - *val = thresh; + *val = thresh * MILLI; break; case IIO_EV_INFO_HYSTERESIS: ret = hdc3020_read_be16(data, reg_clr); @@ -657,12 +670,12 @@ static int hdc3020_read_thresh(struct iio_dev *indio_dev, return ret; clr = hdc3020_thresh_get_hum(ret); - *val = abs(thresh - clr); + *val = abs(thresh - clr) * MILLI; break; default: return -EOPNOTSUPP; } - *val2 = 65535; + *val2 = HDC3020_THRESH_FRACTION; return IIO_VAL_FRACTIONAL; default: return -EOPNOTSUPP; diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h index c225b246c8a5..381b016fa524 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx.h @@ -192,6 +192,22 @@ struct st_lsm6dsx_fifo_ops { * @fifo_en: Hw timer FIFO enable register info (addr + mask). * @decimator: Hw timer FIFO decimator register info (addr + mask). * @freq_fine: Difference in % of ODR with respect to the typical. + * @ts_sensitivity: Nominal timestamp sensitivity. + * @ts_trim_coeff: Coefficient for calculating the calibrated timestamp gain. + * This coefficient comes into play when linearizing the formula + * used to calculate the calibrated timestamp (please see the + * relevant formula in the AN for the specific IMU). + * For example, in the case of LSM6DSO we have: + * + * 1 / (1 + x) ~= 1 - x (Taylor’s Series) + * ttrim[s] = 1 / (40000 * (1 + 0.0015 * val)) (from AN5192) + * ttrim[ns] ~= 25000 - 37.5 * val + * ttrim[ns] ~= 25000 - (37500 * val) / 1000 + * + * so, replacing ts_sensitivity = 25000 and + * ts_trim_coeff = 37500 + * + * ttrim[ns] ~= ts_sensitivity - (ts_trim_coeff * val) / 1000 */ struct st_lsm6dsx_hw_ts_settings { struct st_lsm6dsx_reg timer_en; @@ -199,6 +215,8 @@ struct st_lsm6dsx_hw_ts_settings { struct st_lsm6dsx_reg fifo_en; struct st_lsm6dsx_reg decimator; u8 freq_fine; + u16 ts_sensitivity; + u16 ts_trim_coeff; }; /** @@ -252,6 +270,15 @@ struct st_lsm6dsx_event_settings { u8 wakeup_src_x_mask; }; +enum st_lsm6dsx_sensor_id { + ST_LSM6DSX_ID_GYRO, + ST_LSM6DSX_ID_ACC, + ST_LSM6DSX_ID_EXT0, + ST_LSM6DSX_ID_EXT1, + ST_LSM6DSX_ID_EXT2, + ST_LSM6DSX_ID_MAX +}; + enum st_lsm6dsx_ext_sensor_id { ST_LSM6DSX_ID_MAGN, }; @@ -337,23 +364,14 @@ struct st_lsm6dsx_settings { struct st_lsm6dsx_odr_table_entry odr_table[2]; struct st_lsm6dsx_samples_to_discard samples_to_discard[2]; struct st_lsm6dsx_fs_table_entry fs_table[2]; - struct st_lsm6dsx_reg decimator[ST_LSM6DSX_MAX_ID]; - struct st_lsm6dsx_reg batch[ST_LSM6DSX_MAX_ID]; + struct st_lsm6dsx_reg decimator[ST_LSM6DSX_ID_MAX]; + struct st_lsm6dsx_reg batch[2]; struct st_lsm6dsx_fifo_ops fifo_ops; struct st_lsm6dsx_hw_ts_settings ts_settings; struct st_lsm6dsx_shub_settings shub_settings; struct st_lsm6dsx_event_settings event_settings; }; -enum st_lsm6dsx_sensor_id { - ST_LSM6DSX_ID_GYRO, - ST_LSM6DSX_ID_ACC, - ST_LSM6DSX_ID_EXT0, - ST_LSM6DSX_ID_EXT1, - ST_LSM6DSX_ID_EXT2, - ST_LSM6DSX_ID_MAX, -}; - enum st_lsm6dsx_fifo_mode { ST_LSM6DSX_FIFO_BYPASS = 0x0, ST_LSM6DSX_FIFO_CONT = 0x6, diff --git a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c index d8cb4b0218d5..a2daf0c14d96 100644 --- a/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c +++ b/drivers/iio/imu/st_lsm6dsx/st_lsm6dsx_core.c @@ -94,8 +94,6 @@ #define ST_LSM6DSX_REG_WHOAMI_ADDR 0x0f -#define ST_LSM6DSX_TS_SENSITIVITY 25000UL /* 25us */ - static const struct iio_chan_spec st_lsm6dsx_acc_channels[] = { ST_LSM6DSX_CHANNEL_ACC(IIO_ACCEL, 0x28, IIO_MOD_X, 0), ST_LSM6DSX_CHANNEL_ACC(IIO_ACCEL, 0x2a, IIO_MOD_Y, 1), @@ -983,6 +981,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = { .mask = GENMASK(7, 6), }, .freq_fine = 0x63, + .ts_sensitivity = 25000, + .ts_trim_coeff = 37500, }, .shub_settings = { .page_mux = { @@ -1196,6 +1196,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = { .mask = GENMASK(7, 6), }, .freq_fine = 0x63, + .ts_sensitivity = 25000, + .ts_trim_coeff = 37500, }, .event_settings = { .enable_reg = { @@ -1371,6 +1373,8 @@ static const struct st_lsm6dsx_settings st_lsm6dsx_sensor_settings[] = { .mask = GENMASK(7, 6), }, .freq_fine = 0x4f, + .ts_sensitivity = 21701, + .ts_trim_coeff = 28212, }, .shub_settings = { .page_mux = { @@ -2248,20 +2252,13 @@ static int st_lsm6dsx_init_hw_timer(struct st_lsm6dsx_hw *hw) } /* calibrate timestamp sensitivity */ - hw->ts_gain = ST_LSM6DSX_TS_SENSITIVITY; + hw->ts_gain = ts_settings->ts_sensitivity; if (ts_settings->freq_fine) { err = regmap_read(hw->regmap, ts_settings->freq_fine, &val); if (err < 0) return err; - /* - * linearize the AN5192 formula: - * 1 / (1 + x) ~= 1 - x (Taylor’s Series) - * ttrim[s] = 1 / (40000 * (1 + 0.0015 * val)) - * ttrim[ns] ~= 25000 - 37.5 * val - * ttrim[ns] ~= 25000 - (37500 * val) / 1000 - */ - hw->ts_gain -= ((s8)val * 37500) / 1000; + hw->ts_gain -= ((s8)val * ts_settings->ts_trim_coeff) / 1000; } return 0; diff --git a/drivers/iio/industrialio-buffer.c b/drivers/iio/industrialio-buffer.c index a80f7cc25a27..96ea0f039dfb 100644 --- a/drivers/iio/industrialio-buffer.c +++ b/drivers/iio/industrialio-buffer.c @@ -1623,19 +1623,28 @@ static int iio_dma_resv_lock(struct dma_buf *dmabuf, bool nonblock) return 0; } +static struct device *iio_buffer_get_dma_dev(const struct iio_dev *indio_dev, + struct iio_buffer *buffer) +{ + if (buffer->access->get_dma_dev) + return buffer->access->get_dma_dev(buffer); + + return indio_dev->dev.parent; +} + static struct dma_buf_attachment * iio_buffer_find_attachment(struct iio_dev_buffer_pair *ib, struct dma_buf *dmabuf, bool nonblock) { - struct device *dev = ib->indio_dev->dev.parent; struct iio_buffer *buffer = ib->buffer; + struct device *dma_dev = iio_buffer_get_dma_dev(ib->indio_dev, buffer); struct dma_buf_attachment *attach = NULL; struct iio_dmabuf_priv *priv; guard(mutex)(&buffer->dmabufs_mutex); list_for_each_entry(priv, &buffer->dmabufs, entry) { - if (priv->attach->dev == dev + if (priv->attach->dev == dma_dev && priv->attach->dmabuf == dmabuf) { attach = priv->attach; break; @@ -1653,6 +1662,7 @@ static int iio_buffer_attach_dmabuf(struct iio_dev_buffer_pair *ib, { struct iio_dev *indio_dev = ib->indio_dev; struct iio_buffer *buffer = ib->buffer; + struct device *dma_dev = iio_buffer_get_dma_dev(indio_dev, buffer); struct dma_buf_attachment *attach; struct iio_dmabuf_priv *priv, *each; struct dma_buf *dmabuf; @@ -1679,7 +1689,7 @@ static int iio_buffer_attach_dmabuf(struct iio_dev_buffer_pair *ib, goto err_free_priv; } - attach = dma_buf_attach(dmabuf, indio_dev->dev.parent); + attach = dma_buf_attach(dmabuf, dma_dev); if (IS_ERR(attach)) { err = PTR_ERR(attach); goto err_dmabuf_put; @@ -1719,7 +1729,7 @@ static int iio_buffer_attach_dmabuf(struct iio_dev_buffer_pair *ib, * combo. If we do, refuse to attach. */ list_for_each_entry(each, &buffer->dmabufs, entry) { - if (each->attach->dev == indio_dev->dev.parent + if (each->attach->dev == dma_dev && each->attach->dmabuf == dmabuf) { /* * We unlocked the reservation object, so going through @@ -1758,6 +1768,7 @@ static int iio_buffer_detach_dmabuf(struct iio_dev_buffer_pair *ib, { struct iio_buffer *buffer = ib->buffer; struct iio_dev *indio_dev = ib->indio_dev; + struct device *dma_dev = iio_buffer_get_dma_dev(indio_dev, buffer); struct iio_dmabuf_priv *priv; struct dma_buf *dmabuf; int dmabuf_fd, ret = -EPERM; @@ -1772,7 +1783,7 @@ static int iio_buffer_detach_dmabuf(struct iio_dev_buffer_pair *ib, guard(mutex)(&buffer->dmabufs_mutex); list_for_each_entry(priv, &buffer->dmabufs, entry) { - if (priv->attach->dev == indio_dev->dev.parent + if (priv->attach->dev == dma_dev && priv->attach->dmabuf == dmabuf) { list_del(&priv->entry); diff --git a/drivers/iio/pressure/bmp280-core.c b/drivers/iio/pressure/bmp280-core.c index c04e8bb4c993..d983ce9c0b99 100644 --- a/drivers/iio/pressure/bmp280-core.c +++ b/drivers/iio/pressure/bmp280-core.c @@ -1040,13 +1040,16 @@ static int bmp280_wait_conv(struct bmp280_data *data) unsigned int reg, meas_time_us; int ret; - /* Check if we are using a BME280 device */ - if (data->oversampling_humid) - meas_time_us = BMP280_PRESS_HUMID_MEAS_OFFSET + - BIT(data->oversampling_humid) * BMP280_MEAS_DUR; + /* Constant part of the measurement time */ + meas_time_us = BMP280_MEAS_OFFSET; - else - meas_time_us = 0; + /* + * Check if we are using a BME280 device, + * Humidity measurement time + */ + if (data->chip_info->oversampling_humid_avail) + meas_time_us += BMP280_PRESS_HUMID_MEAS_OFFSET + + BIT(data->oversampling_humid) * BMP280_MEAS_DUR; /* Pressure measurement time */ meas_time_us += BMP280_PRESS_HUMID_MEAS_OFFSET + diff --git a/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h b/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h index 1d7fc3226bca..cfb42a8f5768 100644 --- a/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h +++ b/drivers/infiniband/hw/usnic/usnic_uiom_interval_tree.h @@ -53,6 +53,10 @@ extern void usnic_uiom_interval_tree_remove(struct usnic_uiom_interval_node *node, struct rb_root_cached *root); extern struct usnic_uiom_interval_node * +usnic_uiom_interval_tree_subtree_search(struct usnic_uiom_interval_node *node, + unsigned long start, + unsigned long last); +extern struct usnic_uiom_interval_node * usnic_uiom_interval_tree_iter_first(struct rb_root_cached *root, unsigned long start, unsigned long last); diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 7944a3af4545..f1fb27681b0b 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -2008,7 +2008,7 @@ static void iommu_dma_iova_unlink_range_slow(struct device *dev, end - addr, iovad->granule - iova_start_pad); if (!dev_is_dma_coherent(dev) && - !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) + !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) arch_sync_dma_for_cpu(phys, len, dir); swiotlb_tbl_unmap_single(dev, phys, len, dir, attrs); @@ -2032,7 +2032,8 @@ static void __iommu_dma_iova_unlink(struct device *dev, size_t unmapped; if ((state->__size & DMA_IOVA_USE_SWIOTLB) || - (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))) + (!dev_is_dma_coherent(dev) && + !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO)))) iommu_dma_iova_unlink_range_slow(dev, addr, size, dir, attrs); iommu_iotlb_gather_init(&iotlb_gather); diff --git a/drivers/irqchip/Kconfig b/drivers/irqchip/Kconfig index a61c6dc63c29..f334f49c9791 100644 --- a/drivers/irqchip/Kconfig +++ b/drivers/irqchip/Kconfig @@ -36,7 +36,6 @@ config GIC_NON_BANKED config ARM_GIC_V3 bool select IRQ_DOMAIN_HIERARCHY - select PARTITION_PERCPU select GENERIC_IRQ_EFFECTIVE_AFF_MASK if SMP select HAVE_ARM_SMCCC_DISCOVERY select IRQ_MSI_IOMMU @@ -151,7 +150,7 @@ config BCM6345_L1_IRQ config BCM7038_L1_IRQ tristate "Broadcom STB 7038-style L1/L2 interrupt controller driver" - depends on ARCH_BRCMSTB || BMIPS_GENERIC + depends on ARCH_BRCMSTB || BMIPS_GENERIC || COMPILE_TEST default ARCH_BRCMSTB || BMIPS_GENERIC select GENERIC_IRQ_CHIP select IRQ_DOMAIN @@ -159,14 +158,14 @@ config BCM7038_L1_IRQ config BCM7120_L2_IRQ tristate "Broadcom STB 7120-style L2 interrupt controller driver" - depends on ARCH_BRCMSTB || BMIPS_GENERIC + depends on ARCH_BRCMSTB || BMIPS_GENERIC || COMPILE_TEST default ARCH_BRCMSTB || BMIPS_GENERIC select GENERIC_IRQ_CHIP select IRQ_DOMAIN config BRCMSTB_L2_IRQ tristate "Broadcom STB generic L2 interrupt controller driver" - depends on ARCH_BCM2835 || ARCH_BRCMSTB || BMIPS_GENERIC + depends on ARCH_BCM2835 || ARCH_BRCMSTB || BMIPS_GENERIC || COMPILE_TEST default ARCH_BCM2835 || ARCH_BRCMSTB || BMIPS_GENERIC select GENERIC_IRQ_CHIP select IRQ_DOMAIN @@ -451,9 +450,6 @@ config LS_SCFG_MSI depends on PCI_MSI select IRQ_MSI_LIB -config PARTITION_PERCPU - bool - config STM32MP_EXTI tristate "STM32MP extended interrupts and event controller" depends on (ARCH_STM32 && !ARM_SINGLE_ARMV7M) || COMPILE_TEST diff --git a/drivers/irqchip/Makefile b/drivers/irqchip/Makefile index 3de083f5484c..6a229443efe0 100644 --- a/drivers/irqchip/Makefile +++ b/drivers/irqchip/Makefile @@ -36,7 +36,6 @@ obj-$(CONFIG_ARM_GIC_V3) += irq-gic-v3.o irq-gic-v3-mbi.o irq-gic-common.o obj-$(CONFIG_ARM_GIC_ITS_PARENT) += irq-gic-its-msi-parent.o obj-$(CONFIG_ARM_GIC_V3_ITS) += irq-gic-v3-its.o irq-gic-v4.o obj-$(CONFIG_ARM_GIC_V3_ITS_FSL_MC) += irq-gic-v3-its-fsl-mc-msi.o -obj-$(CONFIG_PARTITION_PERCPU) += irq-partition-percpu.o obj-$(CONFIG_ARM_GIC_V5) += irq-gic-v5.o irq-gic-v5-irs.o irq-gic-v5-its.o \ irq-gic-v5-iwb.o obj-$(CONFIG_HISILICON_IRQ_MBIGEN) += irq-mbigen.o diff --git a/drivers/irqchip/irq-aclint-sswi.c b/drivers/irqchip/irq-aclint-sswi.c index 93e28e9f281f..fee30f3bc5ac 100644 --- a/drivers/irqchip/irq-aclint-sswi.c +++ b/drivers/irqchip/irq-aclint-sswi.c @@ -175,7 +175,8 @@ static int __init generic_aclint_sswi_early_probe(struct device_node *node, { return generic_aclint_sswi_probe(&node->fwnode); } -IRQCHIP_DECLARE(generic_aclint_sswi, "mips,p8700-aclint-sswi", generic_aclint_sswi_early_probe); +IRQCHIP_DECLARE(mips_p8700_sswi, "mips,p8700-aclint-sswi", generic_aclint_sswi_early_probe); +IRQCHIP_DECLARE(nuclei_ux900_sswi, "nuclei,ux900-aclint-sswi", generic_aclint_sswi_early_probe); /* THEAD variant */ #define THEAD_C9XX_CSR_SXSTATUS 0x5c0 diff --git a/drivers/irqchip/irq-apple-aic.c b/drivers/irqchip/irq-apple-aic.c index 032d66dceb8e..795b3db4554a 100644 --- a/drivers/irqchip/irq-apple-aic.c +++ b/drivers/irqchip/irq-apple-aic.c @@ -578,16 +578,9 @@ static void __exception_irq_entry aic_handle_fiq(struct pt_regs *regs) } if ((read_sysreg_s(SYS_IMP_APL_PMCR0_EL1) & (PMCR0_IMODE | PMCR0_IACT)) == - (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) { - int irq; - if (cpumask_test_cpu(smp_processor_id(), - &aic_irqc->fiq_aff[AIC_CPU_PMU_P]->aff)) - irq = AIC_CPU_PMU_P; - else - irq = AIC_CPU_PMU_E; + (FIELD_PREP(PMCR0_IMODE, PMCR0_IMODE_FIQ) | PMCR0_IACT)) generic_handle_domain_irq(aic_irqc->hw_domain, - AIC_FIQ_HWIRQ(irq)); - } + AIC_FIQ_HWIRQ(AIC_CPU_PMU_P)); if (static_branch_likely(&use_fast_ipi) && (FIELD_GET(UPMCR0_IMODE, read_sysreg_s(SYS_IMP_APL_UPMCR0_EL1)) == UPMCR0_IMODE_FIQ) && @@ -632,18 +625,7 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, handle_fasteoi_irq, NULL, NULL); irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); } else { - int fiq = FIELD_GET(AIC_EVENT_NUM, hw); - - switch (fiq) { - case AIC_CPU_PMU_P: - case AIC_CPU_PMU_E: - irq_set_percpu_devid_partition(irq, &ic->fiq_aff[fiq]->aff); - break; - default: - irq_set_percpu_devid(irq); - break; - } - + irq_set_percpu_devid(irq); irq_domain_set_info(id, irq, hw, &fiq_chip, id->host_data, handle_percpu_devid_irq, NULL, NULL); } @@ -651,6 +633,33 @@ static int aic_irq_domain_map(struct irq_domain *id, unsigned int irq, return 0; } +static int aic_irq_get_fwspec_info(struct irq_fwspec *fwspec, struct irq_fwspec_info *info) +{ + const struct cpumask *mask; + u32 intid; + + info->flags = 0; + info->affinity = NULL; + + if (fwspec->param[0] != AIC_FIQ) + return 0; + + if (fwspec->param_count == 3) + intid = fwspec->param[1]; + else + intid = fwspec->param[2]; + + if (aic_irqc->fiq_aff[intid]) + mask = &aic_irqc->fiq_aff[intid]->aff; + else + mask = cpu_possible_mask; + + info->affinity = mask; + info->flags = IRQ_FWSPEC_INFO_AFFINITY_VALID; + + return 0; +} + static int aic_irq_domain_translate(struct irq_domain *id, struct irq_fwspec *fwspec, unsigned long *hwirq, @@ -705,6 +714,10 @@ static int aic_irq_domain_translate(struct irq_domain *id, break; } } + + /* Merge the two PMUs on a single interrupt */ + if (*hwirq == AIC_CPU_PMU_E) + *hwirq = AIC_CPU_PMU_P; break; default: return -EINVAL; @@ -750,9 +763,10 @@ static void aic_irq_domain_free(struct irq_domain *domain, unsigned int virq, } static const struct irq_domain_ops aic_irq_domain_ops = { - .translate = aic_irq_domain_translate, - .alloc = aic_irq_domain_alloc, - .free = aic_irq_domain_free, + .translate = aic_irq_domain_translate, + .alloc = aic_irq_domain_alloc, + .free = aic_irq_domain_free, + .get_fwspec_info = aic_irq_get_fwspec_info, }; /* diff --git a/drivers/irqchip/irq-bcm2712-mip.c b/drivers/irqchip/irq-bcm2712-mip.c index 9bd7bc0bf6d5..4761974ad650 100644 --- a/drivers/irqchip/irq-bcm2712-mip.c +++ b/drivers/irqchip/irq-bcm2712-mip.c @@ -232,17 +232,12 @@ err_put: return ret; } -static int __init mip_of_msi_init(struct device_node *node, struct device_node *parent) +static int mip_msi_probe(struct platform_device *pdev, struct device_node *parent) { - struct platform_device *pdev; + struct device_node *node = pdev->dev.of_node; struct mip_priv *mip; int ret; - pdev = of_find_device_by_node(node); - of_node_put(node); - if (!pdev) - return -EPROBE_DEFER; - mip = kzalloc(sizeof(*mip), GFP_KERNEL); if (!mip) return -ENOMEM; @@ -285,7 +280,7 @@ err_priv: } IRQCHIP_PLATFORM_DRIVER_BEGIN(mip_msi) -IRQCHIP_MATCH("brcm,bcm2712-mip", mip_of_msi_init) +IRQCHIP_MATCH("brcm,bcm2712-mip", mip_msi_probe) IRQCHIP_PLATFORM_DRIVER_END(mip_msi) MODULE_DESCRIPTION("Broadcom BCM2712 MSI-X interrupt controller"); MODULE_AUTHOR("Phil Elwell <phil@raspberrypi.com>"); diff --git a/drivers/irqchip/irq-bcm7038-l1.c b/drivers/irqchip/irq-bcm7038-l1.c index 04fac0cc857f..ea1446c0a09c 100644 --- a/drivers/irqchip/irq-bcm7038-l1.c +++ b/drivers/irqchip/irq-bcm7038-l1.c @@ -82,12 +82,6 @@ static inline unsigned int reg_status(struct bcm7038_l1_chip *intc, return (0 * intc->n_words + word) * sizeof(u32); } -static inline unsigned int reg_mask_status(struct bcm7038_l1_chip *intc, - unsigned int word) -{ - return (1 * intc->n_words + word) * sizeof(u32); -} - static inline unsigned int reg_mask_set(struct bcm7038_l1_chip *intc, unsigned int word) { @@ -219,9 +213,8 @@ static int bcm7038_l1_set_affinity(struct irq_data *d, } #endif -static int __init bcm7038_l1_init_one(struct device_node *dn, - unsigned int idx, - struct bcm7038_l1_chip *intc) +static int bcm7038_l1_init_one(struct device_node *dn, unsigned int idx, + struct bcm7038_l1_chip *intc) { struct resource res; resource_size_t sz; @@ -395,9 +388,9 @@ static const struct irq_domain_ops bcm7038_l1_domain_ops = { .map = bcm7038_l1_map, }; -static int __init bcm7038_l1_of_init(struct device_node *dn, - struct device_node *parent) +static int bcm7038_l1_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *dn = pdev->dev.of_node; struct bcm7038_l1_chip *intc; int idx, ret; @@ -455,7 +448,7 @@ out_free: } IRQCHIP_PLATFORM_DRIVER_BEGIN(bcm7038_l1) -IRQCHIP_MATCH("brcm,bcm7038-l1-intc", bcm7038_l1_of_init) +IRQCHIP_MATCH("brcm,bcm7038-l1-intc", bcm7038_l1_probe) IRQCHIP_PLATFORM_DRIVER_END(bcm7038_l1) MODULE_DESCRIPTION("Broadcom STB 7038-style L1/L2 interrupt controller"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-bcm7120-l2.c b/drivers/irqchip/irq-bcm7120-l2.c index ff22c3104401..518c9d4366a5 100644 --- a/drivers/irqchip/irq-bcm7120-l2.c +++ b/drivers/irqchip/irq-bcm7120-l2.c @@ -143,8 +143,7 @@ static int bcm7120_l2_intc_init_one(struct device_node *dn, return 0; } -static int __init bcm7120_l2_intc_iomap_7120(struct device_node *dn, - struct bcm7120_l2_intc_data *data) +static int bcm7120_l2_intc_iomap_7120(struct device_node *dn, struct bcm7120_l2_intc_data *data) { int ret; @@ -177,8 +176,7 @@ static int __init bcm7120_l2_intc_iomap_7120(struct device_node *dn, return 0; } -static int __init bcm7120_l2_intc_iomap_3380(struct device_node *dn, - struct bcm7120_l2_intc_data *data) +static int bcm7120_l2_intc_iomap_3380(struct device_node *dn, struct bcm7120_l2_intc_data *data) { unsigned int gc_idx; @@ -208,15 +206,14 @@ static int __init bcm7120_l2_intc_iomap_3380(struct device_node *dn, return 0; } -static int __init bcm7120_l2_intc_probe(struct device_node *dn, - struct device_node *parent, +static int bcm7120_l2_intc_probe(struct platform_device *pdev, struct device_node *parent, int (*iomap_regs_fn)(struct device_node *, - struct bcm7120_l2_intc_data *), + struct bcm7120_l2_intc_data *), const char *intc_name) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; + struct device_node *dn = pdev->dev.of_node; struct bcm7120_l2_intc_data *data; - struct platform_device *pdev; struct irq_chip_generic *gc; struct irq_chip_type *ct; int ret = 0; @@ -227,14 +224,7 @@ static int __init bcm7120_l2_intc_probe(struct device_node *dn, if (!data) return -ENOMEM; - pdev = of_find_device_by_node(dn); - if (!pdev) { - ret = -ENODEV; - goto out_free_data; - } - data->num_parent_irqs = platform_irq_count(pdev); - put_device(&pdev->dev); if (data->num_parent_irqs <= 0) { pr_err("invalid number of parent interrupts\n"); ret = -ENOMEM; @@ -334,22 +324,19 @@ out_unmap: if (data->map_base[idx]) iounmap(data->map_base[idx]); } -out_free_data: kfree(data); return ret; } -static int __init bcm7120_l2_intc_probe_7120(struct device_node *dn, - struct device_node *parent) +static int bcm7120_l2_intc_probe_7120(struct platform_device *pdev, struct device_node *parent) { - return bcm7120_l2_intc_probe(dn, parent, bcm7120_l2_intc_iomap_7120, + return bcm7120_l2_intc_probe(pdev, parent, bcm7120_l2_intc_iomap_7120, "BCM7120 L2"); } -static int __init bcm7120_l2_intc_probe_3380(struct device_node *dn, - struct device_node *parent) +static int bcm7120_l2_intc_probe_3380(struct platform_device *pdev, struct device_node *parent) { - return bcm7120_l2_intc_probe(dn, parent, bcm7120_l2_intc_iomap_3380, + return bcm7120_l2_intc_probe(pdev, parent, bcm7120_l2_intc_iomap_3380, "BCM3380 L2"); } diff --git a/drivers/irqchip/irq-brcmstb-l2.c b/drivers/irqchip/irq-brcmstb-l2.c index 1bec5b2cd3f0..bb7078d6524f 100644 --- a/drivers/irqchip/irq-brcmstb-l2.c +++ b/drivers/irqchip/irq-brcmstb-l2.c @@ -138,13 +138,12 @@ static void brcmstb_l2_intc_resume(struct irq_data *d) irq_reg_writel(gc, ~b->saved_mask, ct->regs.enable); } -static int __init brcmstb_l2_intc_of_init(struct device_node *np, - struct device_node *parent, - const struct brcmstb_intc_init_params - *init_params) +static int brcmstb_l2_intc_probe(struct platform_device *pdev, struct device_node *parent, + const struct brcmstb_intc_init_params *init_params) { unsigned int clr = IRQ_NOREQUEST | IRQ_NOPROBE | IRQ_NOAUTOEN; unsigned int set = 0; + struct device_node *np = pdev->dev.of_node; struct brcmstb_l2_intc_data *data; struct irq_chip_type *ct; int ret; @@ -257,23 +256,21 @@ out_free: return ret; } -static int __init brcmstb_l2_edge_intc_of_init(struct device_node *np, - struct device_node *parent) +static int brcmstb_l2_edge_intc_probe(struct platform_device *pdev, struct device_node *parent) { - return brcmstb_l2_intc_of_init(np, parent, &l2_edge_intc_init); + return brcmstb_l2_intc_probe(pdev, parent, &l2_edge_intc_init); } -static int __init brcmstb_l2_lvl_intc_of_init(struct device_node *np, - struct device_node *parent) +static int brcmstb_l2_lvl_intc_probe(struct platform_device *pdev, struct device_node *parent) { - return brcmstb_l2_intc_of_init(np, parent, &l2_lvl_intc_init); + return brcmstb_l2_intc_probe(pdev, parent, &l2_lvl_intc_init); } IRQCHIP_PLATFORM_DRIVER_BEGIN(brcmstb_l2) -IRQCHIP_MATCH("brcm,l2-intc", brcmstb_l2_edge_intc_of_init) -IRQCHIP_MATCH("brcm,hif-spi-l2-intc", brcmstb_l2_edge_intc_of_init) -IRQCHIP_MATCH("brcm,upg-aux-aon-l2-intc", brcmstb_l2_edge_intc_of_init) -IRQCHIP_MATCH("brcm,bcm7271-l2-intc", brcmstb_l2_lvl_intc_of_init) +IRQCHIP_MATCH("brcm,l2-intc", brcmstb_l2_edge_intc_probe) +IRQCHIP_MATCH("brcm,hif-spi-l2-intc", brcmstb_l2_edge_intc_probe) +IRQCHIP_MATCH("brcm,upg-aux-aon-l2-intc", brcmstb_l2_edge_intc_probe) +IRQCHIP_MATCH("brcm,bcm7271-l2-intc", brcmstb_l2_lvl_intc_probe) IRQCHIP_PLATFORM_DRIVER_END(brcmstb_l2) MODULE_DESCRIPTION("Broadcom STB generic L2 interrupt controller"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-gic-its-msi-parent.c b/drivers/irqchip/irq-gic-its-msi-parent.c index eb1473f1448a..12f45228c867 100644 --- a/drivers/irqchip/irq-gic-its-msi-parent.c +++ b/drivers/irqchip/irq-gic-its-msi-parent.c @@ -142,83 +142,38 @@ static int its_v5_pci_msi_prepare(struct irq_domain *domain, struct device *dev, #define its_v5_pci_msi_prepare NULL #endif /* !CONFIG_PCI_MSI */ -static int of_pmsi_get_dev_id(struct irq_domain *domain, struct device *dev, - u32 *dev_id) +static int of_pmsi_get_msi_info(struct irq_domain *domain, struct device *dev, u32 *dev_id, + phys_addr_t *pa) { - int ret, index = 0; + struct of_phandle_iterator it; + int ret; /* Suck the DeviceID out of the msi-parent property */ - do { - struct of_phandle_args args; - - ret = of_parse_phandle_with_args(dev->of_node, - "msi-parent", "#msi-cells", - index, &args); - if (args.np == irq_domain_get_of_node(domain)) { - if (WARN_ON(args.args_count != 1)) - return -EINVAL; - *dev_id = args.args[0]; - break; - } - index++; - } while (!ret); - - if (ret) { - struct device_node *np = NULL; + of_for_each_phandle(&it, ret, dev->of_node, "msi-parent", "#msi-cells", -1) { + /* GICv5 ITS domain matches the MSI controller node parent */ + struct device_node *np __free(device_node) = pa ? of_get_parent(it.node) + : of_node_get(it.node); - ret = of_map_id(dev->of_node, dev->id, "msi-map", "msi-map-mask", &np, dev_id); - if (np) - of_node_put(np); - } + if (np == irq_domain_get_of_node(domain)) { + u32 args; - return ret; -} + if (WARN_ON(of_phandle_iterator_args(&it, &args, 1) != 1)) + ret = -EINVAL; -static int of_v5_pmsi_get_msi_info(struct irq_domain *domain, struct device *dev, - u32 *dev_id, phys_addr_t *pa) -{ - int ret, index = 0; - /* - * Retrieve the DeviceID and the ITS translate frame node pointer - * out of the msi-parent property. - */ - do { - struct of_phandle_args args; - - ret = of_parse_phandle_with_args(dev->of_node, - "msi-parent", "#msi-cells", - index, &args); - if (ret) - break; - /* - * The IRQ domain fwnode is the msi controller parent - * in GICv5 (where the msi controller nodes are the - * ITS translate frames). - */ - if (args.np->parent == irq_domain_get_of_node(domain)) { - if (WARN_ON(args.args_count != 1)) - return -EINVAL; - *dev_id = args.args[0]; - - ret = its_translate_frame_address(args.np, pa); - if (ret) - return -ENODEV; - break; - } - index++; - } while (!ret); + if (!ret && pa) + ret = its_translate_frame_address(it.node, pa); - if (ret) { - struct device_node *np = NULL; + if (!ret) + *dev_id = args; - ret = of_map_id(dev->of_node, dev->id, "msi-map", "msi-map-mask", &np, dev_id); - if (np) { - ret = its_translate_frame_address(np, pa); - of_node_put(np); + of_node_put(it.node); + return ret; } } - return ret; + struct device_node *msi_ctrl __free(device_node) = NULL; + + return of_map_id(dev->of_node, dev->id, "msi-map", "msi-map-mask", &msi_ctrl, dev_id); } int __weak iort_pmsi_get_dev_id(struct device *dev, u32 *dev_id) @@ -234,7 +189,7 @@ static int its_pmsi_prepare(struct irq_domain *domain, struct device *dev, int ret; if (dev->of_node) - ret = of_pmsi_get_dev_id(domain->parent, dev, &dev_id); + ret = of_pmsi_get_msi_info(domain->parent, dev, &dev_id, NULL); else ret = iort_pmsi_get_dev_id(dev, &dev_id); if (ret) @@ -262,7 +217,7 @@ static int its_v5_pmsi_prepare(struct irq_domain *domain, struct device *dev, if (!dev->of_node) return -ENODEV; - ret = of_v5_pmsi_get_msi_info(domain->parent, dev, &dev_id, &pa); + ret = of_pmsi_get_msi_info(domain->parent, dev, &dev_id, &pa); if (ret) return ret; diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c index 3de351e66ee8..6607ab58f72e 100644 --- a/drivers/irqchip/irq-gic-v3.c +++ b/drivers/irqchip/irq-gic-v3.c @@ -26,7 +26,6 @@ #include <linux/irqchip/arm-gic-common.h> #include <linux/irqchip/arm-gic-v3.h> #include <linux/irqchip/arm-gic-v3-prio.h> -#include <linux/irqchip/irq-partition-percpu.h> #include <linux/bitfield.h> #include <linux/bits.h> #include <linux/arm-smccc.h> @@ -46,8 +45,6 @@ static u8 dist_prio_nmi __ro_after_init = GICV3_PRIO_NMI; #define FLAGS_WORKAROUND_ASR_ERRATUM_8601001 (1ULL << 2) #define FLAGS_WORKAROUND_INSECURE (1ULL << 3) -#define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) - static struct cpumask broken_rdists __read_mostly __maybe_unused; struct redist_region { @@ -68,7 +65,13 @@ struct gic_chip_data { u64 flags; bool has_rss; unsigned int ppi_nr; - struct partition_desc **ppi_descs; + struct partition_affinity *parts; + unsigned int nr_parts; +}; + +struct partition_affinity { + cpumask_t mask; + struct fwnode_handle *partition_id; }; #define T241_CHIPS_MAX 4 @@ -228,9 +231,6 @@ static void __init gic_prio_init(void) !cpus_have_group0); } -/* rdist_nmi_refs[n] == number of cpus having the rdist interrupt n set as NMI */ -static refcount_t *rdist_nmi_refs; - static struct gic_kvm_info gic_v3_kvm_info __initdata; static DEFINE_PER_CPU(bool, has_rss); @@ -594,36 +594,6 @@ static void gic_irq_set_prio(struct irq_data *d, u8 prio) writeb_relaxed(prio, base + offset + index); } -static u32 __gic_get_ppi_index(irq_hw_number_t hwirq) -{ - switch (__get_intid_range(hwirq)) { - case PPI_RANGE: - return hwirq - 16; - case EPPI_RANGE: - return hwirq - EPPI_BASE_INTID + 16; - default: - unreachable(); - } -} - -static u32 __gic_get_rdist_index(irq_hw_number_t hwirq) -{ - switch (__get_intid_range(hwirq)) { - case SGI_RANGE: - case PPI_RANGE: - return hwirq; - case EPPI_RANGE: - return hwirq - EPPI_BASE_INTID + 32; - default: - unreachable(); - } -} - -static u32 gic_get_rdist_index(struct irq_data *d) -{ - return __gic_get_rdist_index(d->hwirq); -} - static int gic_irq_nmi_setup(struct irq_data *d) { struct irq_desc *desc = irq_to_desc(d->irq); @@ -644,20 +614,8 @@ static int gic_irq_nmi_setup(struct irq_data *d) return -EINVAL; /* desc lock should already be held */ - if (gic_irq_in_rdist(d)) { - u32 idx = gic_get_rdist_index(d); - - /* - * Setting up a percpu interrupt as NMI, only switch handler - * for first NMI - */ - if (!refcount_inc_not_zero(&rdist_nmi_refs[idx])) { - refcount_set(&rdist_nmi_refs[idx], 1); - desc->handle_irq = handle_percpu_devid_fasteoi_nmi; - } - } else { + if (!gic_irq_in_rdist(d)) desc->handle_irq = handle_fasteoi_nmi; - } gic_irq_set_prio(d, dist_prio_nmi); @@ -684,15 +642,8 @@ static void gic_irq_nmi_teardown(struct irq_data *d) return; /* desc lock should already be held */ - if (gic_irq_in_rdist(d)) { - u32 idx = gic_get_rdist_index(d); - - /* Tearing down NMI, only switch handler for last NMI */ - if (refcount_dec_and_test(&rdist_nmi_refs[idx])) - desc->handle_irq = handle_percpu_devid_irq; - } else { + if (!gic_irq_in_rdist(d)) desc->handle_irq = handle_fasteoi_irq; - } gic_irq_set_prio(d, dist_prio_irq); } @@ -1666,13 +1617,6 @@ static int gic_irq_domain_translate(struct irq_domain *d, case GIC_IRQ_TYPE_LPI: /* LPI */ *hwirq = fwspec->param[1]; break; - case GIC_IRQ_TYPE_PARTITION: - *hwirq = fwspec->param[1]; - if (fwspec->param[1] >= 16) - *hwirq += EPPI_BASE_INTID - 16; - else - *hwirq += 16; - break; default: return -EINVAL; } @@ -1681,10 +1625,8 @@ static int gic_irq_domain_translate(struct irq_domain *d, /* * Make it clear that broken DTs are... broken. - * Partitioned PPIs are an unfortunate exception. */ - WARN_ON(*type == IRQ_TYPE_NONE && - fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); + WARN_ON(*type == IRQ_TYPE_NONE); return 0; } @@ -1741,33 +1683,12 @@ static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, } } -static bool fwspec_is_partitioned_ppi(struct irq_fwspec *fwspec, - irq_hw_number_t hwirq) -{ - enum gic_intid_range range; - - if (!gic_data.ppi_descs) - return false; - - if (!is_of_node(fwspec->fwnode)) - return false; - - if (fwspec->param_count < 4 || !fwspec->param[3]) - return false; - - range = __get_intid_range(hwirq); - if (range != PPI_RANGE && range != EPPI_RANGE) - return false; - - return true; -} - static int gic_irq_domain_select(struct irq_domain *d, struct irq_fwspec *fwspec, enum irq_domain_bus_token bus_token) { - unsigned int type, ppi_idx; irq_hw_number_t hwirq; + unsigned int type; int ret; /* Not for us */ @@ -1786,60 +1707,61 @@ static int gic_irq_domain_select(struct irq_domain *d, if (WARN_ON_ONCE(ret)) return 0; - if (!fwspec_is_partitioned_ppi(fwspec, hwirq)) - return d == gic_data.domain; - - /* - * If this is a PPI and we have a 4th (non-null) parameter, - * then we need to match the partition domain. - */ - ppi_idx = __gic_get_ppi_index(hwirq); - return d == partition_get_domain(gic_data.ppi_descs[ppi_idx]); + return d == gic_data.domain; } -static const struct irq_domain_ops gic_irq_domain_ops = { - .translate = gic_irq_domain_translate, - .alloc = gic_irq_domain_alloc, - .free = gic_irq_domain_free, - .select = gic_irq_domain_select, -}; - -static int partition_domain_translate(struct irq_domain *d, - struct irq_fwspec *fwspec, - unsigned long *hwirq, - unsigned int *type) +static int gic_irq_get_fwspec_info(struct irq_fwspec *fwspec, struct irq_fwspec_info *info) { - unsigned long ppi_intid; - struct device_node *np; - unsigned int ppi_idx; - int ret; - - if (!gic_data.ppi_descs) - return -ENOMEM; + const struct cpumask *mask = NULL; - np = of_find_node_by_phandle(fwspec->param[3]); - if (WARN_ON(!np)) - return -EINVAL; + info->flags = 0; + info->affinity = NULL; - ret = gic_irq_domain_translate(d, fwspec, &ppi_intid, type); - if (WARN_ON_ONCE(ret)) + /* ACPI is not capable of describing PPI affinity -- yet */ + if (!is_of_node(fwspec->fwnode)) return 0; - ppi_idx = __gic_get_ppi_index(ppi_intid); - ret = partition_translate_id(gic_data.ppi_descs[ppi_idx], - of_fwnode_handle(np)); - if (ret < 0) - return ret; + /* If the specifier provides an affinity, use it */ + if (fwspec->param_count == 4 && fwspec->param[3]) { + struct fwnode_handle *fw; + + switch (fwspec->param[0]) { + case 1: /* PPI */ + case 3: /* EPPI */ + break; + default: + return 0; + } + + fw = of_fwnode_handle(of_find_node_by_phandle(fwspec->param[3])); + if (!fw) + return -ENOENT; + + for (int i = 0; i < gic_data.nr_parts; i++) { + if (gic_data.parts[i].partition_id == fw) { + mask = &gic_data.parts[i].mask; + break; + } + } + + if (!mask) + return -ENOENT; + } else { + mask = cpu_possible_mask; + } - *hwirq = ret; - *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; + info->affinity = mask; + info->flags = IRQ_FWSPEC_INFO_AFFINITY_VALID; return 0; } -static const struct irq_domain_ops partition_domain_ops = { - .translate = partition_domain_translate, +static const struct irq_domain_ops gic_irq_domain_ops = { + .translate = gic_irq_domain_translate, + .alloc = gic_irq_domain_alloc, + .free = gic_irq_domain_free, .select = gic_irq_domain_select, + .get_fwspec_info = gic_irq_get_fwspec_info, }; static bool gic_enable_quirk_msm8996(void *data) @@ -2030,19 +1952,9 @@ static const struct gic_quirk gic_quirks[] = { static void gic_enable_nmi_support(void) { - int i; - if (!gic_prio_masking_enabled() || nmi_support_forbidden) return; - rdist_nmi_refs = kcalloc(gic_data.ppi_nr + SGI_NR, - sizeof(*rdist_nmi_refs), GFP_KERNEL); - if (!rdist_nmi_refs) - return; - - for (i = 0; i < gic_data.ppi_nr + SGI_NR; i++) - refcount_set(&rdist_nmi_refs[i], 0); - pr_info("Pseudo-NMIs enabled using %s ICC_PMR_EL1 synchronisation\n", gic_has_relaxed_pmr_sync() ? "relaxed" : "forced"); @@ -2174,12 +2086,7 @@ static void __init gic_populate_ppi_partitions(struct device_node *gic_node) if (!parts_node) return; - gic_data.ppi_descs = kcalloc(gic_data.ppi_nr, sizeof(*gic_data.ppi_descs), GFP_KERNEL); - if (!gic_data.ppi_descs) - goto out_put_node; - nr_parts = of_get_child_count(parts_node); - if (!nr_parts) goto out_put_node; @@ -2232,29 +2139,8 @@ static void __init gic_populate_ppi_partitions(struct device_node *gic_node) part_idx++; } - for (i = 0; i < gic_data.ppi_nr; i++) { - unsigned int irq; - struct partition_desc *desc; - struct irq_fwspec ppi_fwspec = { - .fwnode = gic_data.fwnode, - .param_count = 3, - .param = { - [0] = GIC_IRQ_TYPE_PARTITION, - [1] = i, - [2] = IRQ_TYPE_NONE, - }, - }; - - irq = irq_create_fwspec_mapping(&ppi_fwspec); - if (WARN_ON(!irq)) - continue; - desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, - irq, &partition_domain_ops); - if (WARN_ON(!desc)) - continue; - - gic_data.ppi_descs[i] = desc; - } + gic_data.parts = parts; + gic_data.nr_parts = nr_parts; out_put_node: of_node_put(parts_node); diff --git a/drivers/irqchip/irq-imx-mu-msi.c b/drivers/irqchip/irq-imx-mu-msi.c index d2a4e8a61a42..c598f2f52fc6 100644 --- a/drivers/irqchip/irq-imx-mu-msi.c +++ b/drivers/irqchip/irq-imx-mu-msi.c @@ -296,11 +296,9 @@ static const struct imx_mu_dcfg imx_mu_cfg_imx8ulp = { }, }; -static int __init imx_mu_of_init(struct device_node *dn, - struct device_node *parent, - const struct imx_mu_dcfg *cfg) +static int imx_mu_probe(struct platform_device *pdev, struct device_node *parent, + const struct imx_mu_dcfg *cfg) { - struct platform_device *pdev = of_find_device_by_node(dn); struct device_link *pd_link_a; struct device_link *pd_link_b; struct imx_mu_msi *msi_data; @@ -416,31 +414,27 @@ static const struct dev_pm_ops imx_mu_pm_ops = { imx_mu_runtime_resume, NULL) }; -static int __init imx_mu_imx7ulp_of_init(struct device_node *dn, - struct device_node *parent) +static int imx_mu_imx7ulp_probe(struct platform_device *pdev, struct device_node *parent) { - return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx7ulp); + return imx_mu_probe(pdev, parent, &imx_mu_cfg_imx7ulp); } -static int __init imx_mu_imx6sx_of_init(struct device_node *dn, - struct device_node *parent) +static int imx_mu_imx6sx_probe(struct platform_device *pdev, struct device_node *parent) { - return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx6sx); + return imx_mu_probe(pdev, parent, &imx_mu_cfg_imx6sx); } -static int __init imx_mu_imx8ulp_of_init(struct device_node *dn, - struct device_node *parent) +static int imx_mu_imx8ulp_probe(struct platform_device *pdev, struct device_node *parent) { - return imx_mu_of_init(dn, parent, &imx_mu_cfg_imx8ulp); + return imx_mu_probe(pdev, parent, &imx_mu_cfg_imx8ulp); } IRQCHIP_PLATFORM_DRIVER_BEGIN(imx_mu_msi) -IRQCHIP_MATCH("fsl,imx7ulp-mu-msi", imx_mu_imx7ulp_of_init) -IRQCHIP_MATCH("fsl,imx6sx-mu-msi", imx_mu_imx6sx_of_init) -IRQCHIP_MATCH("fsl,imx8ulp-mu-msi", imx_mu_imx8ulp_of_init) +IRQCHIP_MATCH("fsl,imx7ulp-mu-msi", imx_mu_imx7ulp_probe) +IRQCHIP_MATCH("fsl,imx6sx-mu-msi", imx_mu_imx6sx_probe) +IRQCHIP_MATCH("fsl,imx8ulp-mu-msi", imx_mu_imx8ulp_probe) IRQCHIP_PLATFORM_DRIVER_END(imx_mu_msi, .pm = &imx_mu_pm_ops) - MODULE_AUTHOR("Frank Li <Frank.Li@nxp.com>"); MODULE_DESCRIPTION("Freescale MU MSI controller driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/irqchip/irq-mchp-eic.c b/drivers/irqchip/irq-mchp-eic.c index 516a3a0e359c..b513a899c085 100644 --- a/drivers/irqchip/irq-mchp-eic.c +++ b/drivers/irqchip/irq-mchp-eic.c @@ -199,8 +199,9 @@ static const struct irq_domain_ops mchp_eic_domain_ops = { .free = irq_domain_free_irqs_common, }; -static int mchp_eic_init(struct device_node *node, struct device_node *parent) +static int mchp_eic_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *node = pdev->dev.of_node; struct irq_domain *parent_domain = NULL; int ret, i; @@ -273,7 +274,7 @@ free: } IRQCHIP_PLATFORM_DRIVER_BEGIN(mchp_eic) -IRQCHIP_MATCH("microchip,sama7g5-eic", mchp_eic_init) +IRQCHIP_MATCH("microchip,sama7g5-eic", mchp_eic_probe) IRQCHIP_PLATFORM_DRIVER_END(mchp_eic) MODULE_DESCRIPTION("Microchip External Interrupt Controller"); diff --git a/drivers/irqchip/irq-meson-gpio.c b/drivers/irqchip/irq-meson-gpio.c index 7d177626d64b..3fcbb044ae60 100644 --- a/drivers/irqchip/irq-meson-gpio.c +++ b/drivers/irqchip/irq-meson-gpio.c @@ -174,6 +174,14 @@ static const struct meson_gpio_irq_params s4_params = { INIT_MESON_S4_COMMON_DATA(82) }; +static const struct meson_gpio_irq_params s6_params = { + INIT_MESON_S4_COMMON_DATA(100) +}; + +static const struct meson_gpio_irq_params s7_params = { + INIT_MESON_S4_COMMON_DATA(84) +}; + static const struct meson_gpio_irq_params c3_params = { INIT_MESON_S4_COMMON_DATA(55) }; @@ -195,6 +203,9 @@ static const struct of_device_id meson_irq_gpio_matches[] __maybe_unused = { { .compatible = "amlogic,a4-gpio-ao-intc", .data = &a4_ao_params }, { .compatible = "amlogic,a4-gpio-intc", .data = &a4_params }, { .compatible = "amlogic,a5-gpio-intc", .data = &a5_params }, + { .compatible = "amlogic,s6-gpio-intc", .data = &s6_params }, + { .compatible = "amlogic,s7-gpio-intc", .data = &s7_params }, + { .compatible = "amlogic,s7d-gpio-intc", .data = &s7_params }, { .compatible = "amlogic,c3-gpio-intc", .data = &c3_params }, { .compatible = "amlogic,t7-gpio-intc", .data = &t7_params }, { } @@ -572,8 +583,9 @@ static int meson_gpio_irq_parse_dt(struct device_node *node, struct meson_gpio_i return 0; } -static int meson_gpio_irq_of_init(struct device_node *node, struct device_node *parent) +static int meson_gpio_irq_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *node = pdev->dev.of_node; struct irq_domain *domain, *parent_domain; struct meson_gpio_irq_controller *ctl; int ret; @@ -630,10 +642,9 @@ free_ctl: } IRQCHIP_PLATFORM_DRIVER_BEGIN(meson_gpio_intc) -IRQCHIP_MATCH("amlogic,meson-gpio-intc", meson_gpio_irq_of_init) +IRQCHIP_MATCH("amlogic,meson-gpio-intc", meson_gpio_irq_probe) IRQCHIP_PLATFORM_DRIVER_END(meson_gpio_intc) MODULE_AUTHOR("Jerome Brunet <jbrunet@baylibre.com>"); MODULE_DESCRIPTION("Meson GPIO Interrupt Multiplexer driver"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:meson-gpio-intc"); diff --git a/drivers/irqchip/irq-mvebu-pic.c b/drivers/irqchip/irq-mvebu-pic.c index cd8b73482b9f..10b85128183a 100644 --- a/drivers/irqchip/irq-mvebu-pic.c +++ b/drivers/irqchip/irq-mvebu-pic.c @@ -195,5 +195,3 @@ MODULE_AUTHOR("Yehuda Yitschak <yehuday@marvell.com>"); MODULE_AUTHOR("Thomas Petazzoni <thomas.petazzoni@free-electrons.com>"); MODULE_DESCRIPTION("Marvell Armada 7K/8K PIC driver"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:mvebu_pic"); - diff --git a/drivers/irqchip/irq-partition-percpu.c b/drivers/irqchip/irq-partition-percpu.c deleted file mode 100644 index 4441ffe149ea..000000000000 --- a/drivers/irqchip/irq-partition-percpu.c +++ /dev/null @@ -1,241 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* - * Copyright (C) 2016 ARM Limited, All Rights Reserved. - * Author: Marc Zyngier <marc.zyngier@arm.com> - */ - -#include <linux/bitops.h> -#include <linux/interrupt.h> -#include <linux/irqchip.h> -#include <linux/irqchip/chained_irq.h> -#include <linux/irqchip/irq-partition-percpu.h> -#include <linux/irqdomain.h> -#include <linux/seq_file.h> -#include <linux/slab.h> - -struct partition_desc { - int nr_parts; - struct partition_affinity *parts; - struct irq_domain *domain; - struct irq_desc *chained_desc; - unsigned long *bitmap; - struct irq_domain_ops ops; -}; - -static bool partition_check_cpu(struct partition_desc *part, - unsigned int cpu, unsigned int hwirq) -{ - return cpumask_test_cpu(cpu, &part->parts[hwirq].mask); -} - -static void partition_irq_mask(struct irq_data *d) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (partition_check_cpu(part, smp_processor_id(), d->hwirq) && - chip->irq_mask) - chip->irq_mask(data); -} - -static void partition_irq_unmask(struct irq_data *d) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (partition_check_cpu(part, smp_processor_id(), d->hwirq) && - chip->irq_unmask) - chip->irq_unmask(data); -} - -static int partition_irq_set_irqchip_state(struct irq_data *d, - enum irqchip_irq_state which, - bool val) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (partition_check_cpu(part, smp_processor_id(), d->hwirq) && - chip->irq_set_irqchip_state) - return chip->irq_set_irqchip_state(data, which, val); - - return -EINVAL; -} - -static int partition_irq_get_irqchip_state(struct irq_data *d, - enum irqchip_irq_state which, - bool *val) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (partition_check_cpu(part, smp_processor_id(), d->hwirq) && - chip->irq_get_irqchip_state) - return chip->irq_get_irqchip_state(data, which, val); - - return -EINVAL; -} - -static int partition_irq_set_type(struct irq_data *d, unsigned int type) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - if (chip->irq_set_type) - return chip->irq_set_type(data, type); - - return -EINVAL; -} - -static void partition_irq_print_chip(struct irq_data *d, struct seq_file *p) -{ - struct partition_desc *part = irq_data_get_irq_chip_data(d); - struct irq_chip *chip = irq_desc_get_chip(part->chained_desc); - struct irq_data *data = irq_desc_get_irq_data(part->chained_desc); - - seq_printf(p, "%5s-%lu", chip->name, data->hwirq); -} - -static struct irq_chip partition_irq_chip = { - .irq_mask = partition_irq_mask, - .irq_unmask = partition_irq_unmask, - .irq_set_type = partition_irq_set_type, - .irq_get_irqchip_state = partition_irq_get_irqchip_state, - .irq_set_irqchip_state = partition_irq_set_irqchip_state, - .irq_print_chip = partition_irq_print_chip, -}; - -static void partition_handle_irq(struct irq_desc *desc) -{ - struct partition_desc *part = irq_desc_get_handler_data(desc); - struct irq_chip *chip = irq_desc_get_chip(desc); - int cpu = smp_processor_id(); - int hwirq; - - chained_irq_enter(chip, desc); - - for_each_set_bit(hwirq, part->bitmap, part->nr_parts) { - if (partition_check_cpu(part, cpu, hwirq)) - break; - } - - if (unlikely(hwirq == part->nr_parts)) - handle_bad_irq(desc); - else - generic_handle_domain_irq(part->domain, hwirq); - - chained_irq_exit(chip, desc); -} - -static int partition_domain_alloc(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs, void *arg) -{ - int ret; - irq_hw_number_t hwirq; - unsigned int type; - struct irq_fwspec *fwspec = arg; - struct partition_desc *part; - - BUG_ON(nr_irqs != 1); - ret = domain->ops->translate(domain, fwspec, &hwirq, &type); - if (ret) - return ret; - - part = domain->host_data; - - set_bit(hwirq, part->bitmap); - irq_set_chained_handler_and_data(irq_desc_get_irq(part->chained_desc), - partition_handle_irq, part); - irq_set_percpu_devid_partition(virq, &part->parts[hwirq].mask); - irq_domain_set_info(domain, virq, hwirq, &partition_irq_chip, part, - handle_percpu_devid_irq, NULL, NULL); - irq_set_status_flags(virq, IRQ_NOAUTOEN); - - return 0; -} - -static void partition_domain_free(struct irq_domain *domain, unsigned int virq, - unsigned int nr_irqs) -{ - struct irq_data *d; - - BUG_ON(nr_irqs != 1); - - d = irq_domain_get_irq_data(domain, virq); - irq_set_handler(virq, NULL); - irq_domain_reset_irq_data(d); -} - -int partition_translate_id(struct partition_desc *desc, void *partition_id) -{ - struct partition_affinity *part = NULL; - int i; - - for (i = 0; i < desc->nr_parts; i++) { - if (desc->parts[i].partition_id == partition_id) { - part = &desc->parts[i]; - break; - } - } - - if (WARN_ON(!part)) { - pr_err("Failed to find partition\n"); - return -EINVAL; - } - - return i; -} - -struct partition_desc *partition_create_desc(struct fwnode_handle *fwnode, - struct partition_affinity *parts, - int nr_parts, - int chained_irq, - const struct irq_domain_ops *ops) -{ - struct partition_desc *desc; - struct irq_domain *d; - - BUG_ON(!ops->select || !ops->translate); - - desc = kzalloc(sizeof(*desc), GFP_KERNEL); - if (!desc) - return NULL; - - desc->ops = *ops; - desc->ops.free = partition_domain_free; - desc->ops.alloc = partition_domain_alloc; - - d = irq_domain_create_linear(fwnode, nr_parts, &desc->ops, desc); - if (!d) - goto out; - desc->domain = d; - - desc->bitmap = bitmap_zalloc(nr_parts, GFP_KERNEL); - if (WARN_ON(!desc->bitmap)) - goto out; - - desc->chained_desc = irq_to_desc(chained_irq); - desc->nr_parts = nr_parts; - desc->parts = parts; - - return desc; -out: - if (d) - irq_domain_remove(d); - kfree(desc); - - return NULL; -} - -struct irq_domain *partition_get_domain(struct partition_desc *dsc) -{ - if (dsc) - return dsc->domain; - - return NULL; -} diff --git a/drivers/irqchip/irq-qcom-mpm.c b/drivers/irqchip/irq-qcom-mpm.c index 8d569f7c5a7a..83f31ea657b7 100644 --- a/drivers/irqchip/irq-qcom-mpm.c +++ b/drivers/irqchip/irq-qcom-mpm.c @@ -320,9 +320,9 @@ static bool gic_hwirq_is_mapped(struct mpm_gic_map *maps, int cnt, u32 hwirq) return false; } -static int qcom_mpm_init(struct device_node *np, struct device_node *parent) +static int qcom_mpm_probe(struct platform_device *pdev, struct device_node *parent) { - struct platform_device *pdev = of_find_device_by_node(np); + struct device_node *np = pdev->dev.of_node; struct device *dev = &pdev->dev; struct irq_domain *parent_domain; struct generic_pm_domain *genpd; @@ -478,7 +478,7 @@ remove_genpd: } IRQCHIP_PLATFORM_DRIVER_BEGIN(qcom_mpm) -IRQCHIP_MATCH("qcom,mpm", qcom_mpm_init) +IRQCHIP_MATCH("qcom,mpm", qcom_mpm_probe) IRQCHIP_PLATFORM_DRIVER_END(qcom_mpm) MODULE_DESCRIPTION("Qualcomm Technologies, Inc. MSM Power Manager"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/irqchip/irq-renesas-rzg2l.c b/drivers/irqchip/irq-renesas-rzg2l.c index 2a54adeb4cc7..1bf19deb02c4 100644 --- a/drivers/irqchip/irq-renesas-rzg2l.c +++ b/drivers/irqchip/irq-renesas-rzg2l.c @@ -8,7 +8,6 @@ */ #include <linux/bitfield.h> -#include <linux/cleanup.h> #include <linux/clk.h> #include <linux/err.h> #include <linux/io.h> @@ -528,18 +527,15 @@ static int rzg2l_irqc_parse_interrupts(struct rzg2l_irqc_priv *priv, return 0; } -static int rzg2l_irqc_common_init(struct device_node *node, struct device_node *parent, - const struct irq_chip *irq_chip) +static int rzg2l_irqc_common_probe(struct platform_device *pdev, struct device_node *parent, + const struct irq_chip *irq_chip) { - struct platform_device *pdev = of_find_device_by_node(node); - struct device *dev __free(put_device) = pdev ? &pdev->dev : NULL; struct irq_domain *irq_domain, *parent_domain; + struct device_node *node = pdev->dev.of_node; + struct device *dev = &pdev->dev; struct reset_control *resetn; int ret; - if (!pdev) - return -ENODEV; - parent_domain = irq_find_host(parent); if (!parent_domain) return dev_err_probe(dev, -ENODEV, "cannot find parent domain\n"); @@ -583,35 +579,22 @@ static int rzg2l_irqc_common_init(struct device_node *node, struct device_node * register_syscore_ops(&rzg2l_irqc_syscore_ops); - /* - * Prevent the cleanup function from invoking put_device by assigning - * NULL to dev. - * - * make coccicheck will complain about missing put_device calls, but - * those are false positives, as dev will be automatically "put" via - * __free_put_device on the failing path. - * On the successful path we don't actually want to "put" dev. - */ - dev = NULL; - return 0; } -static int __init rzg2l_irqc_init(struct device_node *node, - struct device_node *parent) +static int rzg2l_irqc_probe(struct platform_device *pdev, struct device_node *parent) { - return rzg2l_irqc_common_init(node, parent, &rzg2l_irqc_chip); + return rzg2l_irqc_common_probe(pdev, parent, &rzg2l_irqc_chip); } -static int __init rzfive_irqc_init(struct device_node *node, - struct device_node *parent) +static int rzfive_irqc_probe(struct platform_device *pdev, struct device_node *parent) { - return rzg2l_irqc_common_init(node, parent, &rzfive_irqc_chip); + return rzg2l_irqc_common_probe(pdev, parent, &rzfive_irqc_chip); } IRQCHIP_PLATFORM_DRIVER_BEGIN(rzg2l_irqc) -IRQCHIP_MATCH("renesas,rzg2l-irqc", rzg2l_irqc_init) -IRQCHIP_MATCH("renesas,r9a07g043f-irqc", rzfive_irqc_init) +IRQCHIP_MATCH("renesas,rzg2l-irqc", rzg2l_irqc_probe) +IRQCHIP_MATCH("renesas,r9a07g043f-irqc", rzfive_irqc_probe) IRQCHIP_PLATFORM_DRIVER_END(rzg2l_irqc) MODULE_AUTHOR("Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com>"); MODULE_DESCRIPTION("Renesas RZ/G2L IRQC Driver"); diff --git a/drivers/irqchip/irq-renesas-rzv2h.c b/drivers/irqchip/irq-renesas-rzv2h.c index 9018d9c3911e..899a423b5da8 100644 --- a/drivers/irqchip/irq-renesas-rzv2h.c +++ b/drivers/irqchip/irq-renesas-rzv2h.c @@ -490,29 +490,15 @@ static int rzv2h_icu_parse_interrupts(struct rzv2h_icu_priv *priv, struct device return 0; } -static void rzv2h_icu_put_device(void *data) -{ - put_device(data); -} - -static int rzv2h_icu_init_common(struct device_node *node, struct device_node *parent, - const struct rzv2h_hw_info *hw_info) +static int rzv2h_icu_probe_common(struct platform_device *pdev, struct device_node *parent, + const struct rzv2h_hw_info *hw_info) { struct irq_domain *irq_domain, *parent_domain; + struct device_node *node = pdev->dev.of_node; struct rzv2h_icu_priv *rzv2h_icu_data; - struct platform_device *pdev; struct reset_control *resetn; int ret; - pdev = of_find_device_by_node(node); - if (!pdev) - return -ENODEV; - - ret = devm_add_action_or_reset(&pdev->dev, rzv2h_icu_put_device, - &pdev->dev); - if (ret < 0) - return ret; - parent_domain = irq_find_host(parent); if (!parent_domain) { dev_err(&pdev->dev, "cannot find parent domain\n"); @@ -618,19 +604,19 @@ static const struct rzv2h_hw_info rzv2h_hw_params = { .field_width = 8, }; -static int rzg3e_icu_init(struct device_node *node, struct device_node *parent) +static int rzg3e_icu_probe(struct platform_device *pdev, struct device_node *parent) { - return rzv2h_icu_init_common(node, parent, &rzg3e_hw_params); + return rzv2h_icu_probe_common(pdev, parent, &rzg3e_hw_params); } -static int rzv2h_icu_init(struct device_node *node, struct device_node *parent) +static int rzv2h_icu_probe(struct platform_device *pdev, struct device_node *parent) { - return rzv2h_icu_init_common(node, parent, &rzv2h_hw_params); + return rzv2h_icu_probe_common(pdev, parent, &rzv2h_hw_params); } IRQCHIP_PLATFORM_DRIVER_BEGIN(rzv2h_icu) -IRQCHIP_MATCH("renesas,r9a09g047-icu", rzg3e_icu_init) -IRQCHIP_MATCH("renesas,r9a09g057-icu", rzv2h_icu_init) +IRQCHIP_MATCH("renesas,r9a09g047-icu", rzg3e_icu_probe) +IRQCHIP_MATCH("renesas,r9a09g057-icu", rzv2h_icu_probe) IRQCHIP_PLATFORM_DRIVER_END(rzv2h_icu) MODULE_AUTHOR("Fabrizio Castro <fabrizio.castro.jz@renesas.com>"); MODULE_DESCRIPTION("Renesas RZ/V2H(P) ICU Driver"); diff --git a/drivers/irqchip/irq-riscv-imsic-early.c b/drivers/irqchip/irq-riscv-imsic-early.c index 2c4c682627b8..6bac67cc0b6d 100644 --- a/drivers/irqchip/irq-riscv-imsic-early.c +++ b/drivers/irqchip/irq-riscv-imsic-early.c @@ -91,9 +91,8 @@ static int __init imsic_ipi_domain_init(void) { return 0; } */ static void imsic_handle_irq(struct irq_desc *desc) { + struct imsic_local_priv *lpriv = this_cpu_ptr(imsic->lpriv); struct irq_chip *chip = irq_desc_get_chip(desc); - int cpu = smp_processor_id(); - struct imsic_vector *vec; unsigned long local_id; /* @@ -113,16 +112,12 @@ static void imsic_handle_irq(struct irq_desc *desc) continue; } - if (unlikely(!imsic->base_domain)) - continue; - - vec = imsic_vector_from_local_id(cpu, local_id); - if (!vec) { + if (unlikely(local_id > imsic->global.nr_ids)) { pr_warn_ratelimited("vector not found for local ID 0x%lx\n", local_id); continue; } - generic_handle_irq(vec->irq); + generic_handle_irq(lpriv->vectors[local_id].irq); } chained_irq_exit(chip, desc); diff --git a/drivers/irqchip/irq-riscv-imsic-platform.c b/drivers/irqchip/irq-riscv-imsic-platform.c index 643c8e459611..7228a33f6c37 100644 --- a/drivers/irqchip/irq-riscv-imsic-platform.c +++ b/drivers/irqchip/irq-riscv-imsic-platform.c @@ -158,11 +158,11 @@ static int imsic_irq_set_affinity(struct irq_data *d, const struct cpumask *mask tmp_vec.local_id = new_vec->local_id; /* Point device to the temporary vector */ - imsic_msi_update_msg(irq_get_irq_data(d->irq), &tmp_vec); + imsic_msi_update_msg(d, &tmp_vec); } /* Point device to the new vector */ - imsic_msi_update_msg(irq_get_irq_data(d->irq), new_vec); + imsic_msi_update_msg(d, new_vec); /* Update irq descriptors with the new vector */ d->chip_data = new_vec; diff --git a/drivers/irqchip/irq-riscv-imsic-state.c b/drivers/irqchip/irq-riscv-imsic-state.c index dc95ad856d80..385368052d5c 100644 --- a/drivers/irqchip/irq-riscv-imsic-state.c +++ b/drivers/irqchip/irq-riscv-imsic-state.c @@ -434,16 +434,6 @@ void imsic_vector_debug_show_summary(struct seq_file *m, int ind) } #endif -struct imsic_vector *imsic_vector_from_local_id(unsigned int cpu, unsigned int local_id) -{ - struct imsic_local_priv *lpriv = per_cpu_ptr(imsic->lpriv, cpu); - - if (!lpriv || imsic->global.nr_ids < local_id) - return NULL; - - return &lpriv->vectors[local_id]; -} - struct imsic_vector *imsic_vector_alloc(unsigned int irq, const struct cpumask *mask) { struct imsic_vector *vec = NULL; @@ -487,7 +477,6 @@ static void __init imsic_local_cleanup(void) lpriv = per_cpu_ptr(imsic->lpriv, cpu); bitmap_free(lpriv->dirty_bitmap); - kfree(lpriv->vectors); } free_percpu(imsic->lpriv); @@ -501,7 +490,8 @@ static int __init imsic_local_init(void) int cpu, i; /* Allocate per-CPU private state */ - imsic->lpriv = alloc_percpu(typeof(*imsic->lpriv)); + imsic->lpriv = __alloc_percpu(struct_size(imsic->lpriv, vectors, global->nr_ids + 1), + __alignof__(*imsic->lpriv)); if (!imsic->lpriv) return -ENOMEM; @@ -521,12 +511,6 @@ static int __init imsic_local_init(void) timer_setup(&lpriv->timer, imsic_local_timer_callback, TIMER_PINNED); #endif - /* Allocate vector array */ - lpriv->vectors = kcalloc(global->nr_ids + 1, sizeof(*lpriv->vectors), - GFP_KERNEL); - if (!lpriv->vectors) - goto fail_local_cleanup; - /* Setup vector array */ for (i = 0; i <= global->nr_ids; i++) { vec = &lpriv->vectors[i]; diff --git a/drivers/irqchip/irq-riscv-imsic-state.h b/drivers/irqchip/irq-riscv-imsic-state.h index 57f951952b0c..6332501dcbd8 100644 --- a/drivers/irqchip/irq-riscv-imsic-state.h +++ b/drivers/irqchip/irq-riscv-imsic-state.h @@ -40,7 +40,7 @@ struct imsic_local_priv { #endif /* Local vector table */ - struct imsic_vector *vectors; + struct imsic_vector vectors[]; }; struct imsic_priv { @@ -95,8 +95,6 @@ static inline struct imsic_vector *imsic_vector_get_move(struct imsic_vector *ve void imsic_vector_force_move_cleanup(struct imsic_vector *vec); void imsic_vector_move(struct imsic_vector *old_vec, struct imsic_vector *new_vec); -struct imsic_vector *imsic_vector_from_local_id(unsigned int cpu, unsigned int local_id); - struct imsic_vector *imsic_vector_alloc(unsigned int irq, const struct cpumask *mask); void imsic_vector_free(struct imsic_vector *vector); diff --git a/drivers/irqchip/irq-sifive-plic.c b/drivers/irqchip/irq-sifive-plic.c index cbd7697bc148..c5db7d6e3f7c 100644 --- a/drivers/irqchip/irq-sifive-plic.c +++ b/drivers/irqchip/irq-sifive-plic.c @@ -49,6 +49,8 @@ #define CONTEXT_ENABLE_BASE 0x2000 #define CONTEXT_ENABLE_SIZE 0x80 +#define PENDING_BASE 0x1000 + /* * Each hart context has a set of control registers associated with it. Right * now there's only two: a source priority threshold over which the hart will @@ -63,6 +65,7 @@ #define PLIC_ENABLE_THRESHOLD 0 #define PLIC_QUIRK_EDGE_INTERRUPT 0 +#define PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM 1 struct plic_priv { struct fwnode_handle *fwnode; @@ -94,15 +97,22 @@ static DEFINE_PER_CPU(struct plic_handler, plic_handlers); static int plic_irq_set_type(struct irq_data *d, unsigned int type); -static void __plic_toggle(void __iomem *enable_base, int hwirq, int enable) +static void __plic_toggle(struct plic_handler *handler, int hwirq, int enable) { - u32 __iomem *reg = enable_base + (hwirq / 32) * sizeof(u32); + u32 __iomem *base = handler->enable_base; u32 hwirq_mask = 1 << (hwirq % 32); + int group = hwirq / 32; + u32 value; + + value = readl(base + group); if (enable) - writel(readl(reg) | hwirq_mask, reg); + value |= hwirq_mask; else - writel(readl(reg) & ~hwirq_mask, reg); + value &= ~hwirq_mask; + + handler->enable_save[group] = value; + writel(value, base + group); } static void plic_toggle(struct plic_handler *handler, int hwirq, int enable) @@ -110,7 +120,7 @@ static void plic_toggle(struct plic_handler *handler, int hwirq, int enable) unsigned long flags; raw_spin_lock_irqsave(&handler->enable_lock, flags); - __plic_toggle(handler->enable_base, hwirq, enable); + __plic_toggle(handler, hwirq, enable); raw_spin_unlock_irqrestore(&handler->enable_lock, flags); } @@ -247,33 +257,16 @@ static int plic_irq_set_type(struct irq_data *d, unsigned int type) static int plic_irq_suspend(void) { - unsigned int i, cpu; - unsigned long flags; - u32 __iomem *reg; struct plic_priv *priv; priv = per_cpu_ptr(&plic_handlers, smp_processor_id())->priv; /* irq ID 0 is reserved */ - for (i = 1; i < priv->nr_irqs; i++) { + for (unsigned int i = 1; i < priv->nr_irqs; i++) { __assign_bit(i, priv->prio_save, readl(priv->regs + PRIORITY_BASE + i * PRIORITY_PER_ID)); } - for_each_present_cpu(cpu) { - struct plic_handler *handler = per_cpu_ptr(&plic_handlers, cpu); - - if (!handler->present) - continue; - - raw_spin_lock_irqsave(&handler->enable_lock, flags); - for (i = 0; i < DIV_ROUND_UP(priv->nr_irqs, 32); i++) { - reg = handler->enable_base + i * sizeof(u32); - handler->enable_save[i] = readl(reg); - } - raw_spin_unlock_irqrestore(&handler->enable_lock, flags); - } - return 0; } @@ -398,6 +391,98 @@ static void plic_handle_irq(struct irq_desc *desc) chained_irq_exit(chip, desc); } +static u32 cp100_isolate_pending_irq(int nr_irq_groups, struct plic_handler *handler) +{ + u32 __iomem *pending = handler->priv->regs + PENDING_BASE; + u32 __iomem *enable = handler->enable_base; + u32 pending_irqs = 0; + int i, j; + + /* Look for first pending interrupt */ + for (i = 0; i < nr_irq_groups; i++) { + /* Any pending interrupts would be annihilated, so skip checking them */ + if (!handler->enable_save[i]) + continue; + + pending_irqs = handler->enable_save[i] & readl_relaxed(pending + i); + if (pending_irqs) + break; + } + + if (!pending_irqs) + return 0; + + /* Isolate lowest set bit */ + pending_irqs &= -pending_irqs; + + /* Disable all interrupts but the first pending one */ + for (j = 0; j < nr_irq_groups; j++) { + u32 new_mask = j == i ? pending_irqs : 0; + + if (new_mask != handler->enable_save[j]) + writel_relaxed(new_mask, enable + j); + } + return pending_irqs; +} + +static irq_hw_number_t cp100_get_hwirq(struct plic_handler *handler, void __iomem *claim) +{ + int nr_irq_groups = DIV_ROUND_UP(handler->priv->nr_irqs, 32); + u32 __iomem *enable = handler->enable_base; + irq_hw_number_t hwirq = 0; + u32 iso_mask; + int i; + + guard(raw_spinlock)(&handler->enable_lock); + + /* Existing enable state is already cached in enable_save */ + iso_mask = cp100_isolate_pending_irq(nr_irq_groups, handler); + if (!iso_mask) + return 0; + + /* + * Interrupts delievered to hardware still become pending, but only + * interrupts that are both pending and enabled can be claimed. + * Clearing the enable bit for all interrupts but the first pending + * one avoids a hardware bug that occurs during read from the claim + * register with more than one eligible interrupt. + */ + hwirq = readl(claim); + + /* Restore previous state */ + for (i = 0; i < nr_irq_groups; i++) { + u32 written = i == hwirq / 32 ? iso_mask : 0; + u32 stored = handler->enable_save[i]; + + if (stored != written) + writel_relaxed(stored, enable + i); + } + return hwirq; +} + +static void plic_handle_irq_cp100(struct irq_desc *desc) +{ + struct plic_handler *handler = this_cpu_ptr(&plic_handlers); + struct irq_chip *chip = irq_desc_get_chip(desc); + void __iomem *claim = handler->hart_base + CONTEXT_CLAIM; + irq_hw_number_t hwirq; + + WARN_ON_ONCE(!handler->present); + + chained_irq_enter(chip, desc); + + while ((hwirq = cp100_get_hwirq(handler, claim))) { + int err = generic_handle_domain_irq(handler->priv->irqdomain, hwirq); + + if (unlikely(err)) { + pr_warn_ratelimited("%pfwP: can't find mapping for hwirq %lu\n", + handler->priv->fwnode, hwirq); + } + } + + chained_irq_exit(chip, desc); +} + static void plic_set_threshold(struct plic_handler *handler, u32 threshold) { /* priority must be > threshold to trigger an interrupt */ @@ -434,6 +519,8 @@ static const struct of_device_id plic_match[] = { .data = (const void *)BIT(PLIC_QUIRK_EDGE_INTERRUPT) }, { .compatible = "thead,c900-plic", .data = (const void *)BIT(PLIC_QUIRK_EDGE_INTERRUPT) }, + { .compatible = "ultrarisc,cp100-plic", + .data = (const void *)BIT(PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM) }, {} }; @@ -592,12 +679,11 @@ static int plic_probe(struct fwnode_handle *fwnode) if (parent_hwirq != RV_IRQ_EXT) { /* Disable S-mode enable bits if running in M-mode. */ if (IS_ENABLED(CONFIG_RISCV_M_MODE)) { - void __iomem *enable_base = priv->regs + - CONTEXT_ENABLE_BASE + - i * CONTEXT_ENABLE_SIZE; + u32 __iomem *enable_base = priv->regs + CONTEXT_ENABLE_BASE + + i * CONTEXT_ENABLE_SIZE; - for (hwirq = 1; hwirq <= nr_irqs; hwirq++) - __plic_toggle(enable_base, hwirq, 0); + for (int j = 0; j <= nr_irqs / 32; j++) + writel(0, enable_base + j); } continue; } @@ -668,12 +754,17 @@ done: } if (global_setup) { + void (*handler_fn)(struct irq_desc *) = plic_handle_irq; + + if (test_bit(PLIC_QUIRK_CP100_CLAIM_REGISTER_ERRATUM, &handler->priv->plic_quirks)) + handler_fn = plic_handle_irq_cp100; + /* Find parent domain and register chained handler */ domain = irq_find_matching_fwnode(riscv_get_intc_hwnode(), DOMAIN_BUS_ANY); if (domain) plic_parent_irq = irq_create_mapping(domain, RV_IRQ_EXT); if (plic_parent_irq) - irq_set_chained_handler(plic_parent_irq, plic_handle_irq); + irq_set_chained_handler(plic_parent_irq, handler_fn); cpuhp_setup_state(CPUHP_AP_IRQ_SIFIVE_PLIC_STARTING, "irqchip/sifive/plic:starting", diff --git a/drivers/irqchip/irq-starfive-jh8100-intc.c b/drivers/irqchip/irq-starfive-jh8100-intc.c index 2460798ec158..705361b4ebe0 100644 --- a/drivers/irqchip/irq-starfive-jh8100-intc.c +++ b/drivers/irqchip/irq-starfive-jh8100-intc.c @@ -114,9 +114,9 @@ static void starfive_intc_irq_handler(struct irq_desc *desc) chained_irq_exit(chip, desc); } -static int __init starfive_intc_init(struct device_node *intc, - struct device_node *parent) +static int starfive_intc_probe(struct platform_device *pdev, struct device_node *parent) { + struct device_node *intc = pdev->dev.of_node; struct starfive_irq_chip *irqc; struct reset_control *rst; struct clk *clk; @@ -199,7 +199,7 @@ err_free: } IRQCHIP_PLATFORM_DRIVER_BEGIN(starfive_intc) -IRQCHIP_MATCH("starfive,jh8100-intc", starfive_intc_init) +IRQCHIP_MATCH("starfive,jh8100-intc", starfive_intc_probe) IRQCHIP_PLATFORM_DRIVER_END(starfive_intc) MODULE_DESCRIPTION("StarFive JH8100 External Interrupt Controller"); diff --git a/drivers/irqchip/irq-ts4800.c b/drivers/irqchip/irq-ts4800.c index 1e236d5b7516..2e4013c6834d 100644 --- a/drivers/irqchip/irq-ts4800.c +++ b/drivers/irqchip/irq-ts4800.c @@ -165,4 +165,3 @@ module_platform_driver(ts4800_ic_driver); MODULE_AUTHOR("Damien Riegel <damien.riegel@savoirfairelinux.com>"); MODULE_DESCRIPTION("Multiplexed-IRQs driver for TS-4800's FPGA"); MODULE_LICENSE("GPL v2"); -MODULE_ALIAS("platform:ts4800_irqc"); diff --git a/drivers/irqchip/irqchip.c b/drivers/irqchip/irqchip.c index 0ee7b6b71f5f..689c8e448901 100644 --- a/drivers/irqchip/irqchip.c +++ b/drivers/irqchip/irqchip.c @@ -36,11 +36,10 @@ int platform_irqchip_probe(struct platform_device *pdev) { struct device_node *np = pdev->dev.of_node; struct device_node *par_np __free(device_node) = of_irq_find_parent(np); - of_irq_init_cb_t irq_init_cb = of_device_get_match_data(&pdev->dev); + platform_irq_probe_t irq_probe = of_device_get_match_data(&pdev->dev); - if (!irq_init_cb) { + if (!irq_probe) return -EINVAL; - } if (par_np == np) par_np = NULL; @@ -53,10 +52,9 @@ int platform_irqchip_probe(struct platform_device *pdev) * interrupt controller. The actual initialization callback of this * interrupt controller can check for specific domains as necessary. */ - if (par_np && !irq_find_matching_host(par_np, DOMAIN_BUS_ANY)) { + if (par_np && !irq_find_matching_host(par_np, DOMAIN_BUS_ANY)) return -EPROBE_DEFER; - } - return irq_init_cb(np, par_np); + return irq_probe(pdev, par_np); } EXPORT_SYMBOL_GPL(platform_irqchip_probe); diff --git a/drivers/irqchip/qcom-irq-combiner.c b/drivers/irqchip/qcom-irq-combiner.c index 18e696dc7f4d..09819007d08e 100644 --- a/drivers/irqchip/qcom-irq-combiner.c +++ b/drivers/irqchip/qcom-irq-combiner.c @@ -222,7 +222,7 @@ static int get_registers(struct platform_device *pdev, struct combiner *comb) return 0; } -static int __init combiner_probe(struct platform_device *pdev) +static int combiner_probe(struct platform_device *pdev) { struct combiner *combiner; int nregs; @@ -266,11 +266,11 @@ static const struct acpi_device_id qcom_irq_combiner_ids[] = { { } }; -static struct platform_driver qcom_irq_combiner_probe = { +static struct platform_driver qcom_irq_combiner_driver = { .driver = { .name = "qcom-irq-combiner", .acpi_match_table = ACPI_PTR(qcom_irq_combiner_ids), }, .probe = combiner_probe, }; -builtin_platform_driver(qcom_irq_combiner_probe); +builtin_platform_driver(qcom_irq_combiner_driver); diff --git a/drivers/irqchip/qcom-pdc.c b/drivers/irqchip/qcom-pdc.c index 52d77546aacb..518f7f0f3dab 100644 --- a/drivers/irqchip/qcom-pdc.c +++ b/drivers/irqchip/qcom-pdc.c @@ -350,9 +350,10 @@ static int pdc_setup_pin_mapping(struct device_node *np) #define QCOM_PDC_SIZE 0x30000 -static int qcom_pdc_init(struct device_node *node, struct device_node *parent) +static int qcom_pdc_probe(struct platform_device *pdev, struct device_node *parent) { struct irq_domain *parent_domain, *pdc_domain; + struct device_node *node = pdev->dev.of_node; resource_size_t res_size; struct resource res; int ret; @@ -428,7 +429,7 @@ fail: } IRQCHIP_PLATFORM_DRIVER_BEGIN(qcom_pdc) -IRQCHIP_MATCH("qcom,pdc", qcom_pdc_init) +IRQCHIP_MATCH("qcom,pdc", qcom_pdc_probe) IRQCHIP_PLATFORM_DRIVER_END(qcom_pdc) MODULE_DESCRIPTION("Qualcomm Technologies, Inc. Power Domain Controller"); MODULE_LICENSE("GPL v2"); diff --git a/drivers/mailbox/mailbox-test.c b/drivers/mailbox/mailbox-test.c index c9dd8c42c0cd..3a28ab5c42e5 100644 --- a/drivers/mailbox/mailbox-test.c +++ b/drivers/mailbox/mailbox-test.c @@ -268,7 +268,7 @@ static int mbox_test_add_debugfs(struct platform_device *pdev, return 0; tdev->root_debugfs_dir = debugfs_create_dir(dev_name(&pdev->dev), NULL); - if (!tdev->root_debugfs_dir) { + if (IS_ERR(tdev->root_debugfs_dir)) { dev_err(&pdev->dev, "Failed to create Mailbox debugfs\n"); return -EINVAL; } diff --git a/drivers/mailbox/mailbox-th1520.c b/drivers/mailbox/mailbox-th1520.c index a6b2aa9ae952..626957c2e435 100644 --- a/drivers/mailbox/mailbox-th1520.c +++ b/drivers/mailbox/mailbox-th1520.c @@ -435,10 +435,8 @@ static int th1520_mbox_probe(struct platform_device *pdev) } ret = devm_add_action_or_reset(dev, th1520_disable_clk, priv); - if (ret) { - clk_bulk_disable_unprepare(ARRAY_SIZE(priv->clocks), priv->clocks); + if (ret) return ret; - } /* * The address mappings in the device tree align precisely with those diff --git a/drivers/mailbox/mtk-cmdq-mailbox.c b/drivers/mailbox/mtk-cmdq-mailbox.c index 654a60f63756..5791f80f995a 100644 --- a/drivers/mailbox/mtk-cmdq-mailbox.c +++ b/drivers/mailbox/mtk-cmdq-mailbox.c @@ -92,6 +92,18 @@ struct gce_plat { u32 gce_num; }; +static inline u32 cmdq_convert_gce_addr(dma_addr_t addr, const struct gce_plat *pdata) +{ + /* Convert DMA addr (PA or IOVA) to GCE readable addr */ + return addr >> pdata->shift; +} + +static inline dma_addr_t cmdq_revert_gce_addr(u32 addr, const struct gce_plat *pdata) +{ + /* Revert GCE readable addr to DMA addr (PA or IOVA) */ + return (dma_addr_t)addr << pdata->shift; +} + u8 cmdq_get_shift_pa(struct mbox_chan *chan) { struct cmdq *cmdq = container_of(chan->mbox, struct cmdq, mbox); @@ -188,13 +200,12 @@ static void cmdq_task_insert_into_thread(struct cmdq_task *task) struct cmdq_task *prev_task = list_last_entry( &thread->task_busy_list, typeof(*task), list_entry); u64 *prev_task_base = prev_task->pkt->va_base; + u32 gce_addr = cmdq_convert_gce_addr(task->pa_base, task->cmdq->pdata); /* let previous task jump to this task */ dma_sync_single_for_cpu(dev, prev_task->pa_base, prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE); - prev_task_base[CMDQ_NUM_CMD(prev_task->pkt) - 1] = - (u64)CMDQ_JUMP_BY_PA << 32 | - (task->pa_base >> task->cmdq->pdata->shift); + prev_task_base[CMDQ_NUM_CMD(prev_task->pkt) - 1] = (u64)CMDQ_JUMP_BY_PA << 32 | gce_addr; dma_sync_single_for_device(dev, prev_task->pa_base, prev_task->pkt->cmd_buf_size, DMA_TO_DEVICE); @@ -237,7 +248,8 @@ static void cmdq_thread_irq_handler(struct cmdq *cmdq, struct cmdq_thread *thread) { struct cmdq_task *task, *tmp, *curr_task = NULL; - u32 curr_pa, irq_flag, task_end_pa; + u32 irq_flag, gce_addr; + dma_addr_t curr_pa, task_end_pa; bool err; irq_flag = readl(thread->base + CMDQ_THR_IRQ_STATUS); @@ -259,7 +271,8 @@ static void cmdq_thread_irq_handler(struct cmdq *cmdq, else return; - curr_pa = readl(thread->base + CMDQ_THR_CURR_ADDR) << cmdq->pdata->shift; + gce_addr = readl(thread->base + CMDQ_THR_CURR_ADDR); + curr_pa = cmdq_revert_gce_addr(gce_addr, cmdq->pdata); list_for_each_entry_safe(task, tmp, &thread->task_busy_list, list_entry) { @@ -378,7 +391,8 @@ static int cmdq_mbox_send_data(struct mbox_chan *chan, void *data) struct cmdq_thread *thread = (struct cmdq_thread *)chan->con_priv; struct cmdq *cmdq = dev_get_drvdata(chan->mbox->dev); struct cmdq_task *task; - unsigned long curr_pa, end_pa; + u32 gce_addr; + dma_addr_t curr_pa, end_pa; /* Client should not flush new tasks if suspended. */ WARN_ON(cmdq->suspended); @@ -402,20 +416,20 @@ static int cmdq_mbox_send_data(struct mbox_chan *chan, void *data) */ WARN_ON(cmdq_thread_reset(cmdq, thread) < 0); - writel(task->pa_base >> cmdq->pdata->shift, - thread->base + CMDQ_THR_CURR_ADDR); - writel((task->pa_base + pkt->cmd_buf_size) >> cmdq->pdata->shift, - thread->base + CMDQ_THR_END_ADDR); + gce_addr = cmdq_convert_gce_addr(task->pa_base, cmdq->pdata); + writel(gce_addr, thread->base + CMDQ_THR_CURR_ADDR); + gce_addr = cmdq_convert_gce_addr(task->pa_base + pkt->cmd_buf_size, cmdq->pdata); + writel(gce_addr, thread->base + CMDQ_THR_END_ADDR); writel(thread->priority, thread->base + CMDQ_THR_PRIORITY); writel(CMDQ_THR_IRQ_EN, thread->base + CMDQ_THR_IRQ_ENABLE); writel(CMDQ_THR_ENABLED, thread->base + CMDQ_THR_ENABLE_TASK); } else { WARN_ON(cmdq_thread_suspend(cmdq, thread) < 0); - curr_pa = readl(thread->base + CMDQ_THR_CURR_ADDR) << - cmdq->pdata->shift; - end_pa = readl(thread->base + CMDQ_THR_END_ADDR) << - cmdq->pdata->shift; + gce_addr = readl(thread->base + CMDQ_THR_CURR_ADDR); + curr_pa = cmdq_revert_gce_addr(gce_addr, cmdq->pdata); + gce_addr = readl(thread->base + CMDQ_THR_END_ADDR); + end_pa = cmdq_revert_gce_addr(gce_addr, cmdq->pdata); /* check boundary */ if (curr_pa == end_pa - CMDQ_INST_SIZE || curr_pa == end_pa) { @@ -646,6 +660,9 @@ static int cmdq_probe(struct platform_device *pdev) if (err) return err; + dma_set_coherent_mask(dev, + DMA_BIT_MASK(sizeof(u32) * BITS_PER_BYTE + cmdq->pdata->shift)); + cmdq->mbox.dev = dev; cmdq->mbox.chans = devm_kcalloc(dev, cmdq->pdata->thread_nr, sizeof(*cmdq->mbox.chans), GFP_KERNEL); diff --git a/drivers/mailbox/mtk-gpueb-mailbox.c b/drivers/mailbox/mtk-gpueb-mailbox.c index 925bcf21f650..f6d2beccd91b 100644 --- a/drivers/mailbox/mtk-gpueb-mailbox.c +++ b/drivers/mailbox/mtk-gpueb-mailbox.c @@ -200,7 +200,7 @@ static bool mtk_gpueb_mbox_last_tx_done(struct mbox_chan *chan) return !(readl(ch->ebm->mbox_ctl + GPUEB_MBOX_CTL_TX_STS) & BIT(ch->num)); } -const struct mbox_chan_ops mtk_gpueb_mbox_ops = { +static const struct mbox_chan_ops mtk_gpueb_mbox_ops = { .send_data = mtk_gpueb_mbox_send_data, .startup = mtk_gpueb_mbox_startup, .shutdown = mtk_gpueb_mbox_shutdown, diff --git a/drivers/mailbox/omap-mailbox.c b/drivers/mailbox/omap-mailbox.c index 680243751d62..17fe6545875d 100644 --- a/drivers/mailbox/omap-mailbox.c +++ b/drivers/mailbox/omap-mailbox.c @@ -68,6 +68,7 @@ struct omap_mbox_fifo { struct omap_mbox_match_data { u32 intr_type; + bool is_exclusive; }; struct omap_mbox_device { @@ -78,6 +79,7 @@ struct omap_mbox_device { u32 num_users; u32 num_fifos; u32 intr_type; + const struct omap_mbox_match_data *mbox_data; }; struct omap_mbox { @@ -341,11 +343,13 @@ static int omap_mbox_suspend(struct device *dev) if (pm_runtime_status_suspended(dev)) return 0; - for (fifo = 0; fifo < mdev->num_fifos; fifo++) { - if (mbox_read_reg(mdev, MAILBOX_MSGSTATUS(fifo))) { - dev_err(mdev->dev, "fifo %d has unexpected unread messages\n", - fifo); - return -EBUSY; + if (mdev->mbox_data->is_exclusive) { + for (fifo = 0; fifo < mdev->num_fifos; fifo++) { + if (mbox_read_reg(mdev, MAILBOX_MSGSTATUS(fifo))) { + dev_err(mdev->dev, "fifo %d has unexpected unread messages\n", + fifo); + return -EBUSY; + } } } @@ -378,8 +382,9 @@ static const struct dev_pm_ops omap_mbox_pm_ops = { SET_SYSTEM_SLEEP_PM_OPS(omap_mbox_suspend, omap_mbox_resume) }; -static const struct omap_mbox_match_data omap2_data = { MBOX_INTR_CFG_TYPE1 }; -static const struct omap_mbox_match_data omap4_data = { MBOX_INTR_CFG_TYPE2 }; +static const struct omap_mbox_match_data omap2_data = { MBOX_INTR_CFG_TYPE1, true }; +static const struct omap_mbox_match_data omap4_data = { MBOX_INTR_CFG_TYPE2, true }; +static const struct omap_mbox_match_data am654_data = { MBOX_INTR_CFG_TYPE2, false }; static const struct of_device_id omap_mailbox_of_match[] = { { @@ -396,11 +401,11 @@ static const struct of_device_id omap_mailbox_of_match[] = { }, { .compatible = "ti,am654-mailbox", - .data = &omap4_data, + .data = &am654_data, }, { .compatible = "ti,am64-mailbox", - .data = &omap4_data, + .data = &am654_data, }, { /* end */ @@ -449,7 +454,6 @@ static int omap_mbox_probe(struct platform_device *pdev) struct omap_mbox_fifo *fifo; struct device_node *node = pdev->dev.of_node; struct device_node *child; - const struct omap_mbox_match_data *match_data; struct mbox_controller *controller; u32 intr_type, info_count; u32 num_users, num_fifos; @@ -462,11 +466,6 @@ static int omap_mbox_probe(struct platform_device *pdev) return -ENODEV; } - match_data = of_device_get_match_data(&pdev->dev); - if (!match_data) - return -ENODEV; - intr_type = match_data->intr_type; - if (of_property_read_u32(node, "ti,mbox-num-users", &num_users)) return -ENODEV; @@ -483,6 +482,12 @@ static int omap_mbox_probe(struct platform_device *pdev) if (!mdev) return -ENOMEM; + mdev->mbox_data = device_get_match_data(&pdev->dev); + if (!mdev->mbox_data) + return -ENODEV; + + intr_type = mdev->mbox_data->intr_type; + mdev->mbox_base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(mdev->mbox_base)) return PTR_ERR(mdev->mbox_base); diff --git a/drivers/mailbox/pcc.c b/drivers/mailbox/pcc.c index 0a00719b2482..ff292b9e0be9 100644 --- a/drivers/mailbox/pcc.c +++ b/drivers/mailbox/pcc.c @@ -276,9 +276,8 @@ static int pcc_mbox_error_check_and_clear(struct pcc_chan_info *pchan) if (ret) return ret; - val &= pchan->error.status_mask; - if (val) { - val &= ~pchan->error.status_mask; + if (val & pchan->error.status_mask) { + val &= pchan->error.preserve_mask; pcc_chan_reg_write(&pchan->error, val); return -EIO; } @@ -745,7 +744,8 @@ static int pcc_parse_subspace_db_reg(struct pcc_chan_info *pchan, ret = pcc_chan_reg_init(&pchan->error, &pcct_ext->error_status_register, - 0, 0, pcct_ext->error_status_mask, + ~pcct_ext->error_status_mask, 0, + pcct_ext->error_status_mask, "Error Status"); } return ret; diff --git a/drivers/md/bcache/alloc.c b/drivers/md/bcache/alloc.c index 48ce750bf70a..7708d92df23e 100644 --- a/drivers/md/bcache/alloc.c +++ b/drivers/md/bcache/alloc.c @@ -24,21 +24,18 @@ * Since the gens and priorities are all stored contiguously on disk, we can * batch this up: We fill up the free_inc list with freshly invalidated buckets, * call prio_write(), and when prio_write() finishes we pull buckets off the - * free_inc list and optionally discard them. + * free_inc list. * * free_inc isn't the only freelist - if it was, we'd often to sleep while * priorities and gens were being written before we could allocate. c->free is a * smaller freelist, and buckets on that list are always ready to be used. * - * If we've got discards enabled, that happens when a bucket moves from the - * free_inc list to the free list. - * * There is another freelist, because sometimes we have buckets that we know * have nothing pointing into them - these we can reuse without waiting for * priorities to be rewritten. These come from freed btree nodes and buckets * that garbage collection discovered no longer had valid keys pointing into * them (because they were overwritten). That's the unused list - buckets on the - * unused list move to the free list, optionally being discarded in the process. + * unused list move to the free list. * * It's also important to ensure that gens don't wrap around - with respect to * either the oldest gen in the btree or the gen on disk. This is quite @@ -118,8 +115,7 @@ void bch_rescale_priorities(struct cache_set *c, int sectors) /* * Background allocation thread: scans for buckets to be invalidated, * invalidates them, rewrites prios/gens (marking them as invalidated on disk), - * then optionally issues discard commands to the newly free buckets, then puts - * them on the various freelists. + * then puts them on the various freelists. */ static inline bool can_inc_bucket_gen(struct bucket *b) @@ -321,8 +317,7 @@ static int bch_allocator_thread(void *arg) while (1) { /* * First, we pull buckets off of the unused and free_inc lists, - * possibly issue discards to them, then we add the bucket to - * the free list: + * then we add the bucket to the free list: */ while (1) { long bucket; @@ -330,14 +325,6 @@ static int bch_allocator_thread(void *arg) if (!fifo_pop(&ca->free_inc, bucket)) break; - if (ca->discard) { - mutex_unlock(&ca->set->bucket_lock); - blkdev_issue_discard(ca->bdev, - bucket_to_sector(ca->set, bucket), - ca->sb.bucket_size, GFP_KERNEL); - mutex_lock(&ca->set->bucket_lock); - } - allocator_wait(ca, bch_allocator_push(ca, bucket)); wake_up(&ca->set->btree_cache_wait); wake_up(&ca->set->bucket_wait); @@ -412,7 +399,11 @@ long bch_bucket_alloc(struct cache *ca, unsigned int reserve, bool wait) TASK_UNINTERRUPTIBLE); mutex_unlock(&ca->set->bucket_lock); + + atomic_inc(&ca->set->bucket_wait_cnt); schedule(); + atomic_dec(&ca->set->bucket_wait_cnt); + mutex_lock(&ca->set->bucket_lock); } while (!fifo_pop(&ca->free[RESERVE_NONE], r) && !fifo_pop(&ca->free[reserve], r)); diff --git a/drivers/md/bcache/bcache.h b/drivers/md/bcache/bcache.h index 1d33e40d26ea..8ccacba85547 100644 --- a/drivers/md/bcache/bcache.h +++ b/drivers/md/bcache/bcache.h @@ -447,8 +447,7 @@ struct cache { * free_inc: Incoming buckets - these are buckets that currently have * cached data in them, and we can't reuse them until after we write * their new gen to disk. After prio_write() finishes writing the new - * gens/prios, they'll be moved to the free list (and possibly discarded - * in the process) + * gens/prios, they'll be moved to the free list. */ DECLARE_FIFO(long, free)[RESERVE_NR]; DECLARE_FIFO(long, free_inc); @@ -467,8 +466,6 @@ struct cache { */ unsigned int invalidate_needs_gc; - bool discard; /* Get rid of? */ - struct journal_device journal; /* The rest of this all shows up in sysfs */ @@ -607,6 +604,7 @@ struct cache_set { */ atomic_t prio_blocked; wait_queue_head_t bucket_wait; + atomic_t bucket_wait_cnt; /* * For any bio we don't skip we subtract the number of sectors from diff --git a/drivers/md/bcache/bset.h b/drivers/md/bcache/bset.h index 011f6062c4c0..6ee2c6a506a2 100644 --- a/drivers/md/bcache/bset.h +++ b/drivers/md/bcache/bset.h @@ -327,9 +327,13 @@ struct btree_iter { /* Fixed-size btree_iter that can be allocated on the stack */ struct btree_iter_stack { - struct btree_iter iter; - struct btree_iter_set stack_data[MAX_BSETS]; + /* Must be last as it ends in a flexible-array member. */ + TRAILING_OVERLAP(struct btree_iter, iter, data, + struct btree_iter_set stack_data[MAX_BSETS]; + ); }; +static_assert(offsetof(struct btree_iter_stack, iter.data) == + offsetof(struct btree_iter_stack, stack_data)); typedef bool (*ptr_filter_fn)(struct btree_keys *b, const struct bkey *k); diff --git a/drivers/md/bcache/btree.c b/drivers/md/bcache/btree.c index 210b59007d98..3ed39c823826 100644 --- a/drivers/md/bcache/btree.c +++ b/drivers/md/bcache/btree.c @@ -89,8 +89,9 @@ * Test module load/unload */ -#define MAX_GC_TIMES 100 -#define MIN_GC_NODES 100 +#define MAX_GC_TIMES_SHIFT 7 /* 128 loops */ +#define GC_NODES_MIN 10 +#define GC_SLEEP_MS_MIN 10 #define GC_SLEEP_MS 100 #define PTR_DIRTY_BIT (((uint64_t) 1 << 36)) @@ -371,7 +372,7 @@ static void do_btree_node_write(struct btree *b) SET_PTR_OFFSET(&k.key, 0, PTR_OFFSET(&k.key, 0) + bset_sector_offset(&b->keys, i)); - if (!bch_bio_alloc_pages(b->bio, __GFP_NOWARN|GFP_NOWAIT)) { + if (!bch_bio_alloc_pages(b->bio, GFP_NOWAIT)) { struct bio_vec *bv; void *addr = (void *) ((unsigned long) i & ~(PAGE_SIZE - 1)); struct bvec_iter_all iter_all; @@ -1578,29 +1579,29 @@ static unsigned int btree_gc_count_keys(struct btree *b) static size_t btree_gc_min_nodes(struct cache_set *c) { - size_t min_nodes; + size_t min_nodes = GC_NODES_MIN; - /* - * Since incremental GC would stop 100ms when front - * side I/O comes, so when there are many btree nodes, - * if GC only processes constant (100) nodes each time, - * GC would last a long time, and the front side I/Os - * would run out of the buckets (since no new bucket - * can be allocated during GC), and be blocked again. - * So GC should not process constant nodes, but varied - * nodes according to the number of btree nodes, which - * realized by dividing GC into constant(100) times, - * so when there are many btree nodes, GC can process - * more nodes each time, otherwise, GC will process less - * nodes each time (but no less than MIN_GC_NODES) - */ - min_nodes = c->gc_stats.nodes / MAX_GC_TIMES; - if (min_nodes < MIN_GC_NODES) - min_nodes = MIN_GC_NODES; + if (atomic_read(&c->search_inflight) == 0) { + size_t n = c->gc_stats.nodes >> MAX_GC_TIMES_SHIFT; + + if (min_nodes < n) + min_nodes = n; + } return min_nodes; } +static uint64_t btree_gc_sleep_ms(struct cache_set *c) +{ + uint64_t sleep_ms; + + if (atomic_read(&c->bucket_wait_cnt) > 0) + sleep_ms = GC_SLEEP_MS_MIN; + else + sleep_ms = GC_SLEEP_MS; + + return sleep_ms; +} static int btree_gc_recurse(struct btree *b, struct btree_op *op, struct closure *writes, struct gc_stat *gc) @@ -1668,8 +1669,7 @@ static int btree_gc_recurse(struct btree *b, struct btree_op *op, memmove(r + 1, r, sizeof(r[0]) * (GC_MERGE_NODES - 1)); r->b = NULL; - if (atomic_read(&b->c->search_inflight) && - gc->nodes >= gc->nodes_pre + btree_gc_min_nodes(b->c)) { + if (gc->nodes >= (gc->nodes_pre + btree_gc_min_nodes(b->c))) { gc->nodes_pre = gc->nodes; ret = -EAGAIN; break; @@ -1846,8 +1846,8 @@ static void bch_btree_gc(struct cache_set *c) cond_resched(); if (ret == -EAGAIN) - schedule_timeout_interruptible(msecs_to_jiffies - (GC_SLEEP_MS)); + schedule_timeout_interruptible( + msecs_to_jiffies(btree_gc_sleep_ms(c))); else if (ret) pr_warn("gc failed!\n"); } while (ret && !test_bit(CACHE_SET_IO_DISABLE, &c->flags)); @@ -2822,7 +2822,8 @@ void bch_btree_exit(void) int __init bch_btree_init(void) { - btree_io_wq = alloc_workqueue("bch_btree_io", WQ_MEM_RECLAIM, 0); + btree_io_wq = alloc_workqueue("bch_btree_io", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!btree_io_wq) return -ENOMEM; diff --git a/drivers/md/bcache/journal.c b/drivers/md/bcache/journal.c index d50eb82ccb4f..144693b7c46a 100644 --- a/drivers/md/bcache/journal.c +++ b/drivers/md/bcache/journal.c @@ -275,8 +275,7 @@ bsearch: * ja->cur_idx */ ja->cur_idx = i; - ja->last_idx = ja->discard_idx = (i + 1) % - ca->sb.njournal_buckets; + ja->last_idx = (i + 1) % ca->sb.njournal_buckets; } @@ -336,16 +335,6 @@ void bch_journal_mark(struct cache_set *c, struct list_head *list) } } -static bool is_discard_enabled(struct cache_set *s) -{ - struct cache *ca = s->cache; - - if (ca->discard) - return true; - - return false; -} - int bch_journal_replay(struct cache_set *s, struct list_head *list) { int ret = 0, keys = 0, entries = 0; @@ -360,15 +349,10 @@ int bch_journal_replay(struct cache_set *s, struct list_head *list) BUG_ON(i->pin && atomic_read(i->pin) != 1); if (n != i->j.seq) { - if (n == start && is_discard_enabled(s)) - pr_info("journal entries %llu-%llu may be discarded! (replaying %llu-%llu)\n", - n, i->j.seq - 1, start, end); - else { - pr_err("journal entries %llu-%llu missing! (replaying %llu-%llu)\n", - n, i->j.seq - 1, start, end); - ret = -EIO; - goto err; - } + pr_err("journal entries %llu-%llu missing! (replaying %llu-%llu)\n", + n, i->j.seq - 1, start, end); + ret = -EIO; + goto err; } for (k = i->j.start; @@ -568,65 +552,6 @@ out: #define last_seq(j) ((j)->seq - fifo_used(&(j)->pin) + 1) -static void journal_discard_endio(struct bio *bio) -{ - struct journal_device *ja = - container_of(bio, struct journal_device, discard_bio); - struct cache *ca = container_of(ja, struct cache, journal); - - atomic_set(&ja->discard_in_flight, DISCARD_DONE); - - closure_wake_up(&ca->set->journal.wait); - closure_put(&ca->set->cl); -} - -static void journal_discard_work(struct work_struct *work) -{ - struct journal_device *ja = - container_of(work, struct journal_device, discard_work); - - submit_bio(&ja->discard_bio); -} - -static void do_journal_discard(struct cache *ca) -{ - struct journal_device *ja = &ca->journal; - struct bio *bio = &ja->discard_bio; - - if (!ca->discard) { - ja->discard_idx = ja->last_idx; - return; - } - - switch (atomic_read(&ja->discard_in_flight)) { - case DISCARD_IN_FLIGHT: - return; - - case DISCARD_DONE: - ja->discard_idx = (ja->discard_idx + 1) % - ca->sb.njournal_buckets; - - atomic_set(&ja->discard_in_flight, DISCARD_READY); - fallthrough; - - case DISCARD_READY: - if (ja->discard_idx == ja->last_idx) - return; - - atomic_set(&ja->discard_in_flight, DISCARD_IN_FLIGHT); - - bio_init_inline(bio, ca->bdev, 1, REQ_OP_DISCARD); - bio->bi_iter.bi_sector = bucket_to_sector(ca->set, - ca->sb.d[ja->discard_idx]); - bio->bi_iter.bi_size = bucket_bytes(ca); - bio->bi_end_io = journal_discard_endio; - - closure_get(&ca->set->cl); - INIT_WORK(&ja->discard_work, journal_discard_work); - queue_work(bch_journal_wq, &ja->discard_work); - } -} - static unsigned int free_journal_buckets(struct cache_set *c) { struct journal *j = &c->journal; @@ -635,10 +560,10 @@ static unsigned int free_journal_buckets(struct cache_set *c) unsigned int n; /* In case njournal_buckets is not power of 2 */ - if (ja->cur_idx >= ja->discard_idx) - n = ca->sb.njournal_buckets + ja->discard_idx - ja->cur_idx; + if (ja->cur_idx >= ja->last_idx) + n = ca->sb.njournal_buckets + ja->last_idx - ja->cur_idx; else - n = ja->discard_idx - ja->cur_idx; + n = ja->last_idx - ja->cur_idx; if (n > (1 + j->do_reserve)) return n - (1 + j->do_reserve); @@ -668,8 +593,6 @@ static void journal_reclaim(struct cache_set *c) ja->last_idx = (ja->last_idx + 1) % ca->sb.njournal_buckets; - do_journal_discard(ca); - if (c->journal.blocks_free) goto out; diff --git a/drivers/md/bcache/journal.h b/drivers/md/bcache/journal.h index cd316b4a1e95..9e9d1b3016a5 100644 --- a/drivers/md/bcache/journal.h +++ b/drivers/md/bcache/journal.h @@ -139,19 +139,6 @@ struct journal_device { /* Last journal bucket that still contains an open journal entry */ unsigned int last_idx; - /* Next journal bucket to be discarded */ - unsigned int discard_idx; - -#define DISCARD_READY 0 -#define DISCARD_IN_FLIGHT 1 -#define DISCARD_DONE 2 - /* 1 - discard in flight, -1 - discard completed */ - atomic_t discard_in_flight; - - struct work_struct discard_work; - struct bio discard_bio; - struct bio_vec discard_bv; - /* Bio for journal reads/writes to this device */ struct bio bio; struct bio_vec bv[8]; diff --git a/drivers/md/bcache/super.c b/drivers/md/bcache/super.c index 6d250e366412..c17d4517af22 100644 --- a/drivers/md/bcache/super.c +++ b/drivers/md/bcache/super.c @@ -1388,7 +1388,7 @@ static CLOSURE_CALLBACK(cached_dev_flush) bch_cache_accounting_destroy(&dc->accounting); kobject_del(&d->kobj); - continue_at(cl, cached_dev_free, system_wq); + continue_at(cl, cached_dev_free, system_percpu_wq); } static int cached_dev_init(struct cached_dev *dc, unsigned int block_size) @@ -1400,7 +1400,7 @@ static int cached_dev_init(struct cached_dev *dc, unsigned int block_size) __module_get(THIS_MODULE); INIT_LIST_HEAD(&dc->list); closure_init(&dc->disk.cl, NULL); - set_closure_fn(&dc->disk.cl, cached_dev_flush, system_wq); + set_closure_fn(&dc->disk.cl, cached_dev_flush, system_percpu_wq); kobject_init(&dc->disk.kobj, &bch_cached_dev_ktype); INIT_WORK(&dc->detach, cached_dev_detach_finish); sema_init(&dc->sb_write_mutex, 1); @@ -1513,7 +1513,7 @@ static CLOSURE_CALLBACK(flash_dev_flush) bcache_device_unlink(d); mutex_unlock(&bch_register_lock); kobject_del(&d->kobj); - continue_at(cl, flash_dev_free, system_wq); + continue_at(cl, flash_dev_free, system_percpu_wq); } static int flash_dev_run(struct cache_set *c, struct uuid_entry *u) @@ -1525,7 +1525,7 @@ static int flash_dev_run(struct cache_set *c, struct uuid_entry *u) goto err_ret; closure_init(&d->cl, NULL); - set_closure_fn(&d->cl, flash_dev_flush, system_wq); + set_closure_fn(&d->cl, flash_dev_flush, system_percpu_wq); kobject_init(&d->kobj, &bch_flash_dev_ktype); @@ -1833,7 +1833,7 @@ static CLOSURE_CALLBACK(__cache_set_unregister) mutex_unlock(&bch_register_lock); - continue_at(cl, cache_set_flush, system_wq); + continue_at(cl, cache_set_flush, system_percpu_wq); } void bch_cache_set_stop(struct cache_set *c) @@ -1863,10 +1863,10 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb) __module_get(THIS_MODULE); closure_init(&c->cl, NULL); - set_closure_fn(&c->cl, cache_set_free, system_wq); + set_closure_fn(&c->cl, cache_set_free, system_percpu_wq); closure_init(&c->caching, &c->cl); - set_closure_fn(&c->caching, __cache_set_unregister, system_wq); + set_closure_fn(&c->caching, __cache_set_unregister, system_percpu_wq); /* Maybe create continue_at_noreturn() and use it here? */ closure_set_stopped(&c->cl); @@ -1939,7 +1939,8 @@ struct cache_set *bch_cache_set_alloc(struct cache_sb *sb) if (!c->uuids) goto err; - c->moving_gc_wq = alloc_workqueue("bcache_gc", WQ_MEM_RECLAIM, 0); + c->moving_gc_wq = alloc_workqueue("bcache_gc", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!c->moving_gc_wq) goto err; @@ -2382,9 +2383,6 @@ static int register_cache(struct cache_sb *sb, struct cache_sb_disk *sb_disk, ca->bdev = file_bdev(bdev_file); ca->sb_disk = sb_disk; - if (bdev_max_discard_sectors(file_bdev(bdev_file))) - ca->discard = CACHE_DISCARD(&ca->sb); - ret = cache_alloc(ca); if (ret != 0) { if (ret == -ENOMEM) @@ -2531,7 +2529,7 @@ static void register_device_async(struct async_reg_args *args) INIT_DELAYED_WORK(&args->reg_work, register_cache_worker); /* 10 jiffies is enough for a delay */ - queue_delayed_work(system_wq, &args->reg_work, 10); + queue_delayed_work(system_percpu_wq, &args->reg_work, 10); } static void *alloc_holder_object(struct cache_sb *sb) @@ -2905,24 +2903,25 @@ static int __init bcache_init(void) if (bch_btree_init()) goto err; - bcache_wq = alloc_workqueue("bcache", WQ_MEM_RECLAIM, 0); + bcache_wq = alloc_workqueue("bcache", WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!bcache_wq) goto err; /* * Let's not make this `WQ_MEM_RECLAIM` for the following reasons: * - * 1. It used `system_wq` before which also does no memory reclaim. + * 1. It used `system_percpu_wq` before which also does no memory reclaim. * 2. With `WQ_MEM_RECLAIM` desktop stalls, increased boot times, and * reduced throughput can be observed. * - * We still want to user our own queue to not congest the `system_wq`. + * We still want to user our own queue to not congest the `system_percpu_wq`. */ - bch_flush_wq = alloc_workqueue("bch_flush", 0, 0); + bch_flush_wq = alloc_workqueue("bch_flush", WQ_PERCPU, 0); if (!bch_flush_wq) goto err; - bch_journal_wq = alloc_workqueue("bch_journal", WQ_MEM_RECLAIM, 0); + bch_journal_wq = alloc_workqueue("bch_journal", + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!bch_journal_wq) goto err; diff --git a/drivers/md/bcache/sysfs.c b/drivers/md/bcache/sysfs.c index 826b14cae4e5..72f38e5b6f5c 100644 --- a/drivers/md/bcache/sysfs.c +++ b/drivers/md/bcache/sysfs.c @@ -134,7 +134,6 @@ read_attribute(partial_stripes_expensive); rw_attribute(synchronous); rw_attribute(journal_delay_ms); rw_attribute(io_disable); -rw_attribute(discard); rw_attribute(running); rw_attribute(label); rw_attribute(errors); @@ -1036,7 +1035,6 @@ SHOW(__bch_cache) sysfs_hprint(bucket_size, bucket_bytes(ca)); sysfs_hprint(block_size, block_bytes(ca)); sysfs_print(nbuckets, ca->sb.nbuckets); - sysfs_print(discard, ca->discard); sysfs_hprint(written, atomic_long_read(&ca->sectors_written) << 9); sysfs_hprint(btree_written, atomic_long_read(&ca->btree_sectors_written) << 9); @@ -1142,18 +1140,6 @@ STORE(__bch_cache) if (bcache_is_reboot) return -EBUSY; - if (attr == &sysfs_discard) { - bool v = strtoul_or_return(buf); - - if (bdev_max_discard_sectors(ca->bdev)) - ca->discard = v; - - if (v != CACHE_DISCARD(&ca->sb)) { - SET_CACHE_DISCARD(&ca->sb, v); - bcache_write_super(ca->set); - } - } - if (attr == &sysfs_cache_replacement_policy) { v = __sysfs_match_string(cache_replacement_policies, -1, buf); if (v < 0) @@ -1185,7 +1171,6 @@ static struct attribute *bch_cache_attrs[] = { &sysfs_block_size, &sysfs_nbuckets, &sysfs_priority_stats, - &sysfs_discard, &sysfs_written, &sysfs_btree_written, &sysfs_metadata_written, diff --git a/drivers/md/bcache/writeback.c b/drivers/md/bcache/writeback.c index 6ba73dc1a3df..4b237074f453 100644 --- a/drivers/md/bcache/writeback.c +++ b/drivers/md/bcache/writeback.c @@ -805,8 +805,7 @@ static int bch_writeback_thread(void *arg) * may set BCH_ENABLE_AUTO_GC via sysfs, then when * BCH_DO_AUTO_GC is set, garbage collection thread * will be wake up here. After moving gc, the shrunk - * btree and discarded free buckets SSD space may be - * helpful for following write requests. + * btree may be helpful for following write requests. */ if (c->gc_after_writeback == (BCH_ENABLE_AUTO_GC|BCH_DO_AUTO_GC)) { @@ -1076,7 +1075,7 @@ void bch_cached_dev_writeback_init(struct cached_dev *dc) int bch_cached_dev_writeback_start(struct cached_dev *dc) { dc->writeback_write_wq = alloc_workqueue("bcache_writeback_wq", - WQ_MEM_RECLAIM, 0); + WQ_MEM_RECLAIM | WQ_PERCPU, 0); if (!dc->writeback_write_wq) return -ENOMEM; diff --git a/drivers/md/dm-vdo/logger.c b/drivers/md/dm-vdo/logger.c index 3f7dc2cb6b98..76a987ccf926 100644 --- a/drivers/md/dm-vdo/logger.c +++ b/drivers/md/dm-vdo/logger.c @@ -34,7 +34,7 @@ static const char *get_current_interrupt_type(void) if (in_nmi()) return "NMI"; - if (in_irq()) + if (in_hardirq()) return "HI"; if (in_softirq()) diff --git a/drivers/md/dm-zone.c b/drivers/md/dm-zone.c index 78e17dd4d01b..5a840c4ae316 100644 --- a/drivers/md/dm-zone.c +++ b/drivers/md/dm-zone.c @@ -17,33 +17,26 @@ * For internal zone reports bypassing the top BIO submission path. */ static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t, - sector_t sector, unsigned int nr_zones, - report_zones_cb cb, void *data) + unsigned int nr_zones, + struct dm_report_zones_args *args) { - struct gendisk *disk = md->disk; - int ret; - struct dm_report_zones_args args = { - .next_sector = sector, - .orig_data = data, - .orig_cb = cb, - }; - do { struct dm_target *tgt; + int ret; - tgt = dm_table_find_target(t, args.next_sector); + tgt = dm_table_find_target(t, args->next_sector); if (WARN_ON_ONCE(!tgt->type->report_zones)) return -EIO; - args.tgt = tgt; - ret = tgt->type->report_zones(tgt, &args, - nr_zones - args.zone_idx); + args->tgt = tgt; + ret = tgt->type->report_zones(tgt, args, + nr_zones - args->zone_idx); if (ret < 0) return ret; - } while (args.zone_idx < nr_zones && - args.next_sector < get_capacity(disk)); + } while (args->zone_idx < nr_zones && + args->next_sector < get_capacity(md->disk)); - return args.zone_idx; + return args->zone_idx; } /* @@ -52,7 +45,8 @@ static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t, * generally implemented by targets using dm_report_zones(). */ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data) + unsigned int nr_zones, + struct blk_report_zones_args *args) { struct mapped_device *md = disk->private_data; struct dm_table *map; @@ -76,9 +70,14 @@ int dm_blk_report_zones(struct gendisk *disk, sector_t sector, map = zone_revalidate_map; } - if (map) - ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, - data); + if (map) { + struct dm_report_zones_args dm_args = { + .disk = md->disk, + .next_sector = sector, + .rep_args = args, + }; + ret = dm_blk_do_report_zones(md, map, nr_zones, &dm_args); + } if (put_table) dm_put_live_table(md, srcu_idx); @@ -113,7 +112,18 @@ static int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx, } args->next_sector = zone->start + zone->len; - return args->orig_cb(zone, args->zone_idx++, args->orig_data); + + /* If we have an internal callback, call it first. */ + if (args->cb) { + int ret; + + ret = args->cb(zone, args->zone_idx, args->data); + if (ret) + return ret; + } + + return disk_report_zone(args->disk, zone, args->zone_idx++, + args->rep_args); } /* @@ -492,10 +502,15 @@ int dm_zone_get_reset_bitmap(struct mapped_device *md, struct dm_table *t, sector_t sector, unsigned int nr_zones, unsigned long *need_reset) { + struct dm_report_zones_args args = { + .disk = md->disk, + .next_sector = sector, + .cb = dm_zone_need_reset_cb, + .data = need_reset, + }; int ret; - ret = dm_blk_do_report_zones(md, t, sector, nr_zones, - dm_zone_need_reset_cb, need_reset); + ret = dm_blk_do_report_zones(md, t, nr_zones, &args); if (ret != nr_zones) { DMERR("Get %s zone reset bitmap failed\n", md->disk->disk_name); diff --git a/drivers/md/dm.h b/drivers/md/dm.h index 245f52b59215..7a795979ec72 100644 --- a/drivers/md/dm.h +++ b/drivers/md/dm.h @@ -109,7 +109,8 @@ void dm_finalize_zone_settings(struct dm_table *t, struct queue_limits *lim); void dm_zone_endio(struct dm_io *io, struct bio *clone); #ifdef CONFIG_BLK_DEV_ZONED int dm_blk_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data); + unsigned int nr_zones, + struct blk_report_zones_args *args); bool dm_is_zone_write(struct mapped_device *md, struct bio *bio); int dm_zone_get_reset_bitmap(struct mapped_device *md, struct dm_table *t, sector_t sector, unsigned int nr_zones, diff --git a/drivers/md/md-linear.c b/drivers/md/md-linear.c index 7033d982d377..8d7b82c4a723 100644 --- a/drivers/md/md-linear.c +++ b/drivers/md/md-linear.c @@ -72,9 +72,11 @@ static int linear_set_limits(struct mddev *mddev) md_init_stacking_limits(&lim); lim.max_hw_sectors = mddev->chunk_sectors; + lim.logical_block_size = mddev->logical_block_size; lim.max_write_zeroes_sectors = mddev->chunk_sectors; lim.max_hw_wzeroes_unmap_sectors = mddev->chunk_sectors; lim.io_min = mddev->chunk_sectors << 9; + lim.features |= BLK_FEAT_ATOMIC_WRITES; err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY); if (err) return err; diff --git a/drivers/md/md-llbitmap.c b/drivers/md/md-llbitmap.c index 1eb434306162..9c1ade19b774 100644 --- a/drivers/md/md-llbitmap.c +++ b/drivers/md/md-llbitmap.c @@ -378,7 +378,7 @@ static void llbitmap_infect_dirty_bits(struct llbitmap *llbitmap, case BitClean: pctl->state[pos] = BitDirty; break; - }; + } } } diff --git a/drivers/md/md.c b/drivers/md/md.c index 41c476b40c7a..e5922a682953 100644 --- a/drivers/md/md.c +++ b/drivers/md/md.c @@ -99,7 +99,7 @@ static int remove_and_add_spares(struct mddev *mddev, struct md_rdev *this); static void mddev_detach(struct mddev *mddev); static void export_rdev(struct md_rdev *rdev, struct mddev *mddev); -static void md_wakeup_thread_directly(struct md_thread __rcu *thread); +static void md_wakeup_thread_directly(struct md_thread __rcu **thread); /* * Default number of read corrections we'll attempt on an rdev @@ -339,6 +339,7 @@ static int start_readonly; */ static bool create_on_open = true; static bool legacy_async_del_gendisk = true; +static bool check_new_feature = true; /* * We have a system wide 'event count' that is incremented @@ -730,6 +731,8 @@ static void mddev_clear_bitmap_ops(struct mddev *mddev) int mddev_init(struct mddev *mddev) { + int err = 0; + if (!IS_ENABLED(CONFIG_MD_BITMAP)) mddev->bitmap_id = ID_BITMAP_NONE; else @@ -741,10 +744,23 @@ int mddev_init(struct mddev *mddev) if (percpu_ref_init(&mddev->writes_pending, no_op, PERCPU_REF_ALLOW_REINIT, GFP_KERNEL)) { - percpu_ref_exit(&mddev->active_io); - return -ENOMEM; + err = -ENOMEM; + goto exit_acitve_io; } + err = bioset_init(&mddev->bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); + if (err) + goto exit_writes_pending; + + err = bioset_init(&mddev->sync_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); + if (err) + goto exit_bio_set; + + err = bioset_init(&mddev->io_clone_set, BIO_POOL_SIZE, + offsetof(struct md_io_clone, bio_clone), 0); + if (err) + goto exit_sync_set; + /* We want to start with the refcount at zero */ percpu_ref_put(&mddev->writes_pending); @@ -773,11 +789,24 @@ int mddev_init(struct mddev *mddev) INIT_WORK(&mddev->del_work, mddev_delayed_delete); return 0; + +exit_sync_set: + bioset_exit(&mddev->sync_set); +exit_bio_set: + bioset_exit(&mddev->bio_set); +exit_writes_pending: + percpu_ref_exit(&mddev->writes_pending); +exit_acitve_io: + percpu_ref_exit(&mddev->active_io); + return err; } EXPORT_SYMBOL_GPL(mddev_init); void mddev_destroy(struct mddev *mddev) { + bioset_exit(&mddev->bio_set); + bioset_exit(&mddev->sync_set); + bioset_exit(&mddev->io_clone_set); percpu_ref_exit(&mddev->active_io); percpu_ref_exit(&mddev->writes_pending); } @@ -941,8 +970,11 @@ void mddev_unlock(struct mddev *mddev) * do_md_stop. dm raid only uses md_stop to stop. So dm raid * doesn't need to check MD_DELETED when getting reconfig lock */ - if (test_bit(MD_DELETED, &mddev->flags)) + if (test_bit(MD_DELETED, &mddev->flags) && + !test_and_set_bit(MD_DO_DELETE, &mddev->flags)) { + kobject_del(&mddev->kobj); del_gendisk(mddev->gendisk); + } } } EXPORT_SYMBOL_GPL(mddev_unlock); @@ -1820,9 +1852,13 @@ static int super_1_load(struct md_rdev *rdev, struct md_rdev *refdev, int minor_ } if (sb->pad0 || sb->pad3[0] || - memcmp(sb->pad3, sb->pad3+1, sizeof(sb->pad3) - sizeof(sb->pad3[1]))) - /* Some padding is non-zero, might be a new feature */ - return -EINVAL; + memcmp(sb->pad3, sb->pad3+1, sizeof(sb->pad3) - sizeof(sb->pad3[1]))) { + pr_warn("Some padding is non-zero on %pg, might be a new feature\n", + rdev->bdev); + if (check_new_feature) + return -EINVAL; + pr_warn("check_new_feature is disabled, data corruption possible\n"); + } rdev->preferred_minor = 0xffff; rdev->data_offset = le64_to_cpu(sb->data_offset); @@ -1963,6 +1999,7 @@ static int super_1_validate(struct mddev *mddev, struct md_rdev *freshest, struc mddev->layout = le32_to_cpu(sb->layout); mddev->raid_disks = le32_to_cpu(sb->raid_disks); mddev->dev_sectors = le64_to_cpu(sb->size); + mddev->logical_block_size = le32_to_cpu(sb->logical_block_size); mddev->events = ev1; mddev->bitmap_info.offset = 0; mddev->bitmap_info.space = 0; @@ -2172,6 +2209,7 @@ static void super_1_sync(struct mddev *mddev, struct md_rdev *rdev) sb->chunksize = cpu_to_le32(mddev->chunk_sectors); sb->level = cpu_to_le32(mddev->level); sb->layout = cpu_to_le32(mddev->layout); + sb->logical_block_size = cpu_to_le32(mddev->logical_block_size); if (test_bit(FailFast, &rdev->flags)) sb->devflags |= FailFast1; else @@ -2750,6 +2788,7 @@ void md_update_sb(struct mddev *mddev, int force_change) if (!md_is_rdwr(mddev)) { if (force_change) set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); + pr_err("%s: can't update sb for read-only array %s\n", __func__, mdname(mddev)); return; } @@ -5134,7 +5173,7 @@ static void stop_sync_thread(struct mddev *mddev, bool locked) * Thread might be blocked waiting for metadata update which will now * never happen */ - md_wakeup_thread_directly(mddev->sync_thread); + md_wakeup_thread_directly(&mddev->sync_thread); if (work_pending(&mddev->sync_work)) flush_work(&mddev->sync_work); @@ -5900,6 +5939,68 @@ static struct md_sysfs_entry md_serialize_policy = __ATTR(serialize_policy, S_IRUGO | S_IWUSR, serialize_policy_show, serialize_policy_store); +static int mddev_set_logical_block_size(struct mddev *mddev, + unsigned int lbs) +{ + int err = 0; + struct queue_limits lim; + + if (queue_logical_block_size(mddev->gendisk->queue) >= lbs) { + pr_err("%s: Cannot set LBS smaller than mddev LBS %u\n", + mdname(mddev), lbs); + return -EINVAL; + } + + lim = queue_limits_start_update(mddev->gendisk->queue); + lim.logical_block_size = lbs; + pr_info("%s: logical_block_size is changed, data may be lost\n", + mdname(mddev)); + err = queue_limits_commit_update(mddev->gendisk->queue, &lim); + if (err) + return err; + + mddev->logical_block_size = lbs; + /* New lbs will be written to superblock after array is running */ + set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags); + return 0; +} + +static ssize_t +lbs_show(struct mddev *mddev, char *page) +{ + return sprintf(page, "%u\n", mddev->logical_block_size); +} + +static ssize_t +lbs_store(struct mddev *mddev, const char *buf, size_t len) +{ + unsigned int lbs; + int err = -EBUSY; + + /* Only 1.x meta supports configurable LBS */ + if (mddev->major_version == 0) + return -EINVAL; + + if (mddev->pers) + return -EBUSY; + + err = kstrtouint(buf, 10, &lbs); + if (err < 0) + return -EINVAL; + + err = mddev_lock(mddev); + if (err) + goto unlock; + + err = mddev_set_logical_block_size(mddev, lbs); + +unlock: + mddev_unlock(mddev); + return err ?: len; +} + +static struct md_sysfs_entry md_logical_block_size = +__ATTR(logical_block_size, 0644, lbs_show, lbs_store); static struct attribute *md_default_attrs[] = { &md_level.attr, @@ -5922,6 +6023,7 @@ static struct attribute *md_default_attrs[] = { &md_consistency_policy.attr, &md_fail_last_dev.attr, &md_serialize_policy.attr, + &md_logical_block_size.attr, NULL, }; @@ -6052,6 +6154,17 @@ int mddev_stack_rdev_limits(struct mddev *mddev, struct queue_limits *lim, return -EINVAL; } + /* + * Before RAID adding folio support, the logical_block_size + * should be smaller than the page size. + */ + if (lim->logical_block_size > PAGE_SIZE) { + pr_err("%s: logical_block_size must not larger than PAGE_SIZE\n", + mdname(mddev)); + return -EINVAL; + } + mddev->logical_block_size = lim->logical_block_size; + return 0; } EXPORT_SYMBOL_GPL(mddev_stack_rdev_limits); @@ -6064,6 +6177,13 @@ int mddev_stack_new_rdev(struct mddev *mddev, struct md_rdev *rdev) if (mddev_is_dm(mddev)) return 0; + if (queue_logical_block_size(rdev->bdev->bd_disk->queue) > + queue_logical_block_size(mddev->gendisk->queue)) { + pr_err("%s: incompatible logical_block_size, can not add\n", + mdname(mddev)); + return -EINVAL; + } + lim = queue_limits_start_update(mddev->gendisk->queue); queue_limits_stack_bdev(&lim, rdev->bdev, rdev->data_offset, mddev->gendisk->disk_name); @@ -6384,29 +6504,9 @@ int md_run(struct mddev *mddev) nowait = nowait && bdev_nowait(rdev->bdev); } - if (!bioset_initialized(&mddev->bio_set)) { - err = bioset_init(&mddev->bio_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); - if (err) - return err; - } - if (!bioset_initialized(&mddev->sync_set)) { - err = bioset_init(&mddev->sync_set, BIO_POOL_SIZE, 0, BIOSET_NEED_BVECS); - if (err) - goto exit_bio_set; - } - - if (!bioset_initialized(&mddev->io_clone_set)) { - err = bioset_init(&mddev->io_clone_set, BIO_POOL_SIZE, - offsetof(struct md_io_clone, bio_clone), 0); - if (err) - goto exit_sync_set; - } - pers = get_pers(mddev->level, mddev->clevel); - if (!pers) { - err = -EINVAL; - goto abort; - } + if (!pers) + return -EINVAL; if (mddev->level != pers->head.id) { mddev->level = pers->head.id; mddev->new_level = pers->head.id; @@ -6417,8 +6517,7 @@ int md_run(struct mddev *mddev) pers->start_reshape == NULL) { /* This personality cannot handle reshaping... */ put_pers(pers); - err = -EINVAL; - goto abort; + return -EINVAL; } if (pers->sync_request) { @@ -6545,12 +6644,6 @@ bitmap_abort: mddev->private = NULL; put_pers(pers); md_bitmap_destroy(mddev); -abort: - bioset_exit(&mddev->io_clone_set); -exit_sync_set: - bioset_exit(&mddev->sync_set); -exit_bio_set: - bioset_exit(&mddev->bio_set); return err; } EXPORT_SYMBOL_GPL(md_run); @@ -6683,6 +6776,7 @@ static void md_clean(struct mddev *mddev) mddev->chunk_sectors = 0; mddev->ctime = mddev->utime = 0; mddev->layout = 0; + mddev->logical_block_size = 0; mddev->max_disks = 0; mddev->events = 0; mddev->can_decrease_events = 0; @@ -6775,10 +6869,6 @@ static void __md_stop(struct mddev *mddev) mddev->private = NULL; put_pers(pers); clear_bit(MD_RECOVERY_FROZEN, &mddev->recovery); - - bioset_exit(&mddev->bio_set); - bioset_exit(&mddev->sync_set); - bioset_exit(&mddev->io_clone_set); } void md_stop(struct mddev *mddev) @@ -6869,6 +6959,10 @@ static int do_md_stop(struct mddev *mddev, int mode) if (!md_is_rdwr(mddev)) set_disk_ro(disk, 0); + if (mode == 2 && mddev->pers->sync_request && + mddev->to_remove == NULL) + mddev->to_remove = &md_redundancy_group; + __md_stop_writes(mddev); __md_stop(mddev); @@ -8373,22 +8467,21 @@ static int md_thread(void *arg) return 0; } -static void md_wakeup_thread_directly(struct md_thread __rcu *thread) +static void md_wakeup_thread_directly(struct md_thread __rcu **thread) { struct md_thread *t; rcu_read_lock(); - t = rcu_dereference(thread); + t = rcu_dereference(*thread); if (t) wake_up_process(t->tsk); rcu_read_unlock(); } -void md_wakeup_thread(struct md_thread __rcu *thread) +void __md_wakeup_thread(struct md_thread __rcu *thread) { struct md_thread *t; - rcu_read_lock(); t = rcu_dereference(thread); if (t) { pr_debug("md: waking up MD thread %s.\n", t->tsk->comm); @@ -8396,9 +8489,8 @@ void md_wakeup_thread(struct md_thread __rcu *thread) if (wq_has_sleeper(&t->wqueue)) wake_up(&t->wqueue); } - rcu_read_unlock(); } -EXPORT_SYMBOL(md_wakeup_thread); +EXPORT_SYMBOL(__md_wakeup_thread); struct md_thread *md_register_thread(void (*run) (struct md_thread *), struct mddev *mddev, const char *name) @@ -9978,6 +10070,52 @@ static void unregister_sync_thread(struct mddev *mddev) md_reap_sync_thread(mddev); } +static bool md_should_do_recovery(struct mddev *mddev) +{ + /* + * As long as one of the following flags is set, + * recovery needs to do or cleanup. + */ + if (test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) || + test_bit(MD_RECOVERY_DONE, &mddev->recovery)) + return true; + + /* + * If no flags are set and it is in read-only status, + * there is nothing to do. + */ + if (!md_is_rdwr(mddev)) + return false; + + /* + * MD_SB_CHANGE_PENDING indicates that the array is switching from clean to + * active, and no action is needed for now. + * All other MD_SB_* flags require to update the superblock. + */ + if (mddev->sb_flags & ~ (1<<MD_SB_CHANGE_PENDING)) + return true; + + /* + * If the array is not using external metadata and there has been no data + * written for some time, then the array's status needs to be set to + * in_sync. + */ + if (mddev->external == 0 && mddev->safemode == 1) + return true; + + /* + * When the system is about to restart or the process receives an signal, + * the array needs to be synchronized as soon as possible. + * Once the data synchronization is completed, need to change the array + * status to in_sync. + */ + if (mddev->safemode == 2 && !mddev->in_sync && + mddev->resync_offset == MaxSector) + return true; + + return false; +} + /* * This routine is regularly called by all per-raid-array threads to * deal with generic issues like resync and super-block update. @@ -10014,18 +10152,7 @@ void md_check_recovery(struct mddev *mddev) flush_signals(current); } - if (!md_is_rdwr(mddev) && - !test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) && - !test_bit(MD_RECOVERY_DONE, &mddev->recovery)) - return; - if ( ! ( - (mddev->sb_flags & ~ (1<<MD_SB_CHANGE_PENDING)) || - test_bit(MD_RECOVERY_NEEDED, &mddev->recovery) || - test_bit(MD_RECOVERY_DONE, &mddev->recovery) || - (mddev->external == 0 && mddev->safemode == 1) || - (mddev->safemode == 2 - && !mddev->in_sync && mddev->resync_offset == MaxSector) - )) + if (!md_should_do_recovery(mddev)) return; if (mddev_trylock(mddev)) { @@ -10281,7 +10408,6 @@ static int md_notify_reboot(struct notifier_block *this, unsigned long code, void *x) { struct mddev *mddev; - int need_delay = 0; spin_lock(&all_mddevs_lock); list_for_each_entry(mddev, &all_mddevs, all_mddevs) { @@ -10295,21 +10421,11 @@ static int md_notify_reboot(struct notifier_block *this, mddev->safemode = 2; mddev_unlock(mddev); } - need_delay = 1; spin_lock(&all_mddevs_lock); mddev_put_locked(mddev); } spin_unlock(&all_mddevs_lock); - /* - * certain more exotic SCSI devices are known to be - * volatile wrt too early system reboots. While the - * right place to handle this issue is the given - * driver, we do want to have a safe RAID driver ... - */ - if (need_delay) - msleep(1000); - return NOTIFY_DONE; } @@ -10697,6 +10813,7 @@ module_param(start_dirty_degraded, int, S_IRUGO|S_IWUSR); module_param_call(new_array, add_named_array, NULL, NULL, S_IWUSR); module_param(create_on_open, bool, S_IRUSR|S_IWUSR); module_param(legacy_async_del_gendisk, bool, 0600); +module_param(check_new_feature, bool, 0600); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("MD RAID framework"); diff --git a/drivers/md/md.h b/drivers/md/md.h index 1979c2d4fe89..6985f2829bbd 100644 --- a/drivers/md/md.h +++ b/drivers/md/md.h @@ -354,6 +354,7 @@ enum mddev_flags { MD_HAS_MULTIPLE_PPLS, MD_NOT_READY, MD_BROKEN, + MD_DO_DELETE, MD_DELETED, }; @@ -432,6 +433,7 @@ struct mddev { sector_t array_sectors; /* exported array size */ int external_size; /* size managed * externally */ + unsigned int logical_block_size; __u64 events; /* If the last 'event' was simply a clean->dirty transition, and * we didn't write it to the spares, then it is safe and simple @@ -882,6 +884,12 @@ struct md_io_clone { #define THREAD_WAKEUP 0 +#define md_wakeup_thread(thread) do { \ + rcu_read_lock(); \ + __md_wakeup_thread(thread); \ + rcu_read_unlock(); \ +} while (0) + static inline void safe_put_page(struct page *p) { if (p) put_page(p); @@ -895,7 +903,7 @@ extern struct md_thread *md_register_thread( struct mddev *mddev, const char *name); extern void md_unregister_thread(struct mddev *mddev, struct md_thread __rcu **threadp); -extern void md_wakeup_thread(struct md_thread __rcu *thread); +extern void __md_wakeup_thread(struct md_thread __rcu *thread); extern void md_check_recovery(struct mddev *mddev); extern void md_reap_sync_thread(struct mddev *mddev); extern enum sync_action md_sync_action(struct mddev *mddev); diff --git a/drivers/md/raid0.c b/drivers/md/raid0.c index e443e478645a..985c377356eb 100644 --- a/drivers/md/raid0.c +++ b/drivers/md/raid0.c @@ -68,7 +68,10 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf) struct strip_zone *zone; int cnt; struct r0conf *conf = kzalloc(sizeof(*conf), GFP_KERNEL); - unsigned blksize = 512; + unsigned int blksize = 512; + + if (!mddev_is_dm(mddev)) + blksize = queue_logical_block_size(mddev->gendisk->queue); *private_conf = ERR_PTR(-ENOMEM); if (!conf) @@ -84,7 +87,8 @@ static int create_strip_zones(struct mddev *mddev, struct r0conf **private_conf) sector_div(sectors, mddev->chunk_sectors); rdev1->sectors = sectors * mddev->chunk_sectors; - blksize = max(blksize, queue_logical_block_size( + if (mddev_is_dm(mddev)) + blksize = max(blksize, queue_logical_block_size( rdev1->bdev->bd_disk->queue)); rdev_for_each(rdev2, mddev) { @@ -383,6 +387,7 @@ static int raid0_set_limits(struct mddev *mddev) lim.max_hw_sectors = mddev->chunk_sectors; lim.max_write_zeroes_sectors = mddev->chunk_sectors; lim.max_hw_wzeroes_unmap_sectors = mddev->chunk_sectors; + lim.logical_block_size = mddev->logical_block_size; lim.io_min = mddev->chunk_sectors << 9; lim.io_opt = lim.io_min * mddev->raid_disks; lim.chunk_sectors = mddev->chunk_sectors; @@ -405,6 +410,12 @@ static int raid0_run(struct mddev *mddev) if (md_check_no_bitmap(mddev)) return -EINVAL; + if (!mddev_is_dm(mddev)) { + ret = raid0_set_limits(mddev); + if (ret) + return ret; + } + /* if private is not null, we are here after takeover */ if (mddev->private == NULL) { ret = create_strip_zones(mddev, &conf); @@ -413,11 +424,6 @@ static int raid0_run(struct mddev *mddev) mddev->private = conf; } conf = mddev->private; - if (!mddev_is_dm(mddev)) { - ret = raid0_set_limits(mddev); - if (ret) - return ret; - } /* calculate array device size */ md_set_array_sectors(mddev, raid0_size(mddev, 0, 0)); diff --git a/drivers/md/raid1.c b/drivers/md/raid1.c index 592a40233004..57d50465eed1 100644 --- a/drivers/md/raid1.c +++ b/drivers/md/raid1.c @@ -3213,6 +3213,7 @@ static int raid1_set_limits(struct mddev *mddev) md_init_stacking_limits(&lim); lim.max_write_zeroes_sectors = 0; lim.max_hw_wzeroes_unmap_sectors = 0; + lim.logical_block_size = mddev->logical_block_size; lim.features |= BLK_FEAT_ATOMIC_WRITES; err = mddev_stack_rdev_limits(mddev, &lim, MDDEV_STACK_INTEGRITY); if (err) diff --git a/drivers/md/raid10.c b/drivers/md/raid10.c index 14dcd5142eb4..84be4cc7e873 100644 --- a/drivers/md/raid10.c +++ b/drivers/md/raid10.c @@ -4000,6 +4000,7 @@ static int raid10_set_queue_limits(struct mddev *mddev) md_init_stacking_limits(&lim); lim.max_write_zeroes_sectors = 0; lim.max_hw_wzeroes_unmap_sectors = 0; + lim.logical_block_size = mddev->logical_block_size; lim.io_min = mddev->chunk_sectors << 9; lim.chunk_sectors = mddev->chunk_sectors; lim.io_opt = lim.io_min * raid10_nr_stripes(conf); diff --git a/drivers/md/raid5-cache.c b/drivers/md/raid5-cache.c index ba768ca7f422..e29e69335c69 100644 --- a/drivers/md/raid5-cache.c +++ b/drivers/md/raid5-cache.c @@ -3104,7 +3104,7 @@ int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev) goto out_mempool; spin_lock_init(&log->tree_lock); - INIT_RADIX_TREE(&log->big_stripe_tree, GFP_NOWAIT | __GFP_NOWARN); + INIT_RADIX_TREE(&log->big_stripe_tree, GFP_NOWAIT); thread = md_register_thread(r5l_reclaim_thread, log->rdev->mddev, "reclaim"); diff --git a/drivers/md/raid5.c b/drivers/md/raid5.c index 24b32a0c95b4..e57ce3295292 100644 --- a/drivers/md/raid5.c +++ b/drivers/md/raid5.c @@ -4956,7 +4956,8 @@ static void handle_stripe(struct stripe_head *sh) goto finish; if (s.handle_bad_blocks || - test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) { + (md_is_rdwr(conf->mddev) && + test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags))) { set_bit(STRIPE_HANDLE, &sh->state); goto finish; } @@ -6768,7 +6769,8 @@ static void raid5d(struct md_thread *thread) int batch_size, released; unsigned int offset; - if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) + if (md_is_rdwr(mddev) && + test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) break; released = release_stripe_list(conf, conf->temp_inactive_list); @@ -7745,6 +7747,7 @@ static int raid5_set_limits(struct mddev *mddev) stripe = roundup_pow_of_two(data_disks * (mddev->chunk_sectors << 9)); md_init_stacking_limits(&lim); + lim.logical_block_size = mddev->logical_block_size; lim.io_min = mddev->chunk_sectors << 9; lim.io_opt = lim.io_min * (conf->raid_disks - conf->max_degraded); lim.features |= BLK_FEAT_RAID_PARTIAL_STRIPES_EXPENSIVE; diff --git a/drivers/media/mc/mc-request.c b/drivers/media/mc/mc-request.c index f66f728b1b43..2ac9ac0a740b 100644 --- a/drivers/media/mc/mc-request.c +++ b/drivers/media/mc/mc-request.c @@ -282,8 +282,6 @@ EXPORT_SYMBOL_GPL(media_request_get_by_fd); int media_request_alloc(struct media_device *mdev, int *alloc_fd) { struct media_request *req; - struct file *filp; - int fd; int ret; /* Either both are NULL or both are non-NULL */ @@ -297,19 +295,6 @@ int media_request_alloc(struct media_device *mdev, int *alloc_fd) if (!req) return -ENOMEM; - fd = get_unused_fd_flags(O_CLOEXEC); - if (fd < 0) { - ret = fd; - goto err_free_req; - } - - filp = anon_inode_getfile("request", &request_fops, NULL, O_CLOEXEC); - if (IS_ERR(filp)) { - ret = PTR_ERR(filp); - goto err_put_fd; - } - - filp->private_data = req; req->mdev = mdev; req->state = MEDIA_REQUEST_STATE_IDLE; req->num_incomplete_objects = 0; @@ -320,19 +305,24 @@ int media_request_alloc(struct media_device *mdev, int *alloc_fd) req->updating_count = 0; req->access_count = 0; - *alloc_fd = fd; + FD_PREPARE(fdf, O_CLOEXEC, + anon_inode_getfile("request", &request_fops, NULL, + O_CLOEXEC)); + if (fdf.err) { + ret = fdf.err; + goto err_free_req; + } + + fd_prepare_file(fdf)->private_data = req; + + *alloc_fd = fd_publish(fdf); snprintf(req->debug_str, sizeof(req->debug_str), "%u:%d", - atomic_inc_return(&mdev->request_id), fd); + atomic_inc_return(&mdev->request_id), *alloc_fd); dev_dbg(mdev->dev, "request: allocated %s\n", req->debug_str); - fd_install(fd, filp); - return 0; -err_put_fd: - put_unused_fd(fd); - err_free_req: if (mdev->ops->req_free) mdev->ops->req_free(req); diff --git a/drivers/media/test-drivers/vivid/vivid-vid-cap.c b/drivers/media/test-drivers/vivid/vivid-vid-cap.c index 8b3162e82032..b95f06a9b5ae 100644 --- a/drivers/media/test-drivers/vivid/vivid-vid-cap.c +++ b/drivers/media/test-drivers/vivid/vivid-vid-cap.c @@ -302,8 +302,10 @@ void vivid_update_quality(struct vivid_dev *dev) */ freq_modulus = (dev->tv_freq - 676 /* (43.25-1) * 16 */) % (6 * 16); if (freq_modulus > 2 * 16) { + struct rnd_state prng; + prandom_seed_state(&prng, dev->tv_freq ^ 0x55); tpg_s_quality(&dev->tpg, TPG_QUAL_NOISE, - next_pseudo_random32(dev->tv_freq ^ 0x55) & 0x3f); + prandom_u32_state(&prng) & 0x3f); return; } if (freq_modulus < 12 /*0.75 * 16*/ || freq_modulus > 20 /*1.25 * 16*/) diff --git a/drivers/misc/mei/pci-me.c b/drivers/misc/mei/pci-me.c index b017ff29dbd1..73cad914be9f 100644 --- a/drivers/misc/mei/pci-me.c +++ b/drivers/misc/mei/pci-me.c @@ -223,6 +223,10 @@ static int mei_me_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw->mem_addr = pcim_iomap_table(pdev)[0]; hw->read_fws = mei_me_read_fws; + err = mei_register(dev, &pdev->dev); + if (err) + goto end; + pci_enable_msi(pdev); hw->irq = pdev->irq; @@ -237,13 +241,9 @@ static int mei_me_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) { dev_err(&pdev->dev, "request_threaded_irq failure. irq = %d\n", pdev->irq); - goto end; + goto deregister; } - err = mei_register(dev, &pdev->dev); - if (err) - goto release_irq; - if (mei_start(dev)) { dev_err(&pdev->dev, "init hw failure.\n"); err = -ENODEV; @@ -283,11 +283,10 @@ static int mei_me_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; deregister: - mei_deregister(dev); -release_irq: mei_cancel_work(dev); mei_disable_interrupts(dev); free_irq(pdev->irq, dev); + mei_deregister(dev); end: dev_err(&pdev->dev, "initialization failed.\n"); return err; diff --git a/drivers/misc/mei/pci-txe.c b/drivers/misc/mei/pci-txe.c index 06b55a891c6b..98d1bc2c7f4b 100644 --- a/drivers/misc/mei/pci-txe.c +++ b/drivers/misc/mei/pci-txe.c @@ -87,6 +87,10 @@ static int mei_txe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) hw = to_txe_hw(dev); hw->mem_addr = pcim_iomap_table(pdev); + err = mei_register(dev, &pdev->dev); + if (err) + goto end; + pci_enable_msi(pdev); /* clear spurious interrupts */ @@ -106,13 +110,9 @@ static int mei_txe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) if (err) { dev_err(&pdev->dev, "mei: request_threaded_irq failure. irq = %d\n", pdev->irq); - goto end; + goto deregister; } - err = mei_register(dev, &pdev->dev); - if (err) - goto release_irq; - if (mei_start(dev)) { dev_err(&pdev->dev, "init hw failure.\n"); err = -ENODEV; @@ -145,11 +145,10 @@ static int mei_txe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) return 0; deregister: - mei_deregister(dev); -release_irq: mei_cancel_work(dev); mei_disable_interrupts(dev); free_irq(pdev->irq, dev); + mei_deregister(dev); end: dev_err(&pdev->dev, "initialization failed.\n"); return err; diff --git a/drivers/misc/mei/platform-vsc.c b/drivers/misc/mei/platform-vsc.c index 288e7b72e942..9787b9cee71c 100644 --- a/drivers/misc/mei/platform-vsc.c +++ b/drivers/misc/mei/platform-vsc.c @@ -362,28 +362,27 @@ static int mei_vsc_probe(struct platform_device *pdev) ret = mei_register(mei_dev, dev); if (ret) - goto err_dereg; + goto err; ret = mei_start(mei_dev); if (ret) { dev_err_probe(dev, ret, "init hw failed\n"); - goto err_cancel; + goto err; } pm_runtime_enable(mei_dev->parent); return 0; -err_dereg: - mei_deregister(mei_dev); - -err_cancel: +err: mei_cancel_work(mei_dev); vsc_tp_register_event_cb(tp, NULL, NULL); mei_disable_interrupts(mei_dev); + mei_deregister(mei_dev); + return ret; } diff --git a/drivers/misc/ntsync.c b/drivers/misc/ntsync.c index 999026a1ae04..9087f045e362 100644 --- a/drivers/misc/ntsync.c +++ b/drivers/misc/ntsync.c @@ -721,21 +721,12 @@ static struct ntsync_obj *ntsync_alloc_obj(struct ntsync_device *dev, static int ntsync_obj_get_fd(struct ntsync_obj *obj) { - struct file *file; - int fd; - - fd = get_unused_fd_flags(O_CLOEXEC); - if (fd < 0) - return fd; - file = anon_inode_getfile("ntsync", &ntsync_obj_fops, obj, O_RDWR); - if (IS_ERR(file)) { - put_unused_fd(fd); - return PTR_ERR(file); - } - obj->file = file; - fd_install(fd, file); - - return fd; + FD_PREPARE(fdf, O_CLOEXEC, + anon_inode_getfile("ntsync", &ntsync_obj_fops, obj, O_RDWR)); + if (fdf.err) + return fdf.err; + obj->file = fd_prepare_file(fdf); + return fd_publish(fdf); } static int ntsync_create_sem(struct ntsync_device *dev, void __user *argp) diff --git a/drivers/mmc/core/mmc_test.c b/drivers/mmc/core/mmc_test.c index 67d4a301895c..d59b5e5a7b45 100644 --- a/drivers/mmc/core/mmc_test.c +++ b/drivers/mmc/core/mmc_test.c @@ -586,14 +586,11 @@ static void mmc_test_print_avg_rate(struct mmc_test_card *test, uint64_t bytes, rate = mmc_test_rate(tot, &ts); iops = mmc_test_rate(count * 100, &ts); /* I/O ops per sec x 100 */ - pr_info("%s: Transfer of %u x %u sectors (%u x %u%s KiB) took " - "%llu.%09u seconds (%u kB/s, %u KiB/s, " - "%u.%02u IOPS, sg_len %d)\n", - mmc_hostname(test->card->host), count, sectors, count, - sectors >> 1, (sectors & 1 ? ".5" : ""), - (u64)ts.tv_sec, (u32)ts.tv_nsec, - rate / 1000, rate / 1024, iops / 100, iops % 100, - test->area.sg_len); + pr_info("%s: Transfer of %u x %u sectors (%u x %u%s KiB) took %ptSp seconds (%u kB/s, %u KiB/s, %u.%02u IOPS, sg_len %d)\n", + mmc_hostname(test->card->host), count, sectors, count, + sectors >> 1, (sectors & 1 ? ".5" : ""), &ts, + rate / 1000, rate / 1024, iops / 100, iops % 100, + test->area.sg_len); mmc_test_save_transfer_result(test, count, sectors, ts, rate, iops); } @@ -3074,10 +3071,9 @@ static int mtf_test_show(struct seq_file *sf, void *data) seq_printf(sf, "Test %d: %d\n", gr->testcase + 1, gr->result); list_for_each_entry(tr, &gr->tr_lst, link) { - seq_printf(sf, "%u %d %llu.%09u %u %u.%02u\n", - tr->count, tr->sectors, - (u64)tr->ts.tv_sec, (u32)tr->ts.tv_nsec, - tr->rate, tr->iops / 100, tr->iops % 100); + seq_printf(sf, "%u %d %ptSp %u %u.%02u\n", + tr->count, tr->sectors, &tr->ts, tr->rate, + tr->iops / 100, tr->iops % 100); } } diff --git a/drivers/mmc/host/sdhci-of-dwcmshc.c b/drivers/mmc/host/sdhci-of-dwcmshc.c index 5b61401a7f3d..4e256673a098 100644 --- a/drivers/mmc/host/sdhci-of-dwcmshc.c +++ b/drivers/mmc/host/sdhci-of-dwcmshc.c @@ -289,6 +289,19 @@ static void dwcmshc_adma_write_desc(struct sdhci_host *host, void **desc, sdhci_adma_write_desc(host, desc, addr, len, cmd); } +static void dwcmshc_reset(struct sdhci_host *host, u8 mask) +{ + sdhci_reset(host, mask); + + /* The dwcmshc does not comply with the SDHCI specification + * regarding the "Software Reset for CMD line should clear 'Command + * Complete' in the Normal Interrupt Status Register." Clear the bit + * here to compensate for this quirk. + */ + if (mask & SDHCI_RESET_CMD) + sdhci_writel(host, SDHCI_INT_RESPONSE, SDHCI_INT_STATUS); +} + static unsigned int dwcmshc_get_max_clock(struct sdhci_host *host) { struct sdhci_pltfm_host *pltfm_host = sdhci_priv(host); @@ -832,15 +845,7 @@ static void th1520_sdhci_reset(struct sdhci_host *host, u8 mask) struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host); u16 ctrl_2; - sdhci_reset(host, mask); - - /* The T-Head 1520 SoC does not comply with the SDHCI specification - * regarding the "Software Reset for CMD line should clear 'Command - * Complete' in the Normal Interrupt Status Register." Clear the bit - * here to compensate for this quirk. - */ - if (mask & SDHCI_RESET_CMD) - sdhci_writel(host, SDHCI_INT_RESPONSE, SDHCI_INT_STATUS); + dwcmshc_reset(host, mask); if (priv->flags & FLAG_IO_FIXED_1V8) { ctrl_2 = sdhci_readw(host, SDHCI_HOST_CONTROL2); @@ -886,7 +891,7 @@ static void cv18xx_sdhci_reset(struct sdhci_host *host, u8 mask) struct dwcmshc_priv *priv = sdhci_pltfm_priv(pltfm_host); u32 val, emmc_caps = MMC_CAP2_NO_SD | MMC_CAP2_NO_SDIO; - sdhci_reset(host, mask); + dwcmshc_reset(host, mask); if ((host->mmc->caps2 & emmc_caps) == emmc_caps) { val = sdhci_readl(host, priv->vendor_specific_area1 + CV18XX_SDHCI_MSHC_CTRL); @@ -958,7 +963,7 @@ static void cv18xx_sdhci_post_tuning(struct sdhci_host *host) val |= SDHCI_INT_DATA_AVAIL; sdhci_writel(host, val, SDHCI_INT_STATUS); - sdhci_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); + dwcmshc_reset(host, SDHCI_RESET_CMD | SDHCI_RESET_DATA); } static int cv18xx_sdhci_execute_tuning(struct sdhci_host *host, u32 opcode) @@ -1100,7 +1105,7 @@ static const struct sdhci_ops sdhci_dwcmshc_ops = { .set_bus_width = sdhci_set_bus_width, .set_uhs_signaling = dwcmshc_set_uhs_signaling, .get_max_clock = dwcmshc_get_max_clock, - .reset = sdhci_reset, + .reset = dwcmshc_reset, .adma_write_desc = dwcmshc_adma_write_desc, .irq = dwcmshc_cqe_irq_handler, }; diff --git a/drivers/most/most_usb.c b/drivers/most/most_usb.c index 10064d7b7249..41ee169f80c5 100644 --- a/drivers/most/most_usb.c +++ b/drivers/most/most_usb.c @@ -1058,7 +1058,7 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id) ret = most_register_interface(&mdev->iface); if (ret) - goto err_free_busy_urbs; + return ret; mutex_lock(&mdev->io_mutex); if (le16_to_cpu(usb_dev->descriptor.idProduct) == USB_DEV_ID_OS81118 || @@ -1068,8 +1068,7 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id) if (!mdev->dci) { mutex_unlock(&mdev->io_mutex); most_deregister_interface(&mdev->iface); - ret = -ENOMEM; - goto err_free_busy_urbs; + return -ENOMEM; } mdev->dci->dev.init_name = "dci"; @@ -1078,18 +1077,15 @@ hdm_probe(struct usb_interface *interface, const struct usb_device_id *id) mdev->dci->dev.release = release_dci; if (device_register(&mdev->dci->dev)) { mutex_unlock(&mdev->io_mutex); + put_device(&mdev->dci->dev); most_deregister_interface(&mdev->iface); - ret = -ENOMEM; - goto err_free_dci; + return -ENOMEM; } mdev->dci->usb_device = mdev->usb_device; } mutex_unlock(&mdev->io_mutex); return 0; -err_free_dci: - put_device(&mdev->dci->dev); -err_free_busy_urbs: - kfree(mdev->busy_urbs); + err_free_ep_address: kfree(mdev->ep_address); err_free_cap: diff --git a/drivers/net/dsa/sja1105/sja1105_tas.c b/drivers/net/dsa/sja1105/sja1105_tas.c index d7818710bc02..d5949d2c3e71 100644 --- a/drivers/net/dsa/sja1105/sja1105_tas.c +++ b/drivers/net/dsa/sja1105/sja1105_tas.c @@ -775,9 +775,8 @@ static void sja1105_tas_state_machine(struct work_struct *work) base_time_ts = ns_to_timespec64(base_time); now_ts = ns_to_timespec64(now); - dev_dbg(ds->dev, "OPER base time %lld.%09ld (now %lld.%09ld)\n", - base_time_ts.tv_sec, base_time_ts.tv_nsec, - now_ts.tv_sec, now_ts.tv_nsec); + dev_dbg(ds->dev, "OPER base time %ptSp (now %ptSp)\n", + &base_time_ts, &now_ts); break; @@ -798,8 +797,7 @@ static void sja1105_tas_state_machine(struct work_struct *work) if (now < tas_data->oper_base_time) { /* TAS has not started yet */ diff = ns_to_timespec64(tas_data->oper_base_time - now); - dev_dbg(ds->dev, "time to start: [%lld.%09ld]", - diff.tv_sec, diff.tv_nsec); + dev_dbg(ds->dev, "time to start: [%ptSp]", &diff); break; } diff --git a/drivers/net/ethernet/intel/e1000e/ptp.c b/drivers/net/ethernet/intel/e1000e/ptp.c index ea3c3eb2ef20..ec39e35f3857 100644 --- a/drivers/net/ethernet/intel/e1000e/ptp.c +++ b/drivers/net/ethernet/intel/e1000e/ptp.c @@ -229,14 +229,11 @@ static void e1000e_systim_overflow_work(struct work_struct *work) systim_overflow_work.work); struct e1000_hw *hw = &adapter->hw; struct timespec64 ts; - u64 ns; /* Update the timecounter */ - ns = timecounter_read(&adapter->tc); + ts = ns_to_timespec64(timecounter_read(&adapter->tc)); - ts = ns_to_timespec64(ns); - e_dbg("SYSTIM overflow check at %lld.%09lu\n", - (long long) ts.tv_sec, ts.tv_nsec); + e_dbg("SYSTIM overflow check at %ptSp\n", &ts); schedule_delayed_work(&adapter->systim_overflow_work, E1000_SYSTIM_OVERFLOW_PERIOD); diff --git a/drivers/net/ethernet/intel/igb/igb_ptp.c b/drivers/net/ethernet/intel/igb/igb_ptp.c index a7876882aeaf..bd85d02ecadd 100644 --- a/drivers/net/ethernet/intel/igb/igb_ptp.c +++ b/drivers/net/ethernet/intel/igb/igb_ptp.c @@ -840,14 +840,11 @@ static void igb_ptp_overflow_check(struct work_struct *work) struct igb_adapter *igb = container_of(work, struct igb_adapter, ptp_overflow_work.work); struct timespec64 ts; - u64 ns; /* Update the timecounter */ - ns = timecounter_read(&igb->tc); + ts = ns_to_timespec64(timecounter_read(&igb->tc)); - ts = ns_to_timespec64(ns); - pr_debug("igb overflow check at %lld.%09lu\n", - (long long) ts.tv_sec, ts.tv_nsec); + pr_debug("igb overflow check at %ptSp\n", &ts); schedule_delayed_work(&igb->ptp_overflow_work, IGB_SYSTIM_OVERFLOW_PERIOD); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/wc.c b/drivers/net/ethernet/mellanox/mlx5/core/wc.c index 05e5fd777d4f..815a7c97d6b0 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/wc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/wc.c @@ -9,6 +9,7 @@ #if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && IS_ENABLED(CONFIG_ARM64) #include <asm/neon.h> +#include <asm/simd.h> #endif #define TEST_WC_NUM_WQES 255 @@ -264,15 +265,15 @@ static void mlx5_iowrite64_copy(struct mlx5_wc_sq *sq, __be32 mmio_wqe[16], { #if IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && IS_ENABLED(CONFIG_ARM64) if (cpu_has_neon()) { - kernel_neon_begin(); - asm volatile - (".arch_extension simd\n\t" - "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%0]\n\t" - "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%1]" - : - : "r"(mmio_wqe), "r"(sq->bfreg.map + offset) - : "memory", "v0", "v1", "v2", "v3"); - kernel_neon_end(); + scoped_ksimd() { + asm volatile( + ".arch_extension simd\n\t" + "ld1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%0]\n\t" + "st1 {v0.16b, v1.16b, v2.16b, v3.16b}, [%1]" + : + : "r"(mmio_wqe), "r"(sq->bfreg.map + offset) + : "memory", "v0", "v1", "v2", "v3"); + } return; } #endif diff --git a/drivers/net/wireguard/cookie.c b/drivers/net/wireguard/cookie.c index 94d0a7206084..08731b3fa32b 100644 --- a/drivers/net/wireguard/cookie.c +++ b/drivers/net/wireguard/cookie.c @@ -33,7 +33,7 @@ static void precompute_key(u8 key[NOISE_SYMMETRIC_KEY_LEN], const u8 pubkey[NOISE_PUBLIC_KEY_LEN], const u8 label[COOKIE_KEY_LABEL_LEN]) { - struct blake2s_state blake; + struct blake2s_ctx blake; blake2s_init(&blake, NOISE_SYMMETRIC_KEY_LEN); blake2s_update(&blake, label, COOKIE_KEY_LABEL_LEN); @@ -77,7 +77,7 @@ static void compute_mac1(u8 mac1[COOKIE_LEN], const void *message, size_t len, { len = len - sizeof(struct message_macs) + offsetof(struct message_macs, mac1); - blake2s(mac1, message, key, COOKIE_LEN, len, NOISE_SYMMETRIC_KEY_LEN); + blake2s(key, NOISE_SYMMETRIC_KEY_LEN, message, len, mac1, COOKIE_LEN); } static void compute_mac2(u8 mac2[COOKIE_LEN], const void *message, size_t len, @@ -85,13 +85,13 @@ static void compute_mac2(u8 mac2[COOKIE_LEN], const void *message, size_t len, { len = len - sizeof(struct message_macs) + offsetof(struct message_macs, mac2); - blake2s(mac2, message, cookie, COOKIE_LEN, len, COOKIE_LEN); + blake2s(cookie, COOKIE_LEN, message, len, mac2, COOKIE_LEN); } static void make_cookie(u8 cookie[COOKIE_LEN], struct sk_buff *skb, struct cookie_checker *checker) { - struct blake2s_state state; + struct blake2s_ctx blake; if (wg_birthdate_has_expired(checker->secret_birthdate, COOKIE_SECRET_MAX_AGE)) { @@ -103,15 +103,15 @@ static void make_cookie(u8 cookie[COOKIE_LEN], struct sk_buff *skb, down_read(&checker->secret_lock); - blake2s_init_key(&state, COOKIE_LEN, checker->secret, NOISE_HASH_LEN); + blake2s_init_key(&blake, COOKIE_LEN, checker->secret, NOISE_HASH_LEN); if (skb->protocol == htons(ETH_P_IP)) - blake2s_update(&state, (u8 *)&ip_hdr(skb)->saddr, + blake2s_update(&blake, (u8 *)&ip_hdr(skb)->saddr, sizeof(struct in_addr)); else if (skb->protocol == htons(ETH_P_IPV6)) - blake2s_update(&state, (u8 *)&ipv6_hdr(skb)->saddr, + blake2s_update(&blake, (u8 *)&ipv6_hdr(skb)->saddr, sizeof(struct in6_addr)); - blake2s_update(&state, (u8 *)&udp_hdr(skb)->source, sizeof(__be16)); - blake2s_final(&state, cookie); + blake2s_update(&blake, (u8 *)&udp_hdr(skb)->source, sizeof(__be16)); + blake2s_final(&blake, cookie); up_read(&checker->secret_lock); } diff --git a/drivers/net/wireguard/noise.c b/drivers/net/wireguard/noise.c index 7eb9a23a3d4d..1fe8468f0bef 100644 --- a/drivers/net/wireguard/noise.c +++ b/drivers/net/wireguard/noise.c @@ -33,10 +33,10 @@ static atomic64_t keypair_counter = ATOMIC64_INIT(0); void __init wg_noise_init(void) { - struct blake2s_state blake; + struct blake2s_ctx blake; - blake2s(handshake_init_chaining_key, handshake_name, NULL, - NOISE_HASH_LEN, sizeof(handshake_name), 0); + blake2s(NULL, 0, handshake_name, sizeof(handshake_name), + handshake_init_chaining_key, NOISE_HASH_LEN); blake2s_init(&blake, NOISE_HASH_LEN); blake2s_update(&blake, handshake_init_chaining_key, NOISE_HASH_LEN); blake2s_update(&blake, identifier_name, sizeof(identifier_name)); @@ -304,33 +304,33 @@ void wg_noise_set_static_identity_private_key( static void hmac(u8 *out, const u8 *in, const u8 *key, const size_t inlen, const size_t keylen) { - struct blake2s_state state; + struct blake2s_ctx blake; u8 x_key[BLAKE2S_BLOCK_SIZE] __aligned(__alignof__(u32)) = { 0 }; u8 i_hash[BLAKE2S_HASH_SIZE] __aligned(__alignof__(u32)); int i; if (keylen > BLAKE2S_BLOCK_SIZE) { - blake2s_init(&state, BLAKE2S_HASH_SIZE); - blake2s_update(&state, key, keylen); - blake2s_final(&state, x_key); + blake2s_init(&blake, BLAKE2S_HASH_SIZE); + blake2s_update(&blake, key, keylen); + blake2s_final(&blake, x_key); } else memcpy(x_key, key, keylen); for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) x_key[i] ^= 0x36; - blake2s_init(&state, BLAKE2S_HASH_SIZE); - blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); - blake2s_update(&state, in, inlen); - blake2s_final(&state, i_hash); + blake2s_init(&blake, BLAKE2S_HASH_SIZE); + blake2s_update(&blake, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&blake, in, inlen); + blake2s_final(&blake, i_hash); for (i = 0; i < BLAKE2S_BLOCK_SIZE; ++i) x_key[i] ^= 0x5c ^ 0x36; - blake2s_init(&state, BLAKE2S_HASH_SIZE); - blake2s_update(&state, x_key, BLAKE2S_BLOCK_SIZE); - blake2s_update(&state, i_hash, BLAKE2S_HASH_SIZE); - blake2s_final(&state, i_hash); + blake2s_init(&blake, BLAKE2S_HASH_SIZE); + blake2s_update(&blake, x_key, BLAKE2S_BLOCK_SIZE); + blake2s_update(&blake, i_hash, BLAKE2S_HASH_SIZE); + blake2s_final(&blake, i_hash); memcpy(out, i_hash, BLAKE2S_HASH_SIZE); memzero_explicit(x_key, BLAKE2S_BLOCK_SIZE); @@ -431,7 +431,7 @@ static bool __must_check mix_precomputed_dh(u8 chaining_key[NOISE_HASH_LEN], static void mix_hash(u8 hash[NOISE_HASH_LEN], const u8 *src, size_t src_len) { - struct blake2s_state blake; + struct blake2s_ctx blake; blake2s_init(&blake, NOISE_HASH_LEN); blake2s_update(&blake, hash, NOISE_HASH_LEN); diff --git a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c index 5232f66c2d52..cc8a84018f70 100644 --- a/drivers/net/wireless/intel/iwlwifi/iwl-trans.c +++ b/drivers/net/wireless/intel/iwlwifi/iwl-trans.c @@ -129,7 +129,7 @@ static enum iwl_reset_mode iwl_trans_determine_restart_mode(struct iwl_trans *trans) { struct iwl_trans_dev_restart_data *data; - enum iwl_reset_mode at_least = 0; + enum iwl_reset_mode min_mode = 0; unsigned int index; static const enum iwl_reset_mode escalation_list_old[] = { IWL_RESET_MODE_SW_RESET, @@ -173,11 +173,11 @@ iwl_trans_determine_restart_mode(struct iwl_trans *trans) } if (trans->restart.during_reset) - at_least = IWL_RESET_MODE_REPROBE; + min_mode = IWL_RESET_MODE_REPROBE; data = iwl_trans_get_restart_data(trans->dev); if (!data) - return at_least; + return min_mode; if (!data->backoff && ktime_get_boottime_seconds() - data->last_error >= @@ -194,7 +194,7 @@ iwl_trans_determine_restart_mode(struct iwl_trans *trans) data->backoff = false; } - return max(at_least, escalation_list[index]); + return max(min_mode, escalation_list[index]); } #define IWL_TRANS_TOP_FOLLOWER_WAIT 180 /* ms */ diff --git a/drivers/nvme/host/apple.c b/drivers/nvme/host/apple.c index f35d3f71d14f..15b3d07f8ccd 100644 --- a/drivers/nvme/host/apple.c +++ b/drivers/nvme/host/apple.c @@ -1283,6 +1283,7 @@ static const struct nvme_ctrl_ops nvme_ctrl_ops = { .reg_read64 = apple_nvme_reg_read64, .free_ctrl = apple_nvme_free_ctrl, .get_address = apple_nvme_get_address, + .get_virt_boundary = nvme_get_virt_boundary, }; static void apple_nvme_async_probe(void *data, async_cookie_t cookie) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index f1f719351f3f..7bf228df6001 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2069,13 +2069,13 @@ static u32 nvme_max_drv_segments(struct nvme_ctrl *ctrl) } static void nvme_set_ctrl_limits(struct nvme_ctrl *ctrl, - struct queue_limits *lim) + struct queue_limits *lim, bool is_admin) { lim->max_hw_sectors = ctrl->max_hw_sectors; lim->max_segments = min_t(u32, USHRT_MAX, min_not_zero(nvme_max_drv_segments(ctrl), ctrl->max_segments)); lim->max_integrity_segments = ctrl->max_integrity_segments; - lim->virt_boundary_mask = NVME_CTRL_PAGE_SIZE - 1; + lim->virt_boundary_mask = ctrl->ops->get_virt_boundary(ctrl, is_admin); lim->max_segment_size = UINT_MAX; lim->dma_alignment = 3; } @@ -2177,7 +2177,7 @@ static int nvme_update_ns_info_generic(struct nvme_ns *ns, int ret; lim = queue_limits_start_update(ns->disk->queue); - nvme_set_ctrl_limits(ns->ctrl, &lim); + nvme_set_ctrl_limits(ns->ctrl, &lim, false); memflags = blk_mq_freeze_queue(ns->disk->queue); ret = queue_limits_commit_update(ns->disk->queue, &lim); @@ -2381,7 +2381,7 @@ static int nvme_update_ns_info_block(struct nvme_ns *ns, ns->head->lba_shift = id->lbaf[lbaf].ds; ns->head->nuse = le64_to_cpu(id->nuse); capacity = nvme_lba_to_sect(ns->head, le64_to_cpu(id->nsze)); - nvme_set_ctrl_limits(ns->ctrl, &lim); + nvme_set_ctrl_limits(ns->ctrl, &lim, false); nvme_configure_metadata(ns->ctrl, ns->head, id, nvm, info); nvme_set_chunk_sectors(ns, id, &lim); if (!nvme_update_disk_info(ns, id, &lim)) @@ -2599,10 +2599,9 @@ static void nvme_configure_opal(struct nvme_ctrl *ctrl, bool was_suspended) #ifdef CONFIG_BLK_DEV_ZONED static int nvme_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data) + unsigned int nr_zones, struct blk_report_zones_args *args) { - return nvme_ns_report_zones(disk->private_data, sector, nr_zones, cb, - data); + return nvme_ns_report_zones(disk->private_data, sector, nr_zones, args); } #else #define nvme_report_zones NULL @@ -3589,7 +3588,7 @@ static int nvme_init_identify(struct nvme_ctrl *ctrl) min_not_zero(ctrl->max_hw_sectors, max_hw_sectors); lim = queue_limits_start_update(ctrl->admin_q); - nvme_set_ctrl_limits(ctrl, &lim); + nvme_set_ctrl_limits(ctrl, &lim, true); ret = queue_limits_commit_update(ctrl->admin_q, &lim); if (ret) goto out_free; diff --git a/drivers/nvme/host/fabrics.h b/drivers/nvme/host/fabrics.h index 1b58ee7d0dce..caf5503d0833 100644 --- a/drivers/nvme/host/fabrics.h +++ b/drivers/nvme/host/fabrics.h @@ -217,6 +217,12 @@ static inline unsigned int nvmf_nr_io_queues(struct nvmf_ctrl_options *opts) min(opts->nr_poll_queues, num_online_cpus()); } +static inline unsigned long nvmf_get_virt_boundary(struct nvme_ctrl *ctrl, + bool is_admin) +{ + return 0; +} + int nvmf_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val); int nvmf_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val); int nvmf_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val); diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index 2c903729b0b9..873954d43b18 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -3361,6 +3361,7 @@ static const struct nvme_ctrl_ops nvme_fc_ctrl_ops = { .submit_async_event = nvme_fc_submit_async_event, .delete_ctrl = nvme_fc_delete_ctrl, .get_address = nvmf_get_address, + .get_virt_boundary = nvmf_get_virt_boundary, }; static void diff --git a/drivers/nvme/host/ioctl.c b/drivers/nvme/host/ioctl.c index c212fa952c0f..4fa8400a5627 100644 --- a/drivers/nvme/host/ioctl.c +++ b/drivers/nvme/host/ioctl.c @@ -398,14 +398,15 @@ static inline struct nvme_uring_cmd_pdu *nvme_uring_cmd_pdu( return io_uring_cmd_to_pdu(ioucmd, struct nvme_uring_cmd_pdu); } -static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd, - unsigned issue_flags) +static void nvme_uring_task_cb(struct io_tw_req tw_req, io_tw_token_t tw) { + struct io_uring_cmd *ioucmd = io_uring_cmd_from_tw(tw_req); struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd); if (pdu->bio) blk_rq_unmap_user(pdu->bio); - io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, issue_flags); + io_uring_cmd_done32(ioucmd, pdu->status, pdu->result, + IO_URING_CMD_TASK_WORK_ISSUE_FLAGS); } static enum rq_end_io_ret nvme_uring_cmd_end_io(struct request *req, diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index e35eccacee8c..174027d1cc19 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -576,7 +576,7 @@ static int nvme_ns_head_get_unique_id(struct gendisk *disk, u8 id[16], #ifdef CONFIG_BLK_DEV_ZONED static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data) + unsigned int nr_zones, struct blk_report_zones_args *args) { struct nvme_ns_head *head = disk->private_data; struct nvme_ns *ns; @@ -585,7 +585,7 @@ static int nvme_ns_head_report_zones(struct gendisk *disk, sector_t sector, srcu_idx = srcu_read_lock(&head->srcu); ns = nvme_find_path(head); if (ns) - ret = nvme_ns_report_zones(ns, sector, nr_zones, cb, data); + ret = nvme_ns_report_zones(ns, sector, nr_zones, args); srcu_read_unlock(&head->srcu, srcu_idx); return ret; } diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 102fae6a231c..9a5f28c5103c 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -558,6 +558,12 @@ static inline bool nvme_ns_has_pi(struct nvme_ns_head *head) return head->pi_type && head->ms == head->pi_size; } +static inline unsigned long nvme_get_virt_boundary(struct nvme_ctrl *ctrl, + bool is_admin) +{ + return NVME_CTRL_PAGE_SIZE - 1; +} + struct nvme_ctrl_ops { const char *name; struct module *module; @@ -578,6 +584,7 @@ struct nvme_ctrl_ops { int (*get_address)(struct nvme_ctrl *ctrl, char *buf, int size); void (*print_device_info)(struct nvme_ctrl *ctrl); bool (*supports_pci_p2pdma)(struct nvme_ctrl *ctrl); + unsigned long (*get_virt_boundary)(struct nvme_ctrl *ctrl, bool is_admin); }; /* @@ -1108,7 +1115,7 @@ struct nvme_zone_info { }; int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data); + unsigned int nr_zones, struct blk_report_zones_args *args); int nvme_query_zone_info(struct nvme_ns *ns, unsigned lbaf, struct nvme_zone_info *zi); void nvme_update_zone_info(struct nvme_ns *ns, struct queue_limits *lim, diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 72fb675a696f..e5ca8301bb8b 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -260,8 +260,20 @@ enum nvme_iod_flags { /* single segment dma mapping */ IOD_SINGLE_SEGMENT = 1U << 2, + /* Data payload contains p2p memory */ + IOD_DATA_P2P = 1U << 3, + + /* Metadata contains p2p memory */ + IOD_META_P2P = 1U << 4, + + /* Data payload contains MMIO memory */ + IOD_DATA_MMIO = 1U << 5, + + /* Metadata contains MMIO memory */ + IOD_META_MMIO = 1U << 6, + /* Metadata using non-coalesced MPTR */ - IOD_SINGLE_META_SEGMENT = 1U << 5, + IOD_SINGLE_META_SEGMENT = 1U << 7, }; struct nvme_dma_vec { @@ -613,9 +625,22 @@ static inline enum nvme_use_sgl nvme_pci_use_sgls(struct nvme_dev *dev, struct nvme_queue *nvmeq = req->mq_hctx->driver_data; if (nvmeq->qid && nvme_ctrl_sgl_supported(&dev->ctrl)) { - if (nvme_req(req)->flags & NVME_REQ_USERCMD) - return SGL_FORCED; - if (req->nr_integrity_segments > 1) + /* + * When the controller is capable of using SGL, there are + * several conditions that we force to use it: + * + * 1. A request containing page gaps within the controller's + * mask can not use the PRP format. + * + * 2. User commands use SGL because that lets the device + * validate the requested transfer lengths. + * + * 3. Multiple integrity segments must use SGL as that's the + * only way to describe such a command in NVMe. + */ + if (req_phys_gap_mask(req) & (NVME_CTRL_PAGE_SIZE - 1) || + nvme_req(req)->flags & NVME_REQ_USERCMD || + req->nr_integrity_segments > 1) return SGL_FORCED; return SGL_SUPPORTED; } @@ -685,20 +710,20 @@ static void nvme_free_descriptors(struct request *req) } } -static void nvme_free_prps(struct request *req) +static void nvme_free_prps(struct request *req, unsigned int attrs) { struct nvme_iod *iod = blk_mq_rq_to_pdu(req); struct nvme_queue *nvmeq = req->mq_hctx->driver_data; unsigned int i; for (i = 0; i < iod->nr_dma_vecs; i++) - dma_unmap_page(nvmeq->dev->dev, iod->dma_vecs[i].addr, - iod->dma_vecs[i].len, rq_dma_dir(req)); + dma_unmap_phys(nvmeq->dev->dev, iod->dma_vecs[i].addr, + iod->dma_vecs[i].len, rq_dma_dir(req), attrs); mempool_free(iod->dma_vecs, nvmeq->dev->dmavec_mempool); } static void nvme_free_sgls(struct request *req, struct nvme_sgl_desc *sge, - struct nvme_sgl_desc *sg_list) + struct nvme_sgl_desc *sg_list, unsigned int attrs) { struct nvme_queue *nvmeq = req->mq_hctx->driver_data; enum dma_data_direction dir = rq_dma_dir(req); @@ -707,22 +732,25 @@ static void nvme_free_sgls(struct request *req, struct nvme_sgl_desc *sge, unsigned int i; if (sge->type == (NVME_SGL_FMT_DATA_DESC << 4)) { - dma_unmap_page(dma_dev, le64_to_cpu(sge->addr), len, dir); + dma_unmap_phys(dma_dev, le64_to_cpu(sge->addr), len, dir, + attrs); return; } for (i = 0; i < len / sizeof(*sg_list); i++) - dma_unmap_page(dma_dev, le64_to_cpu(sg_list[i].addr), - le32_to_cpu(sg_list[i].length), dir); + dma_unmap_phys(dma_dev, le64_to_cpu(sg_list[i].addr), + le32_to_cpu(sg_list[i].length), dir, attrs); } static void nvme_unmap_metadata(struct request *req) { struct nvme_queue *nvmeq = req->mq_hctx->driver_data; + enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE; enum dma_data_direction dir = rq_dma_dir(req); struct nvme_iod *iod = blk_mq_rq_to_pdu(req); struct device *dma_dev = nvmeq->dev->dev; struct nvme_sgl_desc *sge = iod->meta_descriptor; + unsigned int attrs = 0; if (iod->flags & IOD_SINGLE_META_SEGMENT) { dma_unmap_page(dma_dev, iod->meta_dma, @@ -731,13 +759,20 @@ static void nvme_unmap_metadata(struct request *req) return; } - if (!blk_rq_integrity_dma_unmap(req, dma_dev, &iod->meta_dma_state, - iod->meta_total_len)) { + if (iod->flags & IOD_META_P2P) + map = PCI_P2PDMA_MAP_BUS_ADDR; + else if (iod->flags & IOD_META_MMIO) { + map = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; + attrs |= DMA_ATTR_MMIO; + } + + if (!blk_rq_dma_unmap(req, dma_dev, &iod->meta_dma_state, + iod->meta_total_len, map)) { if (nvme_pci_cmd_use_meta_sgl(&iod->cmd)) - nvme_free_sgls(req, sge, &sge[1]); + nvme_free_sgls(req, sge, &sge[1], attrs); else - dma_unmap_page(dma_dev, iod->meta_dma, - iod->meta_total_len, dir); + dma_unmap_phys(dma_dev, iod->meta_dma, + iod->meta_total_len, dir, attrs); } if (iod->meta_descriptor) @@ -747,9 +782,11 @@ static void nvme_unmap_metadata(struct request *req) static void nvme_unmap_data(struct request *req) { + enum pci_p2pdma_map_type map = PCI_P2PDMA_MAP_NONE; struct nvme_iod *iod = blk_mq_rq_to_pdu(req); struct nvme_queue *nvmeq = req->mq_hctx->driver_data; struct device *dma_dev = nvmeq->dev->dev; + unsigned int attrs = 0; if (iod->flags & IOD_SINGLE_SEGMENT) { static_assert(offsetof(union nvme_data_ptr, prp1) == @@ -759,12 +796,20 @@ static void nvme_unmap_data(struct request *req) return; } - if (!blk_rq_dma_unmap(req, dma_dev, &iod->dma_state, iod->total_len)) { + if (iod->flags & IOD_DATA_P2P) + map = PCI_P2PDMA_MAP_BUS_ADDR; + else if (iod->flags & IOD_DATA_MMIO) { + map = PCI_P2PDMA_MAP_THRU_HOST_BRIDGE; + attrs |= DMA_ATTR_MMIO; + } + + if (!blk_rq_dma_unmap(req, dma_dev, &iod->dma_state, iod->total_len, + map)) { if (nvme_pci_cmd_use_sgl(&iod->cmd)) nvme_free_sgls(req, iod->descriptors[0], - &iod->cmd.common.dptr.sgl); + &iod->cmd.common.dptr.sgl, attrs); else - nvme_free_prps(req); + nvme_free_prps(req, attrs); } if (iod->nr_descriptors) @@ -1035,6 +1080,19 @@ static blk_status_t nvme_map_data(struct request *req) if (!blk_rq_dma_map_iter_start(req, dev->dev, &iod->dma_state, &iter)) return iter.status; + switch (iter.p2pdma.map) { + case PCI_P2PDMA_MAP_BUS_ADDR: + iod->flags |= IOD_DATA_P2P; + break; + case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: + iod->flags |= IOD_DATA_MMIO; + break; + case PCI_P2PDMA_MAP_NONE: + break; + default: + return BLK_STS_RESOURCE; + } + if (use_sgl == SGL_FORCED || (use_sgl == SGL_SUPPORTED && (sgl_threshold && nvme_pci_avg_seg_size(req) >= sgl_threshold))) @@ -1057,6 +1115,19 @@ static blk_status_t nvme_pci_setup_meta_iter(struct request *req) &iod->meta_dma_state, &iter)) return iter.status; + switch (iter.p2pdma.map) { + case PCI_P2PDMA_MAP_BUS_ADDR: + iod->flags |= IOD_META_P2P; + break; + case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: + iod->flags |= IOD_META_MMIO; + break; + case PCI_P2PDMA_MAP_NONE: + break; + default: + return BLK_STS_RESOURCE; + } + if (blk_rq_dma_map_coalesce(&iod->meta_dma_state)) entries = 1; @@ -3250,6 +3321,14 @@ static bool nvme_pci_supports_pci_p2pdma(struct nvme_ctrl *ctrl) return dma_pci_p2pdma_supported(dev->dev); } +static unsigned long nvme_pci_get_virt_boundary(struct nvme_ctrl *ctrl, + bool is_admin) +{ + if (!nvme_ctrl_sgl_supported(ctrl) || is_admin) + return NVME_CTRL_PAGE_SIZE - 1; + return 0; +} + static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { .name = "pcie", .module = THIS_MODULE, @@ -3264,6 +3343,7 @@ static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { .get_address = nvme_pci_get_address, .print_device_info = nvme_pci_print_device_info, .supports_pci_p2pdma = nvme_pci_supports_pci_p2pdma, + .get_virt_boundary = nvme_pci_get_virt_boundary, }; static int nvme_dev_map(struct nvme_dev *dev) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 190a4cfa8a5e..35c0822edb2d 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -2202,6 +2202,7 @@ static const struct nvme_ctrl_ops nvme_rdma_ctrl_ops = { .delete_ctrl = nvme_rdma_delete_ctrl, .get_address = nvmf_get_address, .stop_ctrl = nvme_rdma_stop_ctrl, + .get_virt_boundary = nvme_get_virt_boundary, }; /* diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 6795b8286c35..69cb04406b47 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -2865,6 +2865,7 @@ static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = { .delete_ctrl = nvme_tcp_delete_ctrl, .get_address = nvme_tcp_get_address, .stop_ctrl = nvme_tcp_stop_ctrl, + .get_virt_boundary = nvmf_get_virt_boundary, }; static bool diff --git a/drivers/nvme/host/zns.c b/drivers/nvme/host/zns.c index cce4c5b55aa9..deea2dbef5b8 100644 --- a/drivers/nvme/host/zns.c +++ b/drivers/nvme/host/zns.c @@ -148,8 +148,8 @@ static void *nvme_zns_alloc_report_buffer(struct nvme_ns *ns, static int nvme_zone_parse_entry(struct nvme_ns *ns, struct nvme_zone_descriptor *entry, - unsigned int idx, report_zones_cb cb, - void *data) + unsigned int idx, + struct blk_report_zones_args *args) { struct nvme_ns_head *head = ns->head; struct blk_zone zone = { }; @@ -169,11 +169,11 @@ static int nvme_zone_parse_entry(struct nvme_ns *ns, else zone.wp = nvme_lba_to_sect(head, le64_to_cpu(entry->wp)); - return cb(&zone, idx, data); + return disk_report_zone(ns->disk, &zone, idx, args); } int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data) + unsigned int nr_zones, struct blk_report_zones_args *args) { struct nvme_zone_report *report; struct nvme_command c = { }; @@ -213,7 +213,7 @@ int nvme_ns_report_zones(struct nvme_ns *ns, sector_t sector, for (i = 0; i < nz && zone_idx < nr_zones; i++) { ret = nvme_zone_parse_entry(ns, &report->entries[i], - zone_idx, cb, data); + zone_idx, args); if (ret) goto out_free; zone_idx++; diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index f85a8441bcc6..fc8e7c9ad858 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -511,6 +511,7 @@ static const struct nvme_ctrl_ops nvme_loop_ctrl_ops = { .submit_async_event = nvme_loop_submit_async_event, .delete_ctrl = nvme_loop_delete_ctrl_host, .get_address = nvmf_get_address, + .get_virt_boundary = nvme_get_virt_boundary, }; static int nvme_loop_create_io_queues(struct nvme_loop_ctrl *ctrl) diff --git a/drivers/nvmem/layouts.c b/drivers/nvmem/layouts.c index f381ce1e84bd..7ebe53249035 100644 --- a/drivers/nvmem/layouts.c +++ b/drivers/nvmem/layouts.c @@ -51,7 +51,7 @@ static int nvmem_layout_bus_uevent(const struct device *dev, int ret; ret = of_device_uevent_modalias(dev, env); - if (ret != ENODEV) + if (ret != -ENODEV) return ret; return 0; diff --git a/drivers/of/irq.c b/drivers/of/irq.c index 1cd93549d093..b174ec296489 100644 --- a/drivers/of/irq.c +++ b/drivers/of/irq.c @@ -479,6 +479,26 @@ out: } EXPORT_SYMBOL_GPL(of_irq_get); +const struct cpumask *of_irq_get_affinity(struct device_node *dev, int index) +{ + struct of_phandle_args oirq; + struct irq_fwspec_info info; + struct irq_fwspec fwspec; + int rc; + + rc = of_irq_parse_one(dev, index, &oirq); + if (rc) + return NULL; + + of_phandle_args_to_fwspec(oirq.np, oirq.args, oirq.args_count, + &fwspec); + + if (irq_populate_fwspec_info(&fwspec, &info)) + return NULL; + + return info.affinity; +} + /** * of_irq_get_byname - Decode a node's IRQ and return it as a Linux IRQ number * @dev: pointer to device tree node diff --git a/drivers/opp/core.c b/drivers/opp/core.c index bba4f7daff8c..dbebb8c829bc 100644 --- a/drivers/opp/core.c +++ b/drivers/opp/core.c @@ -309,9 +309,9 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_is_turbo); */ unsigned long dev_pm_opp_get_max_clock_latency(struct device *dev) { - struct opp_table *opp_table __free(put_opp_table); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); - opp_table = _find_opp_table(dev); if (IS_ERR(opp_table)) return 0; @@ -327,7 +327,6 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_clock_latency); */ unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev) { - struct opp_table *opp_table __free(put_opp_table); struct dev_pm_opp *opp; struct regulator *reg; unsigned long latency_ns = 0; @@ -337,7 +336,9 @@ unsigned long dev_pm_opp_get_max_volt_latency(struct device *dev) unsigned long max; } *uV; - opp_table = _find_opp_table(dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); + if (IS_ERR(opp_table)) return 0; @@ -409,10 +410,11 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_max_transition_latency); */ unsigned long dev_pm_opp_get_suspend_opp_freq(struct device *dev) { - struct opp_table *opp_table __free(put_opp_table); unsigned long freq = 0; - opp_table = _find_opp_table(dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); + if (IS_ERR(opp_table)) return 0; @@ -447,9 +449,9 @@ int _get_opp_count(struct opp_table *opp_table) */ int dev_pm_opp_get_opp_count(struct device *dev) { - struct opp_table *opp_table __free(put_opp_table); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); - opp_table = _find_opp_table(dev); if (IS_ERR(opp_table)) { dev_dbg(dev, "%s: OPP table not found (%ld)\n", __func__, PTR_ERR(opp_table)); @@ -605,9 +607,9 @@ _find_key(struct device *dev, unsigned long *key, int index, bool available, unsigned long opp_key, unsigned long key), bool (*assert)(struct opp_table *opp_table, unsigned int index)) { - struct opp_table *opp_table __free(put_opp_table); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); - opp_table = _find_opp_table(dev); if (IS_ERR(opp_table)) { dev_err(dev, "%s: OPP table not found (%ld)\n", __func__, PTR_ERR(opp_table)); @@ -1410,12 +1412,13 @@ static int _set_opp(struct device *dev, struct opp_table *opp_table, */ int dev_pm_opp_set_rate(struct device *dev, unsigned long target_freq) { - struct opp_table *opp_table __free(put_opp_table); struct dev_pm_opp *opp __free(put_opp) = NULL; unsigned long freq = 0, temp_freq; bool forced = false; - opp_table = _find_opp_table(dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); + if (IS_ERR(opp_table)) { dev_err(dev, "%s: device's opp table doesn't exist\n", __func__); return PTR_ERR(opp_table); @@ -1477,9 +1480,9 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_set_rate); */ int dev_pm_opp_set_opp(struct device *dev, struct dev_pm_opp *opp) { - struct opp_table *opp_table __free(put_opp_table); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); - opp_table = _find_opp_table(dev); if (IS_ERR(opp_table)) { dev_err(dev, "%s: device opp doesn't exist\n", __func__); return PTR_ERR(opp_table); @@ -1794,10 +1797,11 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_put); */ void dev_pm_opp_remove(struct device *dev, unsigned long freq) { - struct opp_table *opp_table __free(put_opp_table); struct dev_pm_opp *opp = NULL, *iter; - opp_table = _find_opp_table(dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); + if (IS_ERR(opp_table)) return; @@ -1885,9 +1889,9 @@ bool _opp_remove_all_static(struct opp_table *opp_table) */ void dev_pm_opp_remove_all_dynamic(struct device *dev) { - struct opp_table *opp_table __free(put_opp_table); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); - opp_table = _find_opp_table(dev); if (IS_ERR(opp_table)) return; @@ -2871,10 +2875,11 @@ static int _opp_set_availability(struct device *dev, unsigned long freq, bool availability_req) { struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; - struct opp_table *opp_table __free(put_opp_table); /* Find the opp_table */ - opp_table = _find_opp_table(dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); + if (IS_ERR(opp_table)) { dev_warn(dev, "%s: Device OPP not found (%ld)\n", __func__, PTR_ERR(opp_table)); @@ -2932,11 +2937,12 @@ int dev_pm_opp_adjust_voltage(struct device *dev, unsigned long freq, { struct dev_pm_opp *opp __free(put_opp) = ERR_PTR(-ENODEV), *tmp_opp; - struct opp_table *opp_table __free(put_opp_table); int r; /* Find the opp_table */ - opp_table = _find_opp_table(dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); + if (IS_ERR(opp_table)) { r = PTR_ERR(opp_table); dev_warn(dev, "%s: Device OPP not found (%d)\n", __func__, r); @@ -2986,12 +2992,13 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_adjust_voltage); */ int dev_pm_opp_sync_regulators(struct device *dev) { - struct opp_table *opp_table __free(put_opp_table); struct regulator *reg; int ret, i; /* Device may not have OPP table */ - opp_table = _find_opp_table(dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); + if (IS_ERR(opp_table)) return 0; @@ -3062,9 +3069,9 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_disable); */ int dev_pm_opp_register_notifier(struct device *dev, struct notifier_block *nb) { - struct opp_table *opp_table __free(put_opp_table); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); - opp_table = _find_opp_table(dev); if (IS_ERR(opp_table)) return PTR_ERR(opp_table); @@ -3082,9 +3089,9 @@ EXPORT_SYMBOL(dev_pm_opp_register_notifier); int dev_pm_opp_unregister_notifier(struct device *dev, struct notifier_block *nb) { - struct opp_table *opp_table __free(put_opp_table); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); - opp_table = _find_opp_table(dev); if (IS_ERR(opp_table)) return PTR_ERR(opp_table); @@ -3101,10 +3108,10 @@ EXPORT_SYMBOL(dev_pm_opp_unregister_notifier); */ void dev_pm_opp_remove_table(struct device *dev) { - struct opp_table *opp_table __free(put_opp_table); - /* Check for existing table for 'dev' */ - opp_table = _find_opp_table(dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(dev); + if (IS_ERR(opp_table)) { int error = PTR_ERR(opp_table); diff --git a/drivers/opp/cpu.c b/drivers/opp/cpu.c index 97989d4fe336..a6da7ee3ec76 100644 --- a/drivers/opp/cpu.c +++ b/drivers/opp/cpu.c @@ -56,10 +56,10 @@ int dev_pm_opp_init_cpufreq_table(struct device *dev, return -ENOMEM; for (i = 0, rate = 0; i < max_opps; i++, rate++) { - struct dev_pm_opp *opp __free(put_opp); - /* find next rate */ - opp = dev_pm_opp_find_freq_ceil(dev, &rate); + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_ceil(dev, &rate); + if (IS_ERR(opp)) { ret = PTR_ERR(opp); goto out; @@ -154,12 +154,13 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_cpumask_remove_table); int dev_pm_opp_set_sharing_cpus(struct device *cpu_dev, const struct cpumask *cpumask) { - struct opp_table *opp_table __free(put_opp_table); struct opp_device *opp_dev; struct device *dev; int cpu; - opp_table = _find_opp_table(cpu_dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(cpu_dev); + if (IS_ERR(opp_table)) return PTR_ERR(opp_table); @@ -201,10 +202,11 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_set_sharing_cpus); */ int dev_pm_opp_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask) { - struct opp_table *opp_table __free(put_opp_table); struct opp_device *opp_dev; - opp_table = _find_opp_table(cpu_dev); + struct opp_table *opp_table __free(put_opp_table) = + _find_opp_table(cpu_dev); + if (IS_ERR(opp_table)) return PTR_ERR(opp_table); diff --git a/drivers/opp/of.c b/drivers/opp/of.c index 505d79821584..1e0d0adb18e1 100644 --- a/drivers/opp/of.c +++ b/drivers/opp/of.c @@ -45,9 +45,10 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_of_get_opp_desc_node); struct opp_table *_managed_opp(struct device *dev, int index) { struct opp_table *opp_table, *managed_table = NULL; - struct device_node *np __free(device_node); - np = _opp_of_get_opp_desc_node(dev->of_node, index); + struct device_node *np __free(device_node) = + _opp_of_get_opp_desc_node(dev->of_node, index); + if (!np) return NULL; @@ -95,10 +96,11 @@ static struct device_node *of_parse_required_opp(struct device_node *np, /* The caller must call dev_pm_opp_put_opp_table() after the table is used */ static struct opp_table *_find_table_of_opp_np(struct device_node *opp_np) { - struct device_node *opp_table_np __free(device_node); struct opp_table *opp_table; - opp_table_np = of_get_parent(opp_np); + struct device_node *opp_table_np __free(device_node) = + of_get_parent(opp_np); + if (!opp_table_np) return ERR_PTR(-ENODEV); @@ -146,12 +148,13 @@ static void _opp_table_alloc_required_tables(struct opp_table *opp_table, struct device_node *opp_np) { struct opp_table **required_opp_tables; - struct device_node *np __free(device_node); bool lazy = false; int count, i, size; /* Traversing the first OPP node is all we need */ - np = of_get_next_available_child(opp_np, NULL); + struct device_node *np __free(device_node) = + of_get_next_available_child(opp_np, NULL); + if (!np) { dev_warn(dev, "Empty OPP table\n"); return; @@ -171,9 +174,9 @@ static void _opp_table_alloc_required_tables(struct opp_table *opp_table, opp_table->required_opp_count = count; for (i = 0; i < count; i++) { - struct device_node *required_np __free(device_node); + struct device_node *required_np __free(device_node) = + of_parse_required_opp(np, i); - required_np = of_parse_required_opp(np, i); if (!required_np) { _opp_table_free_required_tables(opp_table); return; @@ -199,14 +202,15 @@ static void _opp_table_alloc_required_tables(struct opp_table *opp_table, void _of_init_opp_table(struct opp_table *opp_table, struct device *dev, int index) { - struct device_node *np __free(device_node), *opp_np; + struct device_node *opp_np; u32 val; /* * Only required for backward compatibility with v1 bindings, but isn't * harmful for other cases. And so we do it unconditionally. */ - np = of_node_get(dev->of_node); + struct device_node *np __free(device_node) = of_node_get(dev->of_node); + if (!np) return; @@ -273,9 +277,9 @@ void _of_clear_opp(struct opp_table *opp_table, struct dev_pm_opp *opp) static int _link_required_opps(struct dev_pm_opp *opp, struct opp_table *required_table, int index) { - struct device_node *np __free(device_node); + struct device_node *np __free(device_node) = + of_parse_required_opp(opp->np, index); - np = of_parse_required_opp(opp->np, index); if (unlikely(!np)) return -ENODEV; @@ -349,16 +353,13 @@ static void lazy_link_required_opp_table(struct opp_table *new_table) guard(mutex)(&opp_table_lock); list_for_each_entry_safe(opp_table, temp, &lazy_opp_tables, lazy) { - struct device_node *opp_np __free(device_node); bool lazy = false; /* opp_np can't be invalid here */ - opp_np = of_get_next_available_child(opp_table->np, NULL); + struct device_node *opp_np __free(device_node) = + of_get_next_available_child(opp_table->np, NULL); for (i = 0; i < opp_table->required_opp_count; i++) { - struct device_node *required_np __free(device_node) = NULL; - struct device_node *required_table_np __free(device_node) = NULL; - required_opp_tables = opp_table->required_opp_tables; /* Required opp-table is already parsed */ @@ -366,8 +367,10 @@ static void lazy_link_required_opp_table(struct opp_table *new_table) continue; /* required_np can't be invalid here */ - required_np = of_parse_required_opp(opp_np, i); - required_table_np = of_get_parent(required_np); + struct device_node *required_np __free(device_node) = + of_parse_required_opp(opp_np, i); + struct device_node *required_table_np __free(device_node) = + of_get_parent(required_np); /* * Newly added table isn't the required opp-table for @@ -402,13 +405,12 @@ static void lazy_link_required_opp_table(struct opp_table *new_table) static int _bandwidth_supported(struct device *dev, struct opp_table *opp_table) { struct device_node *opp_np __free(device_node) = NULL; - struct device_node *np __free(device_node) = NULL; struct property *prop; if (!opp_table) { - struct device_node *np __free(device_node); + struct device_node *np __free(device_node) = + of_node_get(dev->of_node); - np = of_node_get(dev->of_node); if (!np) return -ENODEV; @@ -422,7 +424,9 @@ static int _bandwidth_supported(struct device *dev, struct opp_table *opp_table) return 0; /* Checking only first OPP is sufficient */ - np = of_get_next_available_child(opp_np, NULL); + struct device_node *np __free(device_node) = + of_get_next_available_child(opp_np, NULL); + if (!np) { dev_err(dev, "OPP table empty\n"); return -EINVAL; @@ -1269,11 +1273,12 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_of_cpumask_add_table); int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, struct cpumask *cpumask) { - struct device_node *np __free(device_node); int cpu; /* Get OPP descriptor node */ - np = dev_pm_opp_of_get_opp_desc_node(cpu_dev); + struct device_node *np __free(device_node) = + dev_pm_opp_of_get_opp_desc_node(cpu_dev); + if (!np) { dev_dbg(cpu_dev, "%s: Couldn't find opp node.\n", __func__); return -ENOENT; @@ -1286,13 +1291,12 @@ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, return 0; for_each_possible_cpu(cpu) { - struct device_node *cpu_np __free(device_node) = NULL; - struct device_node *tmp_np __free(device_node) = NULL; - if (cpu == cpu_dev->id) continue; - cpu_np = of_cpu_device_node_get(cpu); + struct device_node *cpu_np __free(device_node) = + of_cpu_device_node_get(cpu); + if (!cpu_np) { dev_err(cpu_dev, "%s: failed to get cpu%d node\n", __func__, cpu); @@ -1300,7 +1304,9 @@ int dev_pm_opp_of_get_sharing_cpus(struct device *cpu_dev, } /* Get OPP descriptor node */ - tmp_np = _opp_of_get_opp_desc_node(cpu_np, 0); + struct device_node *tmp_np __free(device_node) = + _opp_of_get_opp_desc_node(cpu_np, 0); + if (!tmp_np) { pr_err("%pOF: Couldn't find opp node\n", cpu_np); return -ENOENT; @@ -1328,16 +1334,17 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_of_get_sharing_cpus); */ int of_get_required_opp_performance_state(struct device_node *np, int index) { - struct device_node *required_np __free(device_node); - struct opp_table *opp_table __free(put_opp_table) = NULL; - struct dev_pm_opp *opp __free(put_opp) = NULL; int pstate = -EINVAL; - required_np = of_parse_required_opp(np, index); + struct device_node *required_np __free(device_node) = + of_parse_required_opp(np, index); + if (!required_np) return -ENODEV; - opp_table = _find_table_of_opp_np(required_np); + struct opp_table *opp_table __free(put_opp_table) = + _find_table_of_opp_np(required_np); + if (IS_ERR(opp_table)) { pr_err("%s: Failed to find required OPP table %pOF: %ld\n", __func__, np, PTR_ERR(opp_table)); @@ -1350,7 +1357,9 @@ int of_get_required_opp_performance_state(struct device_node *np, int index) return -EINVAL; } - opp = _find_opp_of_np(opp_table, required_np); + struct dev_pm_opp *opp __free(put_opp) = + _find_opp_of_np(opp_table, required_np); + if (opp) { if (opp->level == OPP_LEVEL_UNSET) { pr_err("%s: OPP levels aren't available for %pOF\n", @@ -1376,14 +1385,17 @@ EXPORT_SYMBOL_GPL(of_get_required_opp_performance_state); */ bool dev_pm_opp_of_has_required_opp(struct device *dev) { - struct device_node *np __free(device_node) = NULL, *opp_np __free(device_node); int count; - opp_np = _opp_of_get_opp_desc_node(dev->of_node, 0); + struct device_node *opp_np __free(device_node) = + _opp_of_get_opp_desc_node(dev->of_node, 0); + if (!opp_np) return false; - np = of_get_next_available_child(opp_np, NULL); + struct device_node *np __free(device_node) = + of_get_next_available_child(opp_np, NULL); + if (!np) { dev_warn(dev, "Empty OPP table\n"); return false; @@ -1425,12 +1437,14 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_get_of_node); static int __maybe_unused _get_dt_power(struct device *dev, unsigned long *uW, unsigned long *kHz) { - struct dev_pm_opp *opp __free(put_opp); unsigned long opp_freq, opp_power; /* Find the right frequency and related OPP */ opp_freq = *kHz * 1000; - opp = dev_pm_opp_find_freq_ceil(dev, &opp_freq); + + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_ceil(dev, &opp_freq); + if (IS_ERR(opp)) return -EINVAL; @@ -1465,14 +1479,13 @@ _get_dt_power(struct device *dev, unsigned long *uW, unsigned long *kHz) int dev_pm_opp_calc_power(struct device *dev, unsigned long *uW, unsigned long *kHz) { - struct dev_pm_opp *opp __free(put_opp) = NULL; - struct device_node *np __free(device_node); unsigned long mV, Hz; u32 cap; u64 tmp; int ret; - np = of_node_get(dev->of_node); + struct device_node *np __free(device_node) = of_node_get(dev->of_node); + if (!np) return -EINVAL; @@ -1481,7 +1494,10 @@ int dev_pm_opp_calc_power(struct device *dev, unsigned long *uW, return -EINVAL; Hz = *kHz * 1000; - opp = dev_pm_opp_find_freq_ceil(dev, &Hz); + + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_ceil(dev, &Hz); + if (IS_ERR(opp)) return -EINVAL; @@ -1502,11 +1518,12 @@ EXPORT_SYMBOL_GPL(dev_pm_opp_calc_power); static bool _of_has_opp_microwatt_property(struct device *dev) { - struct dev_pm_opp *opp __free(put_opp); unsigned long freq = 0; /* Check if at least one OPP has needed property */ - opp = dev_pm_opp_find_freq_ceil(dev, &freq); + struct dev_pm_opp *opp __free(put_opp) = + dev_pm_opp_find_freq_ceil(dev, &freq); + if (IS_ERR(opp)) return false; @@ -1526,12 +1543,16 @@ static bool _of_has_opp_microwatt_property(struct device *dev) */ int dev_pm_opp_of_register_em(struct device *dev, struct cpumask *cpus) { - struct device_node *np __free(device_node) = NULL; struct em_data_callback em_cb; int ret, nr_opp; u32 cap; - if (IS_ERR_OR_NULL(dev)) { + if (IS_ERR_OR_NULL(dev)) + return -EINVAL; + + struct device_node *np __free(device_node) = of_node_get(dev->of_node); + + if (!np) { ret = -EINVAL; goto failed; } @@ -1548,12 +1569,6 @@ int dev_pm_opp_of_register_em(struct device *dev, struct cpumask *cpus) goto register_em; } - np = of_node_get(dev->of_node); - if (!np) { - ret = -EINVAL; - goto failed; - } - /* * Register an EM only if the 'dynamic-power-coefficient' property is * set in devicetree. It is assumed the voltage values are known if that diff --git a/drivers/pci/controller/pcie-iproc.c b/drivers/pci/controller/pcie-iproc.c index 22134e95574b..ccf71993ea35 100644 --- a/drivers/pci/controller/pcie-iproc.c +++ b/drivers/pci/controller/pcie-iproc.c @@ -17,6 +17,7 @@ #include <linux/irqchip/arm-gic-v3.h> #include <linux/platform_device.h> #include <linux/of_address.h> +#include <linux/of_irq.h> #include <linux/of_pci.h> #include <linux/of_platform.h> #include <linux/phy/phy.h> @@ -1337,29 +1338,16 @@ static int iproc_pcie_msi_steer(struct iproc_pcie *pcie, static int iproc_pcie_msi_enable(struct iproc_pcie *pcie) { - struct device_node *msi_node; + struct device_node *msi_node = NULL; int ret; /* * Either the "msi-parent" or the "msi-map" phandle needs to exist * for us to obtain the MSI node. */ - - msi_node = of_parse_phandle(pcie->dev->of_node, "msi-parent", 0); - if (!msi_node) { - const __be32 *msi_map = NULL; - int len; - u32 phandle; - - msi_map = of_get_property(pcie->dev->of_node, "msi-map", &len); - if (!msi_map) - return -ENODEV; - - phandle = be32_to_cpup(msi_map + 1); - msi_node = of_find_node_by_phandle(phandle); - if (!msi_node) - return -ENODEV; - } + of_msi_xlate(pcie->dev, &msi_node, 0); + if (!msi_node) + return -ENODEV; /* * Certain revisions of the iProc PCIe controller require additional diff --git a/drivers/pci/endpoint/functions/pci-epf-test.c b/drivers/pci/endpoint/functions/pci-epf-test.c index 31617772ad51..0686f7ecbe91 100644 --- a/drivers/pci/endpoint/functions/pci-epf-test.c +++ b/drivers/pci/endpoint/functions/pci-epf-test.c @@ -331,9 +331,8 @@ static void pci_epf_test_print_rate(struct pci_epf_test *epf_test, rate = div64_u64(size * NSEC_PER_SEC, ns * 1000); dev_info(&epf_test->epf->dev, - "%s => Size: %llu B, DMA: %s, Time: %llu.%09u s, Rate: %llu KB/s\n", - op, size, dma ? "YES" : "NO", - (u64)ts.tv_sec, (u32)ts.tv_nsec, rate); + "%s => Size: %llu B, DMA: %s, Time: %ptSp s, Rate: %llu KB/s\n", + op, size, dma ? "YES" : "NO", &ts, rate); } static void pci_epf_test_copy(struct pci_epf_test *epf_test, diff --git a/drivers/pci/hotplug/s390_pci_hpc.c b/drivers/pci/hotplug/s390_pci_hpc.c index d9996516f49e..a55967082ef6 100644 --- a/drivers/pci/hotplug/s390_pci_hpc.c +++ b/drivers/pci/hotplug/s390_pci_hpc.c @@ -8,8 +8,7 @@ * Jan Glauber <jang@linux.vnet.ibm.com> */ -#define KMSG_COMPONENT "zpci" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zpci: " fmt #include <linux/kernel.h> #include <linux/slab.h> diff --git a/drivers/pci/msi/irqdomain.c b/drivers/pci/msi/irqdomain.c index ce741ed9dc3f..a329060287b5 100644 --- a/drivers/pci/msi/irqdomain.c +++ b/drivers/pci/msi/irqdomain.c @@ -49,96 +49,6 @@ static void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg * __pci_write_msi_msg(desc, msg); } -/** - * pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source - * @desc: Pointer to the MSI descriptor - * - * The ID number is only used within the irqdomain. - */ -static irq_hw_number_t pci_msi_domain_calc_hwirq(struct msi_desc *desc) -{ - struct pci_dev *dev = msi_desc_to_pci_dev(desc); - - return (irq_hw_number_t)desc->msi_index | - pci_dev_id(dev) << 11 | - ((irq_hw_number_t)(pci_domain_nr(dev->bus) & 0xFFFFFFFF)) << 27; -} - -static void pci_msi_domain_set_desc(msi_alloc_info_t *arg, - struct msi_desc *desc) -{ - arg->desc = desc; - arg->hwirq = pci_msi_domain_calc_hwirq(desc); -} - -static struct msi_domain_ops pci_msi_domain_ops_default = { - .set_desc = pci_msi_domain_set_desc, -}; - -static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info) -{ - struct msi_domain_ops *ops = info->ops; - - if (ops == NULL) { - info->ops = &pci_msi_domain_ops_default; - } else { - if (ops->set_desc == NULL) - ops->set_desc = pci_msi_domain_set_desc; - } -} - -static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info) -{ - struct irq_chip *chip = info->chip; - - BUG_ON(!chip); - if (!chip->irq_write_msi_msg) - chip->irq_write_msi_msg = pci_msi_domain_write_msg; - if (!chip->irq_mask) - chip->irq_mask = pci_msi_mask_irq; - if (!chip->irq_unmask) - chip->irq_unmask = pci_msi_unmask_irq; -} - -/** - * pci_msi_create_irq_domain - Create a MSI interrupt domain - * @fwnode: Optional fwnode of the interrupt controller - * @info: MSI domain info - * @parent: Parent irq domain - * - * Updates the domain and chip ops and creates a MSI interrupt domain. - * - * Returns: - * A domain pointer or NULL in case of failure. - */ -struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode, - struct msi_domain_info *info, - struct irq_domain *parent) -{ - if (WARN_ON(info->flags & MSI_FLAG_LEVEL_CAPABLE)) - info->flags &= ~MSI_FLAG_LEVEL_CAPABLE; - - if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS) - pci_msi_domain_update_dom_ops(info); - if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS) - pci_msi_domain_update_chip_ops(info); - - /* Let the core code free MSI descriptors when freeing interrupts */ - info->flags |= MSI_FLAG_FREE_MSI_DESCS; - - info->flags |= MSI_FLAG_ACTIVATE_EARLY | MSI_FLAG_DEV_SYSFS; - if (IS_ENABLED(CONFIG_GENERIC_IRQ_RESERVATION_MODE)) - info->flags |= MSI_FLAG_MUST_REACTIVATE; - - /* PCI-MSI is oneshot-safe */ - info->chip->flags |= IRQCHIP_ONESHOT_SAFE; - /* Let the core update the bus token */ - info->bus_token = DOMAIN_BUS_PCI_MSI; - - return msi_create_irq_domain(fwnode, info, parent); -} -EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain); - /* * Per device MSI[-X] domain functionality */ diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 9d6f74bd95f8..3881359440b1 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -1517,8 +1517,8 @@ static ssize_t reset_method_store(struct device *dev, return count; } - ACQUIRE(pm_runtime_active_try, pm)(dev); - if (ACQUIRE_ERR(pm_runtime_active_try, &pm)) + PM_RUNTIME_ACQUIRE(dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) return -ENXIO; if (sysfs_streq(buf, "default")) { diff --git a/drivers/perf/arm-ni.c b/drivers/perf/arm-ni.c index 1615a0564031..66858c65215d 100644 --- a/drivers/perf/arm-ni.c +++ b/drivers/perf/arm-ni.c @@ -21,6 +21,11 @@ #define NI_CHILD_NODE_INFO 0x004 #define NI_CHILD_PTR(n) (0x008 + (n) * 4) +#define NI_NUM_SUB_FEATURES 0x100 +#define NI_SUB_FEATURE_TYPE(n) (0x108 + (n) * 8) +#define NI_SUB_FEATURE_PTR(n) (0x10c + (n) * 8) + +#define NI_SUB_FEATURE_TYPE_FCU 0x2 #define NI700_PMUSELA 0x00c @@ -33,9 +38,10 @@ #define NI_PIDR2_VERSION GENMASK(7, 4) /* PMU node */ -#define NI_PMEVCNTR(n) (0x008 + (n) * 8) -#define NI_PMCCNTR_L 0x0f8 -#define NI_PMCCNTR_U 0x0fc +#define NI700_PMEVCNTR(n) (0x008 + (n) * 8) +#define NI700_PMCCNTR_L 0x0f8 +#define NI_PMEVCNTR(n) (0x200 + (n) * 8) +#define NI_PMCCNTR_L 0x2f8 #define NI_PMEVTYPER(n) (0x400 + (n) * 4) #define NI_PMEVTYPER_NODE_TYPE GENMASK(12, 9) #define NI_PMEVTYPER_NODE_ID GENMASK(8, 0) @@ -66,6 +72,8 @@ enum ni_part { PART_NI_700 = 0x43b, PART_NI_710AE = 0x43d, + PART_NOC_S3 = 0x43f, + PART_SI_L1 = 0x455, }; enum ni_node_type { @@ -79,6 +87,10 @@ enum ni_node_type { NI_HSNI, NI_HMNI, NI_PMNI, + NI_TSNI, + NI_TMNI, + NI_CMNI = 0x0e, + NI_MCN = 0x63, }; struct arm_ni_node { @@ -179,6 +191,9 @@ static struct attribute *arm_ni_event_attrs[] = { NI_EVENT_ATTR(hsni, NI_HSNI), NI_EVENT_ATTR(hmni, NI_HMNI), NI_EVENT_ATTR(pmni, NI_PMNI), + NI_EVENT_ATTR(tsni, NI_TSNI), + NI_EVENT_ATTR(tmni, NI_TMNI), + NI_EVENT_ATTR(cmni, NI_CMNI), NULL }; @@ -308,9 +323,15 @@ static int arm_ni_validate_group(struct perf_event *event) return 0; } +static bool arm_ni_is_7xx(const struct arm_ni *ni) +{ + return ni->part == PART_NI_700 || ni->part == PART_NI_710AE; +} + static int arm_ni_event_init(struct perf_event *event) { struct arm_ni_cd *cd = pmu_to_cd(event->pmu); + struct arm_ni *ni; if (event->attr.type != event->pmu->type) return -ENOENT; @@ -318,7 +339,10 @@ static int arm_ni_event_init(struct perf_event *event) if (is_sampling_event(event)) return -EINVAL; - event->cpu = cd_to_ni(cd)->cpu; + ni = cd_to_ni(cd); + event->cpu = ni->cpu; + event->hw.flags = arm_ni_is_7xx(ni); + if (NI_EVENT_TYPE(event) == NI_PMU) return arm_ni_validate_group(event); @@ -332,16 +356,16 @@ static int arm_ni_event_init(struct perf_event *event) return -EINVAL; } -static u64 arm_ni_read_ccnt(struct arm_ni_cd *cd) +static u64 arm_ni_read_ccnt(void __iomem *pmccntr) { u64 l, u_old, u_new; int retries = 3; /* 1st time unlucky, 2nd improbable, 3rd just broken */ - u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U); + u_new = readl_relaxed(pmccntr + 4); do { u_old = u_new; - l = readl_relaxed(cd->pmu_base + NI_PMCCNTR_L); - u_new = readl_relaxed(cd->pmu_base + NI_PMCCNTR_U); + l = readl_relaxed(pmccntr); + u_new = readl_relaxed(pmccntr + 4); } while (u_new != u_old && --retries); WARN_ON(!retries); @@ -350,7 +374,6 @@ static u64 arm_ni_read_ccnt(struct arm_ni_cd *cd) static void arm_ni_event_read(struct perf_event *event) { - struct arm_ni_cd *cd = pmu_to_cd(event->pmu); struct hw_perf_event *hw = &event->hw; u64 count, prev; bool ccnt = hw->idx == NI_CCNT_IDX; @@ -358,9 +381,9 @@ static void arm_ni_event_read(struct perf_event *event) do { prev = local64_read(&hw->prev_count); if (ccnt) - count = arm_ni_read_ccnt(cd); + count = arm_ni_read_ccnt((void __iomem *)event->hw.event_base); else - count = readl_relaxed(cd->pmu_base + NI_PMEVCNTR(hw->idx)); + count = readl_relaxed((void __iomem *)event->hw.event_base); } while (local64_cmpxchg(&hw->prev_count, prev, count) != prev); count -= prev; @@ -385,16 +408,16 @@ static void arm_ni_event_stop(struct perf_event *event, int flags) arm_ni_event_read(event); } -static void arm_ni_init_ccnt(struct arm_ni_cd *cd) +static void arm_ni_init_ccnt(struct hw_perf_event *hw) { - local64_set(&cd->ccnt->hw.prev_count, S64_MIN); - lo_hi_writeq_relaxed(S64_MIN, cd->pmu_base + NI_PMCCNTR_L); + local64_set(&hw->prev_count, S64_MIN); + lo_hi_writeq_relaxed(S64_MIN, (void __iomem *)hw->event_base); } -static void arm_ni_init_evcnt(struct arm_ni_cd *cd, int idx) +static void arm_ni_init_evcnt(struct hw_perf_event *hw) { - local64_set(&cd->evcnt[idx]->hw.prev_count, S32_MIN); - writel_relaxed(S32_MIN, cd->pmu_base + NI_PMEVCNTR(idx)); + local64_set(&hw->prev_count, S32_MIN); + writel_relaxed(S32_MIN, (void __iomem *)hw->event_base); } static int arm_ni_event_add(struct perf_event *event, int flags) @@ -409,8 +432,10 @@ static int arm_ni_event_add(struct perf_event *event, int flags) if (cd->ccnt) return -ENOSPC; hw->idx = NI_CCNT_IDX; + hw->event_base = (unsigned long)cd->pmu_base + + (hw->flags ? NI700_PMCCNTR_L : NI_PMCCNTR_L); cd->ccnt = event; - arm_ni_init_ccnt(cd); + arm_ni_init_ccnt(hw); } else { hw->idx = 0; while (cd->evcnt[hw->idx]) { @@ -420,7 +445,9 @@ static int arm_ni_event_add(struct perf_event *event, int flags) cd->evcnt[hw->idx] = event; unit = (void *)hw->config_base; unit->event[hw->idx] = NI_EVENT_EVENTID(event); - arm_ni_init_evcnt(cd, hw->idx); + hw->event_base = (unsigned long)cd->pmu_base + + (hw->flags ? NI700_PMEVCNTR(hw->idx) : NI_PMEVCNTR(hw->idx)); + arm_ni_init_evcnt(hw); lo_hi_writeq_relaxed(le64_to_cpu(unit->pmusel), unit->pmusela); reg = FIELD_PREP(NI_PMEVTYPER_NODE_TYPE, type) | @@ -457,7 +484,7 @@ static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id) ret = IRQ_HANDLED; if (!(WARN_ON(!cd->ccnt))) { arm_ni_event_read(cd->ccnt); - arm_ni_init_ccnt(cd); + arm_ni_init_ccnt(&cd->ccnt->hw); } } for (int i = 0; i < NI_NUM_COUNTERS; i++) { @@ -466,7 +493,7 @@ static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id) ret = IRQ_HANDLED; if (!(WARN_ON(!cd->evcnt[i]))) { arm_ni_event_read(cd->evcnt[i]); - arm_ni_init_evcnt(cd, i); + arm_ni_init_evcnt(&cd->evcnt[i]->hw); } } writel_relaxed(reg, cd->pmu_base + NI_PMOVSCLR); @@ -476,6 +503,25 @@ static irqreturn_t arm_ni_handle_irq(int irq, void *dev_id) } } +static void __iomem *arm_ni_get_pmusel(struct arm_ni *ni, void __iomem *unit_base) +{ + u32 type, ptr, num; + + if (arm_ni_is_7xx(ni)) + return unit_base + NI700_PMUSELA; + + num = readl_relaxed(unit_base + NI_NUM_SUB_FEATURES); + for (int i = 0; i < num; i++) { + type = readl_relaxed(unit_base + NI_SUB_FEATURE_TYPE(i)); + if (type != NI_SUB_FEATURE_TYPE_FCU) + continue; + ptr = readl_relaxed(unit_base + NI_SUB_FEATURE_PTR(i)); + return ni->base + ptr; + } + /* Should be impossible */ + return NULL; +} + static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_start) { struct arm_ni_cd *cd = ni->cds + node->id; @@ -512,13 +558,18 @@ static int arm_ni_init_cd(struct arm_ni *ni, struct arm_ni_node *node, u64 res_s case NI_HSNI: case NI_HMNI: case NI_PMNI: - unit->pmusela = unit_base + NI700_PMUSELA; + case NI_TSNI: + case NI_TMNI: + case NI_CMNI: + unit->pmusela = arm_ni_get_pmusel(ni, unit_base); writel_relaxed(1, unit->pmusela); if (readl_relaxed(unit->pmusela) != 1) dev_info(ni->dev, "No access to node 0x%04x%04x\n", unit->id, unit->type); else unit->ns = true; break; + case NI_MCN: + break; default: /* * e.g. FMU - thankfully bits 3:2 of FMU_ERR_FR0 are RES0 so @@ -649,6 +700,8 @@ static int arm_ni_probe(struct platform_device *pdev) switch (part) { case PART_NI_700: case PART_NI_710AE: + case PART_NOC_S3: + case PART_SI_L1: break; default: dev_WARN(&pdev->dev, "Unknown part number: 0x%03x, this may go badly\n", part); diff --git a/drivers/perf/arm_cspmu/arm_cspmu.c b/drivers/perf/arm_cspmu/arm_cspmu.c index efa9b229e701..34430b68f602 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.c +++ b/drivers/perf/arm_cspmu/arm_cspmu.c @@ -322,14 +322,14 @@ static struct arm_cspmu_impl_match impl_match[] = { { .module_name = "nvidia_cspmu", .pmiidr_val = ARM_CSPMU_IMPL_ID_NVIDIA, - .pmiidr_mask = ARM_CSPMU_PMIIDR_IMPLEMENTER, + .pmiidr_mask = PMIIDR_IMPLEMENTER, .module = NULL, .impl_init_ops = NULL, }, { .module_name = "ampere_cspmu", .pmiidr_val = ARM_CSPMU_IMPL_ID_AMPERE, - .pmiidr_mask = ARM_CSPMU_PMIIDR_IMPLEMENTER, + .pmiidr_mask = PMIIDR_IMPLEMENTER, .module = NULL, .impl_init_ops = NULL, }, @@ -351,6 +351,44 @@ static struct arm_cspmu_impl_match *arm_cspmu_impl_match_get(u32 pmiidr) return NULL; } +static u32 arm_cspmu_get_pmiidr(struct arm_cspmu *cspmu) +{ + u32 pmiidr, pmpidr; + + pmiidr = readl(cspmu->base0 + PMIIDR); + + if (pmiidr != 0) + return pmiidr; + + /* Construct PMIIDR value from PMPIDRs. */ + + pmpidr = readl(cspmu->base0 + PMPIDR0); + pmiidr |= FIELD_PREP(PMIIDR_PRODUCTID_PART_0, + FIELD_GET(PMPIDR0_PART_0, pmpidr)); + + pmpidr = readl(cspmu->base0 + PMPIDR1); + pmiidr |= FIELD_PREP(PMIIDR_PRODUCTID_PART_1, + FIELD_GET(PMPIDR1_PART_1, pmpidr)); + pmiidr |= FIELD_PREP(PMIIDR_IMPLEMENTER_DES_0, + FIELD_GET(PMPIDR1_DES_0, pmpidr)); + + pmpidr = readl(cspmu->base0 + PMPIDR2); + pmiidr |= FIELD_PREP(PMIIDR_VARIANT, + FIELD_GET(PMPIDR2_REVISION, pmpidr)); + pmiidr |= FIELD_PREP(PMIIDR_IMPLEMENTER_DES_1, + FIELD_GET(PMPIDR2_DES_1, pmpidr)); + + pmpidr = readl(cspmu->base0 + PMPIDR3); + pmiidr |= FIELD_PREP(PMIIDR_REVISION, + FIELD_GET(PMPIDR3_REVAND, pmpidr)); + + pmpidr = readl(cspmu->base0 + PMPIDR4); + pmiidr |= FIELD_PREP(PMIIDR_IMPLEMENTER_DES_2, + FIELD_GET(PMPIDR4_DES_2, pmpidr)); + + return pmiidr; +} + #define DEFAULT_IMPL_OP(name) .name = arm_cspmu_##name static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu) @@ -361,7 +399,7 @@ static int arm_cspmu_init_impl_ops(struct arm_cspmu *cspmu) /* Start with a default PMU implementation */ cspmu->impl.module = THIS_MODULE; - cspmu->impl.pmiidr = readl(cspmu->base0 + PMIIDR); + cspmu->impl.pmiidr = arm_cspmu_get_pmiidr(cspmu); cspmu->impl.ops = (struct arm_cspmu_impl_ops) { DEFAULT_IMPL_OP(get_event_attrs), DEFAULT_IMPL_OP(get_format_attrs), @@ -815,6 +853,10 @@ static void arm_cspmu_stop(struct perf_event *event, int pmu_flags) return; arm_cspmu_disable_counter(cspmu, hwc->idx); + + if (cspmu->impl.ops.reset_ev_filter) + cspmu->impl.ops.reset_ev_filter(cspmu, event); + arm_cspmu_event_update(event); hwc->state |= PERF_HES_STOPPED | PERF_HES_UPTODATE; @@ -1365,8 +1407,10 @@ void arm_cspmu_impl_unregister(const struct arm_cspmu_impl_match *impl_match) /* Unbind the driver from all matching backend devices. */ while ((dev = driver_find_device(&arm_cspmu_driver.driver, NULL, - match, arm_cspmu_match_device))) + match, arm_cspmu_match_device))) { device_release_driver(dev); + put_device(dev); + } mutex_lock(&arm_cspmu_lock); diff --git a/drivers/perf/arm_cspmu/arm_cspmu.h b/drivers/perf/arm_cspmu/arm_cspmu.h index 19684b76bd96..cd65a58dbd88 100644 --- a/drivers/perf/arm_cspmu/arm_cspmu.h +++ b/drivers/perf/arm_cspmu/arm_cspmu.h @@ -86,6 +86,11 @@ #define PMCFGR 0xE00 #define PMCR 0xE04 #define PMIIDR 0xE08 +#define PMPIDR0 0xFE0 +#define PMPIDR1 0xFE4 +#define PMPIDR2 0xFE8 +#define PMPIDR3 0xFEC +#define PMPIDR4 0xFD0 /* PMCFGR register field */ #define PMCFGR_NCG GENMASK(31, 28) @@ -115,8 +120,34 @@ #define PMCR_E BIT(0) /* PMIIDR register field */ -#define ARM_CSPMU_PMIIDR_IMPLEMENTER GENMASK(11, 0) -#define ARM_CSPMU_PMIIDR_PRODUCTID GENMASK(31, 20) +#define PMIIDR_IMPLEMENTER GENMASK(11, 0) +#define PMIIDR_IMPLEMENTER_DES_0 GENMASK(3, 0) +#define PMIIDR_IMPLEMENTER_DES_1 GENMASK(6, 4) +#define PMIIDR_IMPLEMENTER_DES_2 GENMASK(11, 8) +#define PMIIDR_REVISION GENMASK(15, 12) +#define PMIIDR_VARIANT GENMASK(19, 16) +#define PMIIDR_PRODUCTID GENMASK(31, 20) +#define PMIIDR_PRODUCTID_PART_0 GENMASK(27, 20) +#define PMIIDR_PRODUCTID_PART_1 GENMASK(31, 28) + +/* PMPIDR0 register field */ +#define PMPIDR0_PART_0 GENMASK(7, 0) + +/* PMPIDR1 register field */ +#define PMPIDR1_DES_0 GENMASK(7, 4) +#define PMPIDR1_PART_1 GENMASK(3, 0) + +/* PMPIDR2 register field */ +#define PMPIDR2_REVISION GENMASK(7, 4) +#define PMPIDR2_DES_1 GENMASK(2, 0) + +/* PMPIDR3 register field */ +#define PMPIDR3_REVAND GENMASK(7, 4) +#define PMPIDR3_CMOD GENMASK(3, 0) + +/* PMPIDR4 register field */ +#define PMPIDR4_SIZE GENMASK(7, 4) +#define PMPIDR4_DES_2 GENMASK(3, 0) /* JEDEC-assigned JEP106 identification code */ #define ARM_CSPMU_IMPL_ID_NVIDIA 0x36B @@ -152,11 +183,13 @@ struct arm_cspmu_impl_ops { bool (*is_cycle_counter_event)(const struct perf_event *event); /* Decode event type/id from configs */ u32 (*event_type)(const struct perf_event *event); - /* Set event filters */ + /* Set/reset event filters */ void (*set_cc_filter)(struct arm_cspmu *cspmu, const struct perf_event *event); void (*set_ev_filter)(struct arm_cspmu *cspmu, const struct perf_event *event); + void (*reset_ev_filter)(struct arm_cspmu *cspmu, + const struct perf_event *event); /* Implementation specific event validation */ int (*validate_event)(struct arm_cspmu *cspmu, struct perf_event *event); diff --git a/drivers/perf/arm_cspmu/nvidia_cspmu.c b/drivers/perf/arm_cspmu/nvidia_cspmu.c index dc6d4e3e2a1b..e06a06d3407b 100644 --- a/drivers/perf/arm_cspmu/nvidia_cspmu.c +++ b/drivers/perf/arm_cspmu/nvidia_cspmu.c @@ -23,7 +23,7 @@ #define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0) -#define NV_PRODID_MASK GENMASK(31, 0) +#define NV_PRODID_MASK (PMIIDR_PRODUCTID | PMIIDR_VARIANT | PMIIDR_REVISION) #define NV_FORMAT_NAME_GENERIC 0 @@ -40,10 +40,21 @@ struct nv_cspmu_ctx { const char *name; - u32 filter_mask; - u32 filter_default_val; + struct attribute **event_attr; struct attribute **format_attr; + + u32 filter_mask; + u32 filter_default_val; + u32 filter2_mask; + u32 filter2_default_val; + + u32 (*get_filter)(const struct perf_event *event); + u32 (*get_filter2)(const struct perf_event *event); + + void *data; + + int (*init_data)(struct arm_cspmu *cspmu); }; static struct attribute *scf_pmu_event_attrs[] = { @@ -144,6 +155,7 @@ static struct attribute *cnvlink_pmu_format_attrs[] = { static struct attribute *generic_pmu_format_attrs[] = { ARM_CSPMU_FORMAT_EVENT_ATTR, ARM_CSPMU_FORMAT_FILTER_ATTR, + ARM_CSPMU_FORMAT_FILTER2_ATTR, NULL, }; @@ -184,13 +196,36 @@ static u32 nv_cspmu_event_filter(const struct perf_event *event) return filter_val; } +static u32 nv_cspmu_event_filter2(const struct perf_event *event) +{ + const struct nv_cspmu_ctx *ctx = + to_nv_cspmu_ctx(to_arm_cspmu(event->pmu)); + + const u32 filter_val = event->attr.config2 & ctx->filter2_mask; + + if (filter_val == 0) + return ctx->filter2_default_val; + + return filter_val; +} + static void nv_cspmu_set_ev_filter(struct arm_cspmu *cspmu, const struct perf_event *event) { - u32 filter = nv_cspmu_event_filter(event); - u32 offset = PMEVFILTR + (4 * event->hw.idx); + u32 filter, offset; + const struct nv_cspmu_ctx *ctx = + to_nv_cspmu_ctx(to_arm_cspmu(event->pmu)); + offset = 4 * event->hw.idx; + + if (ctx->get_filter) { + filter = ctx->get_filter(event); + writel(filter, cspmu->base0 + PMEVFILTR + offset); + } - writel(filter, cspmu->base0 + offset); + if (ctx->get_filter2) { + filter = ctx->get_filter2(event); + writel(filter, cspmu->base0 + PMEVFILT2R + offset); + } } static void nv_cspmu_set_cc_filter(struct arm_cspmu *cspmu, @@ -210,74 +245,120 @@ enum nv_cspmu_name_fmt { struct nv_cspmu_match { u32 prodid; u32 prodid_mask; - u64 filter_mask; - u32 filter_default_val; const char *name_pattern; enum nv_cspmu_name_fmt name_fmt; - struct attribute **event_attr; - struct attribute **format_attr; + struct nv_cspmu_ctx template_ctx; + struct arm_cspmu_impl_ops ops; }; static const struct nv_cspmu_match nv_cspmu_match[] = { { - .prodid = 0x103, + .prodid = 0x10300000, .prodid_mask = NV_PRODID_MASK, - .filter_mask = NV_PCIE_FILTER_ID_MASK, - .filter_default_val = NV_PCIE_FILTER_ID_MASK, .name_pattern = "nvidia_pcie_pmu_%u", .name_fmt = NAME_FMT_SOCKET, - .event_attr = mcf_pmu_event_attrs, - .format_attr = pcie_pmu_format_attrs + .template_ctx = { + .event_attr = mcf_pmu_event_attrs, + .format_attr = pcie_pmu_format_attrs, + .filter_mask = NV_PCIE_FILTER_ID_MASK, + .filter_default_val = NV_PCIE_FILTER_ID_MASK, + .filter2_mask = 0x0, + .filter2_default_val = 0x0, + .get_filter = nv_cspmu_event_filter, + .get_filter2 = NULL, + .data = NULL, + .init_data = NULL + }, }, { - .prodid = 0x104, + .prodid = 0x10400000, .prodid_mask = NV_PRODID_MASK, - .filter_mask = NV_NVL_C2C_FILTER_ID_MASK, - .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, .name_pattern = "nvidia_nvlink_c2c1_pmu_%u", .name_fmt = NAME_FMT_SOCKET, - .event_attr = mcf_pmu_event_attrs, - .format_attr = nvlink_c2c_pmu_format_attrs + .template_ctx = { + .event_attr = mcf_pmu_event_attrs, + .format_attr = nvlink_c2c_pmu_format_attrs, + .filter_mask = NV_NVL_C2C_FILTER_ID_MASK, + .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, + .filter2_mask = 0x0, + .filter2_default_val = 0x0, + .get_filter = nv_cspmu_event_filter, + .get_filter2 = NULL, + .data = NULL, + .init_data = NULL + }, }, { - .prodid = 0x105, + .prodid = 0x10500000, .prodid_mask = NV_PRODID_MASK, - .filter_mask = NV_NVL_C2C_FILTER_ID_MASK, - .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, .name_pattern = "nvidia_nvlink_c2c0_pmu_%u", .name_fmt = NAME_FMT_SOCKET, - .event_attr = mcf_pmu_event_attrs, - .format_attr = nvlink_c2c_pmu_format_attrs + .template_ctx = { + .event_attr = mcf_pmu_event_attrs, + .format_attr = nvlink_c2c_pmu_format_attrs, + .filter_mask = NV_NVL_C2C_FILTER_ID_MASK, + .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK, + .filter2_mask = 0x0, + .filter2_default_val = 0x0, + .get_filter = nv_cspmu_event_filter, + .get_filter2 = NULL, + .data = NULL, + .init_data = NULL + }, }, { - .prodid = 0x106, + .prodid = 0x10600000, .prodid_mask = NV_PRODID_MASK, - .filter_mask = NV_CNVL_FILTER_ID_MASK, - .filter_default_val = NV_CNVL_FILTER_ID_MASK, .name_pattern = "nvidia_cnvlink_pmu_%u", .name_fmt = NAME_FMT_SOCKET, - .event_attr = mcf_pmu_event_attrs, - .format_attr = cnvlink_pmu_format_attrs + .template_ctx = { + .event_attr = mcf_pmu_event_attrs, + .format_attr = cnvlink_pmu_format_attrs, + .filter_mask = NV_CNVL_FILTER_ID_MASK, + .filter_default_val = NV_CNVL_FILTER_ID_MASK, + .filter2_mask = 0x0, + .filter2_default_val = 0x0, + .get_filter = nv_cspmu_event_filter, + .get_filter2 = NULL, + .data = NULL, + .init_data = NULL + }, }, { - .prodid = 0x2CF, + .prodid = 0x2CF00000, .prodid_mask = NV_PRODID_MASK, - .filter_mask = 0x0, - .filter_default_val = 0x0, .name_pattern = "nvidia_scf_pmu_%u", .name_fmt = NAME_FMT_SOCKET, - .event_attr = scf_pmu_event_attrs, - .format_attr = scf_pmu_format_attrs + .template_ctx = { + .event_attr = scf_pmu_event_attrs, + .format_attr = scf_pmu_format_attrs, + .filter_mask = 0x0, + .filter_default_val = 0x0, + .filter2_mask = 0x0, + .filter2_default_val = 0x0, + .get_filter = nv_cspmu_event_filter, + .get_filter2 = NULL, + .data = NULL, + .init_data = NULL + }, }, { .prodid = 0, .prodid_mask = 0, - .filter_mask = NV_GENERIC_FILTER_ID_MASK, - .filter_default_val = NV_GENERIC_FILTER_ID_MASK, .name_pattern = "nvidia_uncore_pmu_%u", .name_fmt = NAME_FMT_GENERIC, - .event_attr = generic_pmu_event_attrs, - .format_attr = generic_pmu_format_attrs + .template_ctx = { + .event_attr = generic_pmu_event_attrs, + .format_attr = generic_pmu_format_attrs, + .filter_mask = NV_GENERIC_FILTER_ID_MASK, + .filter_default_val = NV_GENERIC_FILTER_ID_MASK, + .filter2_mask = NV_GENERIC_FILTER_ID_MASK, + .filter2_default_val = NV_GENERIC_FILTER_ID_MASK, + .get_filter = nv_cspmu_event_filter, + .get_filter2 = nv_cspmu_event_filter2, + .data = NULL, + .init_data = NULL + }, }, }; @@ -310,9 +391,16 @@ static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu, return name; } +#define SET_OP(name, impl, match, default_op) \ + do { \ + if (match->ops.name) \ + impl->name = match->ops.name; \ + else if (default_op != NULL) \ + impl->name = default_op; \ + } while (false) + static int nv_cspmu_init_ops(struct arm_cspmu *cspmu) { - u32 prodid; struct nv_cspmu_ctx *ctx; struct device *dev = cspmu->dev; struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops; @@ -322,30 +410,30 @@ static int nv_cspmu_init_ops(struct arm_cspmu *cspmu) if (!ctx) return -ENOMEM; - prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr); - /* Find matching PMU. */ for (; match->prodid; match++) { const u32 prodid_mask = match->prodid_mask; - if ((match->prodid & prodid_mask) == (prodid & prodid_mask)) + if ((match->prodid & prodid_mask) == + (cspmu->impl.pmiidr & prodid_mask)) break; } - ctx->name = nv_cspmu_format_name(cspmu, match); - ctx->filter_mask = match->filter_mask; - ctx->filter_default_val = match->filter_default_val; - ctx->event_attr = match->event_attr; - ctx->format_attr = match->format_attr; + /* Initialize the context with the matched template. */ + memcpy(ctx, &match->template_ctx, sizeof(struct nv_cspmu_ctx)); + ctx->name = nv_cspmu_format_name(cspmu, match); cspmu->impl.ctx = ctx; /* NVIDIA specific callbacks. */ - impl_ops->set_cc_filter = nv_cspmu_set_cc_filter; - impl_ops->set_ev_filter = nv_cspmu_set_ev_filter; - impl_ops->get_event_attrs = nv_cspmu_get_event_attrs; - impl_ops->get_format_attrs = nv_cspmu_get_format_attrs; - impl_ops->get_name = nv_cspmu_get_name; + SET_OP(set_cc_filter, impl_ops, match, nv_cspmu_set_cc_filter); + SET_OP(set_ev_filter, impl_ops, match, nv_cspmu_set_ev_filter); + SET_OP(get_event_attrs, impl_ops, match, nv_cspmu_get_event_attrs); + SET_OP(get_format_attrs, impl_ops, match, nv_cspmu_get_format_attrs); + SET_OP(get_name, impl_ops, match, nv_cspmu_get_name); + + if (ctx->init_data) + return ctx->init_data(cspmu); return 0; } diff --git a/drivers/perf/arm_pmu.c b/drivers/perf/arm_pmu.c index 5c310e803dd7..973a027d9063 100644 --- a/drivers/perf/arm_pmu.c +++ b/drivers/perf/arm_pmu.c @@ -26,7 +26,8 @@ #include <asm/irq_regs.h> -static int armpmu_count_irq_users(const int irq); +static int armpmu_count_irq_users(const struct cpumask *affinity, + const int irq); struct pmu_irq_ops { void (*enable_pmuirq)(unsigned int irq); @@ -64,7 +65,9 @@ static void armpmu_enable_percpu_pmuirq(unsigned int irq) static void armpmu_free_percpu_pmuirq(unsigned int irq, int cpu, void __percpu *devid) { - if (armpmu_count_irq_users(irq) == 1) + struct arm_pmu *armpmu = *per_cpu_ptr((void * __percpu *)devid, cpu); + + if (armpmu_count_irq_users(&armpmu->supported_cpus, irq) == 1) free_percpu_irq(irq, devid); } @@ -89,7 +92,9 @@ static void armpmu_disable_percpu_pmunmi(unsigned int irq) static void armpmu_free_percpu_pmunmi(unsigned int irq, int cpu, void __percpu *devid) { - if (armpmu_count_irq_users(irq) == 1) + struct arm_pmu *armpmu = *per_cpu_ptr((void * __percpu *)devid, cpu); + + if (armpmu_count_irq_users(&armpmu->supported_cpus, irq) == 1) free_percpu_nmi(irq, devid); } @@ -99,7 +104,6 @@ static const struct pmu_irq_ops percpu_pmunmi_ops = { .free_pmuirq = armpmu_free_percpu_pmunmi }; -DEFINE_PER_CPU(struct arm_pmu *, cpu_armpmu); static DEFINE_PER_CPU(int, cpu_irq); static DEFINE_PER_CPU(const struct pmu_irq_ops *, cpu_irq_ops); @@ -580,11 +584,11 @@ static const struct attribute_group armpmu_common_attr_group = { .attrs = armpmu_common_attrs, }; -static int armpmu_count_irq_users(const int irq) +static int armpmu_count_irq_users(const struct cpumask *affinity, const int irq) { int cpu, count = 0; - for_each_possible_cpu(cpu) { + for_each_cpu(cpu, affinity) { if (per_cpu(cpu_irq, cpu) == irq) count++; } @@ -592,12 +596,13 @@ static int armpmu_count_irq_users(const int irq) return count; } -static const struct pmu_irq_ops *armpmu_find_irq_ops(int irq) +static const struct pmu_irq_ops * +armpmu_find_irq_ops(const struct cpumask *affinity, int irq) { const struct pmu_irq_ops *ops = NULL; int cpu; - for_each_possible_cpu(cpu) { + for_each_cpu(cpu, affinity) { if (per_cpu(cpu_irq, cpu) != irq) continue; @@ -609,22 +614,25 @@ static const struct pmu_irq_ops *armpmu_find_irq_ops(int irq) return ops; } -void armpmu_free_irq(int irq, int cpu) +void armpmu_free_irq(struct arm_pmu * __percpu *armpmu, int irq, int cpu) { if (per_cpu(cpu_irq, cpu) == 0) return; if (WARN_ON(irq != per_cpu(cpu_irq, cpu))) return; - per_cpu(cpu_irq_ops, cpu)->free_pmuirq(irq, cpu, &cpu_armpmu); + per_cpu(cpu_irq_ops, cpu)->free_pmuirq(irq, cpu, armpmu); per_cpu(cpu_irq, cpu) = 0; per_cpu(cpu_irq_ops, cpu) = NULL; } -int armpmu_request_irq(int irq, int cpu) +int armpmu_request_irq(struct arm_pmu * __percpu *pcpu_armpmu, int irq, int cpu) { int err = 0; + struct arm_pmu **armpmu = per_cpu_ptr(pcpu_armpmu, cpu); + const struct cpumask *affinity = *armpmu ? &(*armpmu)->supported_cpus : + cpu_possible_mask; /* ACPI */ const irq_handler_t handler = armpmu_dispatch_irq; const struct pmu_irq_ops *irq_ops; @@ -646,25 +654,24 @@ int armpmu_request_irq(int irq, int cpu) IRQF_NOBALANCING | IRQF_NO_AUTOEN | IRQF_NO_THREAD; - err = request_nmi(irq, handler, irq_flags, "arm-pmu", - per_cpu_ptr(&cpu_armpmu, cpu)); + err = request_nmi(irq, handler, irq_flags, "arm-pmu", armpmu); /* If cannot get an NMI, get a normal interrupt */ if (err) { err = request_irq(irq, handler, irq_flags, "arm-pmu", - per_cpu_ptr(&cpu_armpmu, cpu)); + armpmu); irq_ops = &pmuirq_ops; } else { has_nmi = true; irq_ops = &pmunmi_ops; } - } else if (armpmu_count_irq_users(irq) == 0) { - err = request_percpu_nmi(irq, handler, "arm-pmu", &cpu_armpmu); + } else if (armpmu_count_irq_users(affinity, irq) == 0) { + err = request_percpu_nmi(irq, handler, "arm-pmu", affinity, pcpu_armpmu); /* If cannot get an NMI, get a normal interrupt */ if (err) { - err = request_percpu_irq(irq, handler, "arm-pmu", - &cpu_armpmu); + err = request_percpu_irq_affinity(irq, handler, "arm-pmu", + affinity, pcpu_armpmu); irq_ops = &percpu_pmuirq_ops; } else { has_nmi = true; @@ -672,7 +679,7 @@ int armpmu_request_irq(int irq, int cpu) } } else { /* Per cpudevid irq was already requested by another CPU */ - irq_ops = armpmu_find_irq_ops(irq); + irq_ops = armpmu_find_irq_ops(affinity, irq); if (WARN_ON(!irq_ops)) err = -EINVAL; @@ -717,8 +724,6 @@ static int arm_perf_starting_cpu(unsigned int cpu, struct hlist_node *node) if (pmu->reset) pmu->reset(pmu); - per_cpu(cpu_armpmu, cpu) = pmu; - irq = armpmu_get_cpu_irq(pmu, cpu); if (irq) per_cpu(cpu_irq_ops, cpu)->enable_pmuirq(irq); @@ -738,8 +743,6 @@ static int arm_perf_teardown_cpu(unsigned int cpu, struct hlist_node *node) if (irq) per_cpu(cpu_irq_ops, cpu)->disable_pmuirq(irq); - per_cpu(cpu_armpmu, cpu) = NULL; - return 0; } @@ -925,6 +928,12 @@ int armpmu_register(struct arm_pmu *pmu) if (ret) return ret; + /* + * By this stage we know our supported CPUs on either DT/ACPI platforms, + * detect the SMT implementation. + */ + pmu->has_smt = topology_core_has_smt(cpumask_first(&pmu->supported_cpus)); + if (!pmu->set_event_filter) pmu->pmu.capabilities |= PERF_PMU_CAP_NO_EXCLUDE; diff --git a/drivers/perf/arm_pmu_acpi.c b/drivers/perf/arm_pmu_acpi.c index 05dda19c5359..e80f76d95e68 100644 --- a/drivers/perf/arm_pmu_acpi.c +++ b/drivers/perf/arm_pmu_acpi.c @@ -218,7 +218,7 @@ static int arm_pmu_acpi_parse_irqs(void) * them with their PMUs. */ per_cpu(pmu_irqs, cpu) = irq; - err = armpmu_request_irq(irq, cpu); + err = armpmu_request_irq(&probed_pmus, irq, cpu); if (err) goto out_err; } diff --git a/drivers/perf/arm_pmu_platform.c b/drivers/perf/arm_pmu_platform.c index 118170a5cede..1c9e50a13201 100644 --- a/drivers/perf/arm_pmu_platform.c +++ b/drivers/perf/arm_pmu_platform.c @@ -42,14 +42,13 @@ static int probe_current_pmu(struct arm_pmu *pmu, return ret; } -static int pmu_parse_percpu_irq(struct arm_pmu *pmu, int irq) +static int pmu_parse_percpu_irq(struct arm_pmu *pmu, int irq, + const struct cpumask *affinity) { - int cpu, ret; struct pmu_hw_events __percpu *hw_events = pmu->hw_events; + int cpu; - ret = irq_get_percpu_devid_partition(irq, &pmu->supported_cpus); - if (ret) - return ret; + cpumask_copy(&pmu->supported_cpus, affinity); for_each_cpu(cpu, &pmu->supported_cpus) per_cpu(hw_events->irq, cpu) = irq; @@ -115,9 +114,12 @@ static int pmu_parse_irqs(struct arm_pmu *pmu) } if (num_irqs == 1) { - int irq = platform_get_irq(pdev, 0); + const struct cpumask *affinity; + int irq; + + irq = platform_get_irq_affinity(pdev, 0, &affinity); if ((irq > 0) && irq_is_percpu_devid(irq)) - return pmu_parse_percpu_irq(pmu, irq); + return pmu_parse_percpu_irq(pmu, irq, affinity); } if (nr_cpu_ids != 1 && !pmu_has_irq_affinity(dev->of_node)) @@ -163,7 +165,7 @@ static int armpmu_request_irqs(struct arm_pmu *armpmu) if (!irq) continue; - err = armpmu_request_irq(irq, cpu); + err = armpmu_request_irq(&hw_events->percpu_pmu, irq, cpu); if (err) break; } @@ -179,7 +181,7 @@ static void armpmu_free_irqs(struct arm_pmu *armpmu) for_each_cpu(cpu, &armpmu->supported_cpus) { int irq = per_cpu(hw_events->irq, cpu); - armpmu_free_irq(irq, cpu); + armpmu_free_irq(&hw_events->percpu_pmu, irq, cpu); } } diff --git a/drivers/perf/arm_pmuv3.c b/drivers/perf/arm_pmuv3.c index 69c5cc8f5606..8014ff766cff 100644 --- a/drivers/perf/arm_pmuv3.c +++ b/drivers/perf/arm_pmuv3.c @@ -981,6 +981,7 @@ static int armv8pmu_get_chain_idx(struct pmu_hw_events *cpuc, static bool armv8pmu_can_use_pmccntr(struct pmu_hw_events *cpuc, struct perf_event *event) { + struct arm_pmu *cpu_pmu = to_arm_pmu(event->pmu); struct hw_perf_event *hwc = &event->hw; unsigned long evtype = hwc->config_base & ARMV8_PMU_EVTYPE_EVENT; @@ -1001,6 +1002,15 @@ static bool armv8pmu_can_use_pmccntr(struct pmu_hw_events *cpuc, if (has_branch_stack(event)) return false; + /* + * The PMCCNTR_EL0 increments from the processor clock rather than + * the PE clock (ARM DDI0487 L.b D13.1.3) which means it'll continue + * counting on a WFI PE if one of its SMT sibling is not idle on a + * multi-threaded implementation. So don't use it on SMT cores. + */ + if (cpu_pmu->has_smt) + return false; + return true; } @@ -1064,7 +1074,7 @@ static int armv8pmu_user_event_idx(struct perf_event *event) static void armv8pmu_sched_task(struct perf_event_pmu_context *pmu_ctx, struct task_struct *task, bool sched_in) { - struct arm_pmu *armpmu = *this_cpu_ptr(&cpu_armpmu); + struct arm_pmu *armpmu = to_arm_pmu(pmu_ctx->pmu); struct pmu_hw_events *hw_events = this_cpu_ptr(armpmu->hw_events); if (!hw_events->branch_users) @@ -1465,6 +1475,10 @@ static int name##_pmu_init(struct arm_pmu *cpu_pmu) \ PMUV3_INIT_SIMPLE(armv8_pmuv3) +PMUV3_INIT_SIMPLE(armv8_c1_nano) +PMUV3_INIT_SIMPLE(armv8_c1_premium) +PMUV3_INIT_SIMPLE(armv8_c1_pro) +PMUV3_INIT_SIMPLE(armv8_c1_ultra) PMUV3_INIT_SIMPLE(armv8_cortex_a34) PMUV3_INIT_SIMPLE(armv8_cortex_a55) PMUV3_INIT_SIMPLE(armv8_cortex_a65) @@ -1472,11 +1486,14 @@ PMUV3_INIT_SIMPLE(armv8_cortex_a75) PMUV3_INIT_SIMPLE(armv8_cortex_a76) PMUV3_INIT_SIMPLE(armv8_cortex_a77) PMUV3_INIT_SIMPLE(armv8_cortex_a78) +PMUV3_INIT_SIMPLE(armv9_cortex_a320) PMUV3_INIT_SIMPLE(armv9_cortex_a510) PMUV3_INIT_SIMPLE(armv9_cortex_a520) +PMUV3_INIT_SIMPLE(armv9_cortex_a520ae) PMUV3_INIT_SIMPLE(armv9_cortex_a710) PMUV3_INIT_SIMPLE(armv9_cortex_a715) PMUV3_INIT_SIMPLE(armv9_cortex_a720) +PMUV3_INIT_SIMPLE(armv9_cortex_a720ae) PMUV3_INIT_SIMPLE(armv9_cortex_a725) PMUV3_INIT_SIMPLE(armv8_cortex_x1) PMUV3_INIT_SIMPLE(armv9_cortex_x2) @@ -1508,6 +1525,10 @@ PMUV3_INIT_MAP_EVENT(armv8_brcm_vulcan, armv8_vulcan_map_event) static const struct of_device_id armv8_pmu_of_device_ids[] = { {.compatible = "arm,armv8-pmuv3", .data = armv8_pmuv3_pmu_init}, + {.compatible = "arm,c1-nano-pmu", .data = armv8_c1_nano_pmu_init}, + {.compatible = "arm,c1-premium-pmu", .data = armv8_c1_premium_pmu_init}, + {.compatible = "arm,c1-pro-pmu", .data = armv8_c1_pro_pmu_init}, + {.compatible = "arm,c1-ultra-pmu", .data = armv8_c1_ultra_pmu_init}, {.compatible = "arm,cortex-a34-pmu", .data = armv8_cortex_a34_pmu_init}, {.compatible = "arm,cortex-a35-pmu", .data = armv8_cortex_a35_pmu_init}, {.compatible = "arm,cortex-a53-pmu", .data = armv8_cortex_a53_pmu_init}, @@ -1520,11 +1541,14 @@ static const struct of_device_id armv8_pmu_of_device_ids[] = { {.compatible = "arm,cortex-a76-pmu", .data = armv8_cortex_a76_pmu_init}, {.compatible = "arm,cortex-a77-pmu", .data = armv8_cortex_a77_pmu_init}, {.compatible = "arm,cortex-a78-pmu", .data = armv8_cortex_a78_pmu_init}, + {.compatible = "arm,cortex-a320-pmu", .data = armv9_cortex_a320_pmu_init}, {.compatible = "arm,cortex-a510-pmu", .data = armv9_cortex_a510_pmu_init}, {.compatible = "arm,cortex-a520-pmu", .data = armv9_cortex_a520_pmu_init}, + {.compatible = "arm,cortex-a520ae-pmu", .data = armv9_cortex_a520ae_pmu_init}, {.compatible = "arm,cortex-a710-pmu", .data = armv9_cortex_a710_pmu_init}, {.compatible = "arm,cortex-a715-pmu", .data = armv9_cortex_a715_pmu_init}, {.compatible = "arm,cortex-a720-pmu", .data = armv9_cortex_a720_pmu_init}, + {.compatible = "arm,cortex-a720ae-pmu", .data = armv9_cortex_a720ae_pmu_init}, {.compatible = "arm,cortex-a725-pmu", .data = armv9_cortex_a725_pmu_init}, {.compatible = "arm,cortex-x1-pmu", .data = armv8_cortex_x1_pmu_init}, {.compatible = "arm,cortex-x2-pmu", .data = armv9_cortex_x2_pmu_init}, diff --git a/drivers/perf/arm_spe_pmu.c b/drivers/perf/arm_spe_pmu.c index fa50645fedda..4801115f2b54 100644 --- a/drivers/perf/arm_spe_pmu.c +++ b/drivers/perf/arm_spe_pmu.c @@ -87,6 +87,7 @@ struct arm_spe_pmu { #define SPE_PMU_FEAT_INV_FILT_EVT (1UL << 6) #define SPE_PMU_FEAT_DISCARD (1UL << 7) #define SPE_PMU_FEAT_EFT (1UL << 8) +#define SPE_PMU_FEAT_FDS (1UL << 9) #define SPE_PMU_FEAT_DEV_PROBED (1UL << 63) u64 features; @@ -252,6 +253,10 @@ static const struct attribute_group arm_spe_pmu_cap_group = { #define ATTR_CFG_FLD_inv_event_filter_LO 0 #define ATTR_CFG_FLD_inv_event_filter_HI 63 +#define ATTR_CFG_FLD_inv_data_src_filter_CFG config4 /* inverse of PMSDSFR_EL1 */ +#define ATTR_CFG_FLD_inv_data_src_filter_LO 0 +#define ATTR_CFG_FLD_inv_data_src_filter_HI 63 + GEN_PMU_FORMAT_ATTR(ts_enable); GEN_PMU_FORMAT_ATTR(pa_enable); GEN_PMU_FORMAT_ATTR(pct_enable); @@ -268,6 +273,7 @@ GEN_PMU_FORMAT_ATTR(float_filter); GEN_PMU_FORMAT_ATTR(float_filter_mask); GEN_PMU_FORMAT_ATTR(event_filter); GEN_PMU_FORMAT_ATTR(inv_event_filter); +GEN_PMU_FORMAT_ATTR(inv_data_src_filter); GEN_PMU_FORMAT_ATTR(min_latency); GEN_PMU_FORMAT_ATTR(discard); @@ -288,6 +294,7 @@ static struct attribute *arm_spe_pmu_formats_attr[] = { &format_attr_float_filter_mask.attr, &format_attr_event_filter.attr, &format_attr_inv_event_filter.attr, + &format_attr_inv_data_src_filter.attr, &format_attr_min_latency.attr, &format_attr_discard.attr, NULL, @@ -306,6 +313,10 @@ static umode_t arm_spe_pmu_format_attr_is_visible(struct kobject *kobj, if (attr == &format_attr_inv_event_filter.attr && !(spe_pmu->features & SPE_PMU_FEAT_INV_FILT_EVT)) return 0; + if (attr == &format_attr_inv_data_src_filter.attr && + !(spe_pmu->features & SPE_PMU_FEAT_FDS)) + return 0; + if ((attr == &format_attr_branch_filter_mask.attr || attr == &format_attr_load_filter_mask.attr || attr == &format_attr_store_filter_mask.attr || @@ -430,6 +441,9 @@ static u64 arm_spe_event_to_pmsfcr(struct perf_event *event) if (ATTR_CFG_GET_FLD(attr, inv_event_filter)) reg |= PMSFCR_EL1_FnE; + if (ATTR_CFG_GET_FLD(attr, inv_data_src_filter)) + reg |= PMSFCR_EL1_FDS; + if (ATTR_CFG_GET_FLD(attr, min_latency)) reg |= PMSFCR_EL1_FL; @@ -454,6 +468,17 @@ static u64 arm_spe_event_to_pmslatfr(struct perf_event *event) return FIELD_PREP(PMSLATFR_EL1_MINLAT, ATTR_CFG_GET_FLD(attr, min_latency)); } +static u64 arm_spe_event_to_pmsdsfr(struct perf_event *event) +{ + struct perf_event_attr *attr = &event->attr; + + /* + * Data src filter is inverted so that the default value of 0 is + * equivalent to no filtering. + */ + return ~ATTR_CFG_GET_FLD(attr, inv_data_src_filter); +} + static void arm_spe_pmu_pad_buf(struct perf_output_handle *handle, int len) { struct arm_spe_pmu_buf *buf = perf_get_aux(handle); @@ -791,6 +816,10 @@ static int arm_spe_pmu_event_init(struct perf_event *event) if (arm_spe_event_to_pmsnevfr(event) & spe_pmu->pmsevfr_res0) return -EOPNOTSUPP; + if (arm_spe_event_to_pmsdsfr(event) != U64_MAX && + !(spe_pmu->features & SPE_PMU_FEAT_FDS)) + return -EOPNOTSUPP; + if (attr->exclude_idle) return -EOPNOTSUPP; @@ -866,6 +895,11 @@ static void arm_spe_pmu_start(struct perf_event *event, int flags) write_sysreg_s(reg, SYS_PMSNEVFR_EL1); } + if (spe_pmu->features & SPE_PMU_FEAT_FDS) { + reg = arm_spe_event_to_pmsdsfr(event); + write_sysreg_s(reg, SYS_PMSDSFR_EL1); + } + reg = arm_spe_event_to_pmslatfr(event); write_sysreg_s(reg, SYS_PMSLATFR_EL1); @@ -1125,6 +1159,9 @@ static void __arm_spe_pmu_dev_probe(void *info) if (FIELD_GET(PMSIDR_EL1_EFT, reg)) spe_pmu->features |= SPE_PMU_FEAT_EFT; + if (FIELD_GET(PMSIDR_EL1_FDS, reg)) + spe_pmu->features |= SPE_PMU_FEAT_FDS; + /* This field has a spaced out encoding, so just use a look-up */ fld = FIELD_GET(PMSIDR_EL1_INTERVAL, reg); switch (fld) { @@ -1259,8 +1296,8 @@ static int arm_spe_pmu_dev_init(struct arm_spe_pmu *spe_pmu) return -ENXIO; /* Request our PPIs (note that the IRQ is still disabled) */ - ret = request_percpu_irq(spe_pmu->irq, arm_spe_pmu_irq_handler, DRVNAME, - spe_pmu->handle); + ret = request_percpu_irq_affinity(spe_pmu->irq, arm_spe_pmu_irq_handler, + DRVNAME, mask, spe_pmu->handle); if (ret) return ret; @@ -1287,8 +1324,10 @@ static void arm_spe_pmu_dev_teardown(struct arm_spe_pmu *spe_pmu) static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu) { struct platform_device *pdev = spe_pmu->pdev; - int irq = platform_get_irq(pdev, 0); + const struct cpumask *affinity; + int irq; + irq = platform_get_irq_affinity(pdev, 0, &affinity); if (irq < 0) return -ENXIO; @@ -1297,10 +1336,7 @@ static int arm_spe_pmu_irq_probe(struct arm_spe_pmu *spe_pmu) return -EINVAL; } - if (irq_get_percpu_devid_partition(irq, &spe_pmu->supported_cpus)) { - dev_err(&pdev->dev, "failed to get PPI partition (%d)\n", irq); - return -EINVAL; - } + cpumask_copy(&spe_pmu->supported_cpus, affinity); spe_pmu->irq = irq; return 0; diff --git a/drivers/perf/fsl_imx8_ddr_perf.c b/drivers/perf/fsl_imx8_ddr_perf.c index b989ffa95d69..bcdf5575d71c 100644 --- a/drivers/perf/fsl_imx8_ddr_perf.c +++ b/drivers/perf/fsl_imx8_ddr_perf.c @@ -5,6 +5,7 @@ */ #include <linux/bitfield.h> +#include <linux/clk.h> #include <linux/init.h> #include <linux/interrupt.h> #include <linux/io.h> @@ -52,18 +53,27 @@ #define to_ddr_pmu(p) container_of(p, struct ddr_pmu, pmu) #define DDR_PERF_DEV_NAME "imx8_ddr" +#define DB_PERF_DEV_NAME "imx8_db" #define DDR_CPUHP_CB_NAME DDR_PERF_DEV_NAME "_perf_pmu" static DEFINE_IDA(ddr_ida); +static DEFINE_IDA(db_ida); /* DDR Perf hardware feature */ #define DDR_CAP_AXI_ID_FILTER 0x1 /* support AXI ID filter */ #define DDR_CAP_AXI_ID_FILTER_ENHANCED 0x3 /* support enhanced AXI ID filter */ #define DDR_CAP_AXI_ID_PORT_CHANNEL_FILTER 0x4 /* support AXI ID PORT CHANNEL filter */ +/* Perf type */ +enum fsl_ddr_type { + DDR_PERF_TYPE = 0, /* ddr Perf (default) */ + DB_PERF_TYPE, /* db Perf */ +}; + struct fsl_ddr_devtype_data { unsigned int quirks; /* quirks needed for different DDR Perf core */ const char *identifier; /* system PMU identifier for userspace */ + enum fsl_ddr_type type; /* types of Perf, ddr or db */ }; static const struct fsl_ddr_devtype_data imx8_devtype_data; @@ -97,6 +107,12 @@ static const struct fsl_ddr_devtype_data imx8dxl_devtype_data = { .identifier = "i.MX8DXL", }; +static const struct fsl_ddr_devtype_data imx8dxl_db_devtype_data = { + .quirks = DDR_CAP_AXI_ID_PORT_CHANNEL_FILTER, + .identifier = "i.MX8DXL", + .type = DB_PERF_TYPE, +}; + static const struct of_device_id imx_ddr_pmu_dt_ids[] = { { .compatible = "fsl,imx8-ddr-pmu", .data = &imx8_devtype_data}, { .compatible = "fsl,imx8m-ddr-pmu", .data = &imx8m_devtype_data}, @@ -105,6 +121,7 @@ static const struct of_device_id imx_ddr_pmu_dt_ids[] = { { .compatible = "fsl,imx8mn-ddr-pmu", .data = &imx8mn_devtype_data}, { .compatible = "fsl,imx8mp-ddr-pmu", .data = &imx8mp_devtype_data}, { .compatible = "fsl,imx8dxl-ddr-pmu", .data = &imx8dxl_devtype_data}, + { .compatible = "fsl,imx8dxl-db-pmu", .data = &imx8dxl_db_devtype_data}, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, imx_ddr_pmu_dt_ids); @@ -284,9 +301,37 @@ static struct attribute *ddr_perf_events_attrs[] = { NULL, }; +static const int ddr_perf_db_visible_event_list[] = { + EVENT_CYCLES_ID, + 0x41, + 0x42, +}; + +static umode_t ddr_perf_events_attrs_is_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct ddr_pmu *pmu = dev_get_drvdata(dev); + struct perf_pmu_events_attr *pmu_attr; + unsigned int i; + + pmu_attr = container_of(attr, struct perf_pmu_events_attr, attr.attr); + + if (pmu->devtype_data->type == DDR_PERF_TYPE) + return attr->mode; + + /* DB Type */ + for (i = 0; i < ARRAY_SIZE(ddr_perf_db_visible_event_list); i++) + if (pmu_attr->id == ddr_perf_db_visible_event_list[i]) + return attr->mode; + + return 0; +} + static const struct attribute_group ddr_perf_events_attr_group = { .name = "events", .attrs = ddr_perf_events_attrs, + .is_visible = ddr_perf_events_attrs_is_visible, }; PMU_FORMAT_ATTR(event, "config:0-7"); @@ -645,8 +690,8 @@ static void ddr_perf_pmu_disable(struct pmu *pmu) { } -static int ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base, - struct device *dev) +static void ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base, + struct device *dev) { *pmu = (struct ddr_pmu) { .pmu = (struct pmu) { @@ -667,9 +712,6 @@ static int ddr_perf_init(struct ddr_pmu *pmu, void __iomem *base, .base = base, .dev = dev, }; - - pmu->id = ida_alloc(&ddr_ida, GFP_KERNEL); - return pmu->id; } static irqreturn_t ddr_perf_irq_handler(int irq, void *p) @@ -735,10 +777,13 @@ static int ddr_perf_offline_cpu(unsigned int cpu, struct hlist_node *node) static int ddr_perf_probe(struct platform_device *pdev) { + struct clk_bulk_data *clks; struct ddr_pmu *pmu; struct device_node *np; void __iomem *base; + struct ida *ida; char *name; + int nclks; int num; int ret; int irq; @@ -753,19 +798,33 @@ static int ddr_perf_probe(struct platform_device *pdev) if (!pmu) return -ENOMEM; - num = ddr_perf_init(pmu, base, &pdev->dev); + ddr_perf_init(pmu, base, &pdev->dev); platform_set_drvdata(pdev, pmu); - name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", - num); + nclks = devm_clk_bulk_get_all_enabled(&pdev->dev, &clks); + if (nclks < 0) + return dev_err_probe(&pdev->dev, nclks, "Failure get clks\n"); + + pmu->devtype_data = of_device_get_match_data(&pdev->dev); + + ida = pmu->devtype_data->type == DDR_PERF_TYPE ? &ddr_ida : &db_ida; + num = ida_alloc(ida, GFP_KERNEL); + if (num < 0) + return num; + + pmu->id = num; + + if (pmu->devtype_data->type == DDR_PERF_TYPE) + name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DDR_PERF_DEV_NAME "%d", num); + else + name = devm_kasprintf(&pdev->dev, GFP_KERNEL, DB_PERF_DEV_NAME "%d", num); + if (!name) { ret = -ENOMEM; - goto cpuhp_state_err; + goto idr_free; } - pmu->devtype_data = of_device_get_match_data(&pdev->dev); - pmu->cpu = raw_smp_processor_id(); ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, DDR_CPUHP_CB_NAME, @@ -774,7 +833,7 @@ static int ddr_perf_probe(struct platform_device *pdev) if (ret < 0) { dev_err(&pdev->dev, "cpuhp_setup_state_multi failed\n"); - goto cpuhp_state_err; + goto idr_free; } pmu->cpuhp_state = ret; @@ -821,8 +880,8 @@ ddr_perf_err: cpuhp_state_remove_instance_nocalls(pmu->cpuhp_state, &pmu->node); cpuhp_instance_err: cpuhp_remove_multi_state(pmu->cpuhp_state); -cpuhp_state_err: - ida_free(&ddr_ida, pmu->id); +idr_free: + ida_free(ida, pmu->id); dev_warn(&pdev->dev, "i.MX8 DDR Perf PMU failed (%d), disabled\n", ret); return ret; } @@ -836,7 +895,11 @@ static void ddr_perf_remove(struct platform_device *pdev) perf_pmu_unregister(&pmu->pmu); - ida_free(&ddr_ida, pmu->id); + if (pmu->devtype_data->type == DDR_PERF_TYPE) + ida_free(&ddr_ida, pmu->id); + else + ida_free(&db_ida, pmu->id); + } static struct platform_driver imx_ddr_pmu_driver = { diff --git a/drivers/pmdomain/core.c b/drivers/pmdomain/core.c index 61c2277c9ce3..4fd546ef0448 100644 --- a/drivers/pmdomain/core.c +++ b/drivers/pmdomain/core.c @@ -1425,8 +1425,14 @@ static void genpd_sync_power_off(struct generic_pm_domain *genpd, bool use_lock, return; } - /* Choose the deepest state when suspending */ - genpd->state_idx = genpd->state_count - 1; + if (genpd->gov && genpd->gov->system_power_down_ok) { + if (!genpd->gov->system_power_down_ok(&genpd->domain)) + return; + } else { + /* Default to the deepest state. */ + genpd->state_idx = genpd->state_count - 1; + } + if (_genpd_power_off(genpd, false)) { genpd->states[genpd->state_idx].rejected++; return; diff --git a/drivers/pmdomain/governor.c b/drivers/pmdomain/governor.c index 39359811a930..05e68680f34b 100644 --- a/drivers/pmdomain/governor.c +++ b/drivers/pmdomain/governor.c @@ -351,7 +351,7 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) ktime_t domain_wakeup, next_hrtimer; ktime_t now = ktime_get(); struct device *cpu_dev; - s64 cpu_constraint, global_constraint; + s64 cpu_constraint, global_constraint, wakeup_constraint; s64 idle_duration_ns; int cpu, i; @@ -362,7 +362,11 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) return true; + wakeup_constraint = cpu_wakeup_latency_qos_limit(); global_constraint = cpu_latency_qos_limit(); + if (global_constraint > wakeup_constraint) + global_constraint = wakeup_constraint; + /* * Find the next wakeup for any of the online CPUs within the PM domain * and its subdomains. Note, we only need the genpd->cpus, as it already @@ -415,9 +419,36 @@ static bool cpu_power_down_ok(struct dev_pm_domain *pd) return false; } +static bool cpu_system_power_down_ok(struct dev_pm_domain *pd) +{ + s64 constraint_ns = cpu_wakeup_latency_qos_limit() * NSEC_PER_USEC; + struct generic_pm_domain *genpd = pd_to_genpd(pd); + int state_idx = genpd->state_count - 1; + + if (!(genpd->flags & GENPD_FLAG_CPU_DOMAIN)) { + genpd->state_idx = state_idx; + return true; + } + + /* Find the deepest state for the latency constraint. */ + while (state_idx >= 0) { + s64 latency_ns = genpd->states[state_idx].power_off_latency_ns + + genpd->states[state_idx].power_on_latency_ns; + + if (latency_ns <= constraint_ns) { + genpd->state_idx = state_idx; + return true; + } + state_idx--; + } + + return false; +} + struct dev_power_governor pm_domain_cpu_gov = { .suspend_ok = default_suspend_ok, .power_down_ok = cpu_power_down_ok, + .system_power_down_ok = cpu_system_power_down_ok, }; #endif diff --git a/drivers/pmdomain/mediatek/mtk-pm-domains.c b/drivers/pmdomain/mediatek/mtk-pm-domains.c index 0ebe7379b94e..9c9323c8c93a 100644 --- a/drivers/pmdomain/mediatek/mtk-pm-domains.c +++ b/drivers/pmdomain/mediatek/mtk-pm-domains.c @@ -748,6 +748,18 @@ static void scpsys_domain_cleanup(struct scpsys *scpsys) } } +static struct device_node *scpsys_get_legacy_regmap(struct device_node *np, const char *pn) +{ + struct device_node *local_node; + + for_each_child_of_node(np, local_node) { + if (of_property_present(local_node, pn)) + return local_node; + } + + return NULL; +} + static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *scpsys) { const u8 bp_blocks[3] = { @@ -769,7 +781,7 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s * this makes it then possible to allocate the array of bus_prot * regmaps and convert all to the new style handling. */ - node = of_find_node_with_property(np, "mediatek,infracfg"); + node = scpsys_get_legacy_regmap(np, "mediatek,infracfg"); if (node) { regmap[0] = syscon_regmap_lookup_by_phandle(node, "mediatek,infracfg"); of_node_put(node); @@ -782,7 +794,7 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s regmap[0] = NULL; } - node = of_find_node_with_property(np, "mediatek,smi"); + node = scpsys_get_legacy_regmap(np, "mediatek,smi"); if (node) { smi_np = of_parse_phandle(node, "mediatek,smi", 0); of_node_put(node); @@ -800,7 +812,7 @@ static int scpsys_get_bus_protection_legacy(struct device *dev, struct scpsys *s regmap[1] = NULL; } - node = of_find_node_with_property(np, "mediatek,infracfg-nao"); + node = scpsys_get_legacy_regmap(np, "mediatek,infracfg-nao"); if (node) { regmap[2] = syscon_regmap_lookup_by_phandle(node, "mediatek,infracfg-nao"); num_regmaps++; diff --git a/drivers/pmdomain/tegra/powergate-bpmp.c b/drivers/pmdomain/tegra/powergate-bpmp.c index b0138ca9f851..9f4366250bfd 100644 --- a/drivers/pmdomain/tegra/powergate-bpmp.c +++ b/drivers/pmdomain/tegra/powergate-bpmp.c @@ -184,6 +184,7 @@ tegra_powergate_add(struct tegra_bpmp *bpmp, powergate->genpd.name = kstrdup(info->name, GFP_KERNEL); powergate->genpd.power_on = tegra_powergate_power_on; powergate->genpd.power_off = tegra_powergate_power_off; + powergate->genpd.flags = GENPD_FLAG_NO_STAY_ON; err = pm_genpd_init(&powergate->genpd, NULL, off); if (err < 0) { diff --git a/drivers/pnp/driver.c b/drivers/pnp/driver.c index 7de7aabb275e..05e9840bc3d4 100644 --- a/drivers/pnp/driver.c +++ b/drivers/pnp/driver.c @@ -150,6 +150,24 @@ static void pnp_device_shutdown(struct device *dev) drv->shutdown(pnp_dev); } +static int pnp_uevent(const struct device *dev, struct kobj_uevent_env *env) +{ + struct pnp_id *pos; + const struct pnp_dev *pnp_dev = to_pnp_dev(dev); + + if (!dev) + return -ENODEV; + + pos = pnp_dev->id; + while (pos) { + if (add_uevent_var(env, "MODALIAS=pnp:d%s", pos->id)) + return -ENOMEM; + pos = pos->next; + } + + return 0; +} + static int pnp_bus_match(struct device *dev, const struct device_driver *drv) { struct pnp_dev *pnp_dev = to_pnp_dev(dev); @@ -259,6 +277,7 @@ static const struct dev_pm_ops pnp_bus_dev_pm_ops = { const struct bus_type pnp_bus_type = { .name = "pnp", .match = pnp_bus_match, + .uevent = pnp_uevent, .probe = pnp_device_probe, .remove = pnp_device_remove, .shutdown = pnp_device_shutdown, diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c index c7e7f9bf5313..b9d87e56cbbc 100644 --- a/drivers/powercap/intel_rapl_common.c +++ b/drivers/powercap/intel_rapl_common.c @@ -253,7 +253,8 @@ struct rapl_primitive_info { static void rapl_init_domains(struct rapl_package *rp); static int rapl_read_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, - bool xlate, u64 *data); + bool xlate, u64 *data, + bool atomic); static int rapl_write_data_raw(struct rapl_domain *rd, enum rapl_primitives prim, unsigned long long value); @@ -289,7 +290,7 @@ static int get_energy_counter(struct powercap_zone *power_zone, cpus_read_lock(); rd = power_zone_to_rapl_domain(power_zone); - if (!rapl_read_data_raw(rd, ENERGY_COUNTER, true, &energy_now)) { + if (!rapl_read_data_raw(rd, ENERGY_COUNTER, true, &energy_now, false)) { *energy_raw = energy_now; cpus_read_unlock(); @@ -830,7 +831,8 @@ prim_fixups(struct rapl_domain *rd, enum rapl_primitives prim) * 63-------------------------- 31--------------------------- 0 */ static int rapl_read_data_raw(struct rapl_domain *rd, - enum rapl_primitives prim, bool xlate, u64 *data) + enum rapl_primitives prim, bool xlate, u64 *data, + bool atomic) { u64 value; enum rapl_primitives prim_fixed = prim_fixups(rd, prim); @@ -852,7 +854,7 @@ static int rapl_read_data_raw(struct rapl_domain *rd, ra.mask = rpi->mask; - if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, atomic)) { pr_debug("failed to read reg 0x%llx for %s:%s\n", ra.reg.val, rd->rp->name, rd->name); return -EIO; } @@ -904,7 +906,7 @@ static int rapl_read_pl_data(struct rapl_domain *rd, int pl, if (!is_pl_valid(rd, pl)) return -EINVAL; - return rapl_read_data_raw(rd, prim, xlate, data); + return rapl_read_data_raw(rd, prim, xlate, data, false); } static int rapl_write_pl_data(struct rapl_domain *rd, int pl, @@ -941,7 +943,7 @@ static int rapl_check_unit_core(struct rapl_domain *rd) ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; ra.mask = ~0; - if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, false)) { pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n", ra.reg.val, rd->rp->name, rd->name); return -ENODEV; @@ -969,7 +971,7 @@ static int rapl_check_unit_atom(struct rapl_domain *rd) ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; ra.mask = ~0; - if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, false)) { pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n", ra.reg.val, rd->rp->name, rd->name); return -ENODEV; @@ -1156,7 +1158,7 @@ static int rapl_check_unit_tpmi(struct rapl_domain *rd) ra.reg = rd->regs[RAPL_DOMAIN_REG_UNIT]; ra.mask = ~0; - if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra)) { + if (rd->rp->priv->read_raw(get_rid(rd->rp), &ra, false)) { pr_err("Failed to read power unit REG 0x%llx on %s:%s, exit.\n", ra.reg.val, rd->rp->name, rd->name); return -ENODEV; @@ -1284,6 +1286,9 @@ static const struct x86_cpu_id rapl_ids[] __initconst = { X86_MATCH_VFM(INTEL_EMERALDRAPIDS_X, &rapl_defaults_spr_server), X86_MATCH_VFM(INTEL_LUNARLAKE_M, &rapl_defaults_core), X86_MATCH_VFM(INTEL_PANTHERLAKE_L, &rapl_defaults_core), + X86_MATCH_VFM(INTEL_WILDCATLAKE_L, &rapl_defaults_core), + X86_MATCH_VFM(INTEL_NOVALAKE, &rapl_defaults_core), + X86_MATCH_VFM(INTEL_NOVALAKE_L, &rapl_defaults_core), X86_MATCH_VFM(INTEL_ARROWLAKE_H, &rapl_defaults_core), X86_MATCH_VFM(INTEL_ARROWLAKE, &rapl_defaults_core), X86_MATCH_VFM(INTEL_ARROWLAKE_U, &rapl_defaults_core), @@ -1325,7 +1330,7 @@ static void rapl_update_domain_data(struct rapl_package *rp) struct rapl_primitive_info *rpi = get_rpi(rp, prim); if (!rapl_read_data_raw(&rp->domains[dmn], prim, - rpi->unit, &val)) + rpi->unit, &val, false)) rp->domains[dmn].rdd.primitives[prim] = val; } } @@ -1425,7 +1430,7 @@ static int rapl_check_domain(int domain, struct rapl_package *rp) */ ra.mask = ENERGY_STATUS_MASK; - if (rp->priv->read_raw(get_rid(rp), &ra) || !ra.value) + if (rp->priv->read_raw(get_rid(rp), &ra, false) || !ra.value) return -ENODEV; return 0; @@ -1592,11 +1597,11 @@ static int get_pmu_cpu(struct rapl_package *rp) if (!rp->has_pmu) return nr_cpu_ids; - /* Only TPMI RAPL is supported for now */ - if (rp->priv->type != RAPL_IF_TPMI) + /* Only TPMI & MSR RAPL are supported for now */ + if (rp->priv->type != RAPL_IF_TPMI && rp->priv->type != RAPL_IF_MSR) return nr_cpu_ids; - /* TPMI RAPL uses any CPU in the package for PMU */ + /* TPMI/MSR RAPL uses any CPU in the package for PMU */ for_each_online_cpu(cpu) if (topology_physical_package_id(cpu) == rp->id) return cpu; @@ -1609,11 +1614,11 @@ static bool is_rp_pmu_cpu(struct rapl_package *rp, int cpu) if (!rp->has_pmu) return false; - /* Only TPMI RAPL is supported for now */ - if (rp->priv->type != RAPL_IF_TPMI) + /* Only TPMI & MSR RAPL are supported for now */ + if (rp->priv->type != RAPL_IF_TPMI && rp->priv->type != RAPL_IF_MSR) return false; - /* TPMI RAPL uses any CPU in the package for PMU */ + /* TPMI/MSR RAPL uses any CPU in the package for PMU */ return topology_physical_package_id(cpu) == rp->id; } @@ -1636,7 +1641,7 @@ static u64 event_read_counter(struct perf_event *event) if (event->hw.idx < 0) return 0; - ret = rapl_read_data_raw(&rp->domains[event->hw.idx], ENERGY_COUNTER, false, &val); + ret = rapl_read_data_raw(&rp->domains[event->hw.idx], ENERGY_COUNTER, false, &val, true); /* Return 0 for failed read */ if (ret) diff --git a/drivers/powercap/intel_rapl_msr.c b/drivers/powercap/intel_rapl_msr.c index 4ed06c71a3ac..0ce1096b6314 100644 --- a/drivers/powercap/intel_rapl_msr.c +++ b/drivers/powercap/intel_rapl_msr.c @@ -33,6 +33,8 @@ /* private data for RAPL MSR Interface */ static struct rapl_if_priv *rapl_msr_priv; +static bool rapl_msr_pmu __ro_after_init; + static struct rapl_if_priv rapl_msr_priv_intel = { .type = RAPL_IF_MSR, .reg_unit.msr = MSR_RAPL_POWER_UNIT, @@ -79,6 +81,8 @@ static int rapl_cpu_online(unsigned int cpu) rp = rapl_add_package_cpuslocked(cpu, rapl_msr_priv, true); if (IS_ERR(rp)) return PTR_ERR(rp); + if (rapl_msr_pmu) + rapl_package_add_pmu(rp); } cpumask_set_cpu(cpu, &rp->cpumask); return 0; @@ -95,19 +99,37 @@ static int rapl_cpu_down_prep(unsigned int cpu) cpumask_clear_cpu(cpu, &rp->cpumask); lead_cpu = cpumask_first(&rp->cpumask); - if (lead_cpu >= nr_cpu_ids) + if (lead_cpu >= nr_cpu_ids) { + if (rapl_msr_pmu) + rapl_package_remove_pmu(rp); rapl_remove_package_cpuslocked(rp); - else if (rp->lead_cpu == cpu) + } else if (rp->lead_cpu == cpu) { rp->lead_cpu = lead_cpu; + } + return 0; } -static int rapl_msr_read_raw(int cpu, struct reg_action *ra) +static int rapl_msr_read_raw(int cpu, struct reg_action *ra, bool atomic) { + /* + * When called from atomic-context (eg PMU event handler) + * perform MSR read directly using rdmsrq(). + */ + if (atomic) { + if (unlikely(smp_processor_id() != cpu)) + return -EIO; + + rdmsrq(ra->reg.msr, ra->value); + goto out; + } + if (rdmsrq_safe_on_cpu(cpu, ra->reg.msr, &ra->value)) { pr_debug("failed to read msr 0x%x on cpu %d\n", ra->reg.msr, cpu); return -EIO; } + +out: ra->value &= ra->mask; return 0; } @@ -151,6 +173,16 @@ static const struct x86_cpu_id pl4_support_ids[] = { X86_MATCH_VFM(INTEL_ARROWLAKE_U, NULL), X86_MATCH_VFM(INTEL_ARROWLAKE_H, NULL), X86_MATCH_VFM(INTEL_PANTHERLAKE_L, NULL), + X86_MATCH_VFM(INTEL_WILDCATLAKE_L, NULL), + X86_MATCH_VFM(INTEL_NOVALAKE, NULL), + X86_MATCH_VFM(INTEL_NOVALAKE_L, NULL), + {} +}; + +/* List of MSR-based RAPL PMU support CPUs */ +static const struct x86_cpu_id pmu_support_ids[] = { + X86_MATCH_VFM(INTEL_PANTHERLAKE_L, NULL), + X86_MATCH_VFM(INTEL_WILDCATLAKE_L, NULL), {} }; @@ -181,6 +213,11 @@ static int rapl_msr_probe(struct platform_device *pdev) pr_info("PL4 support detected.\n"); } + if (x86_match_cpu(pmu_support_ids)) { + rapl_msr_pmu = true; + pr_info("MSR-based RAPL PMU support enabled\n"); + } + rapl_msr_priv->control_type = powercap_register_control_type(NULL, "intel-rapl", NULL); if (IS_ERR(rapl_msr_priv->control_type)) { pr_debug("failed to register powercap control_type.\n"); diff --git a/drivers/powercap/intel_rapl_tpmi.c b/drivers/powercap/intel_rapl_tpmi.c index 82201bf4685d..0a0b85f4528b 100644 --- a/drivers/powercap/intel_rapl_tpmi.c +++ b/drivers/powercap/intel_rapl_tpmi.c @@ -60,7 +60,7 @@ static DEFINE_MUTEX(tpmi_rapl_lock); static struct powercap_control_type *tpmi_control_type; -static int tpmi_rapl_read_raw(int id, struct reg_action *ra) +static int tpmi_rapl_read_raw(int id, struct reg_action *ra, bool atomic) { if (!ra->reg.mmio) return -EINVAL; diff --git a/drivers/pps/generators/pps_gen_parport.c b/drivers/pps/generators/pps_gen_parport.c index f5eeb4dd01ad..05bbf8d30ef1 100644 --- a/drivers/pps/generators/pps_gen_parport.c +++ b/drivers/pps/generators/pps_gen_parport.c @@ -80,8 +80,7 @@ static enum hrtimer_restart hrtimer_event(struct hrtimer *timer) /* check if we are late */ if (expire_time.tv_sec != ts1.tv_sec || ts1.tv_nsec > lim) { local_irq_restore(flags); - pr_err("we are late this time %lld.%09ld\n", - (s64)ts1.tv_sec, ts1.tv_nsec); + pr_err("we are late this time %ptSp\n", &ts1); goto done; } diff --git a/drivers/pps/kapi.c b/drivers/pps/kapi.c index e9389876229e..6985c34de2ce 100644 --- a/drivers/pps/kapi.c +++ b/drivers/pps/kapi.c @@ -163,8 +163,7 @@ void pps_event(struct pps_device *pps, struct pps_event_time *ts, int event, /* check event type */ BUG_ON((event & (PPS_CAPTUREASSERT | PPS_CAPTURECLEAR)) == 0); - dev_dbg(&pps->dev, "PPS event at %lld.%09ld\n", - (s64)ts->ts_real.tv_sec, ts->ts_real.tv_nsec); + dev_dbg(&pps->dev, "PPS event at %ptSp\n", &ts->ts_real); timespec_to_pps_ktime(&ts_real, ts->ts_real); diff --git a/drivers/ptp/ptp_ocp.c b/drivers/ptp/ptp_ocp.c index bf8187c963aa..65fe05cac8c4 100644 --- a/drivers/ptp/ptp_ocp.c +++ b/drivers/ptp/ptp_ocp.c @@ -4291,11 +4291,9 @@ ptp_ocp_summary_show(struct seq_file *s, void *data) ns += (s64)bp->utc_tai_offset * NSEC_PER_SEC; sys_ts = ns_to_timespec64(ns); - seq_printf(s, "%7s: %lld.%ld == %ptT TAI\n", "PHC", - ts.tv_sec, ts.tv_nsec, &ts); - seq_printf(s, "%7s: %lld.%ld == %ptT UTC offset %d\n", "SYS", - sys_ts.tv_sec, sys_ts.tv_nsec, &sys_ts, - bp->utc_tai_offset); + seq_printf(s, "%7s: %ptSp == %ptS TAI\n", "PHC", &ts, &ts); + seq_printf(s, "%7s: %ptSp == %ptS UTC offset %d\n", "SYS", + &sys_ts, &sys_ts, bp->utc_tai_offset); seq_printf(s, "%7s: PHC:SYS offset: %lld window: %lld\n", "", timespec64_to_ns(&ts) - ns, post_ns - pre_ns); @@ -4503,9 +4501,8 @@ ptp_ocp_phc_info(struct ptp_ocp *bp) ptp_clock_index(bp->ptp)); if (!ptp_ocp_gettimex(&bp->ptp_info, &ts, NULL)) - dev_info(&bp->pdev->dev, "Time: %lld.%ld, %s\n", - ts.tv_sec, ts.tv_nsec, - bp->sync ? "in-sync" : "UNSYNCED"); + dev_info(&bp->pdev->dev, "Time: %ptSp, %s\n", + &ts, bp->sync ? "in-sync" : "UNSYNCED"); } static void diff --git a/drivers/ras/amd/atl/core.c b/drivers/ras/amd/atl/core.c index 4197e10993ac..0f7cd6dab0b0 100644 --- a/drivers/ras/amd/atl/core.c +++ b/drivers/ras/amd/atl/core.c @@ -194,6 +194,8 @@ MODULE_DEVICE_TABLE(x86cpu, amd_atl_cpuids); static int __init amd_atl_init(void) { + int ret; + if (!x86_match_cpu(amd_atl_cpuids)) return -ENODEV; @@ -202,8 +204,9 @@ static int __init amd_atl_init(void) check_for_legacy_df_access(); - if (get_df_system_info()) - return -ENODEV; + ret = get_df_system_info(); + if (ret) + return ret; /* Increment this module's recount so that it can't be easily unloaded. */ __module_get(THIS_MODULE); diff --git a/drivers/ras/amd/atl/internal.h b/drivers/ras/amd/atl/internal.h index 2b6279d32774..82a56d9c2be1 100644 --- a/drivers/ras/amd/atl/internal.h +++ b/drivers/ras/amd/atl/internal.h @@ -138,7 +138,8 @@ struct df_flags { __u8 legacy_ficaa : 1, socket_id_shift_quirk : 1, heterogeneous : 1, - __reserved_0 : 5; + prm_only : 1, + __reserved_0 : 4; }; struct df_config { @@ -283,6 +284,9 @@ unsigned long convert_umc_mca_addr_to_sys_addr(struct atl_err *err); u64 add_base_and_hole(struct addr_ctx *ctx, u64 addr); u64 remove_base_and_hole(struct addr_ctx *ctx, u64 addr); +/* GUIDs for PRM handlers */ +extern const guid_t norm_to_sys_guid; + #ifdef CONFIG_AMD_ATL_PRM unsigned long prm_umc_norm_to_sys_addr(u8 socket_id, u64 umc_bank_inst_id, unsigned long addr); #else diff --git a/drivers/ras/amd/atl/prm.c b/drivers/ras/amd/atl/prm.c index 0931a20d213b..0f9bfa96e16a 100644 --- a/drivers/ras/amd/atl/prm.c +++ b/drivers/ras/amd/atl/prm.c @@ -29,10 +29,6 @@ struct norm_to_sys_param_buf { void *out_buf; } __packed; -static const guid_t norm_to_sys_guid = GUID_INIT(0xE7180659, 0xA65D, 0x451D, - 0x92, 0xCD, 0x2B, 0x56, 0xF1, - 0x2B, 0xEB, 0xA6); - unsigned long prm_umc_norm_to_sys_addr(u8 socket_id, u64 bank_id, unsigned long addr) { struct norm_to_sys_param_buf p_buf; diff --git a/drivers/ras/amd/atl/system.c b/drivers/ras/amd/atl/system.c index e18d916d5e8b..812a30e21d3a 100644 --- a/drivers/ras/amd/atl/system.c +++ b/drivers/ras/amd/atl/system.c @@ -12,6 +12,12 @@ #include "internal.h" +#include <linux/prmt.h> + +const guid_t norm_to_sys_guid = GUID_INIT(0xE7180659, 0xA65D, 0x451D, + 0x92, 0xCD, 0x2B, 0x56, 0xF1, + 0x2B, 0xEB, 0xA6); + int determine_node_id(struct addr_ctx *ctx, u8 socket_id, u8 die_id) { u16 socket_id_bits, die_id_bits; @@ -212,15 +218,17 @@ static int determine_df_rev(void) if (!rev) return determine_df_rev_legacy(); - /* - * Fail out for major revisions other than '4'. - * - * Explicit support should be added for newer systems to avoid issues. - */ if (rev == 4) return df4_determine_df_rev(reg); - return -EINVAL; + /* All other systems should have PRM handlers. */ + if (!acpi_prm_handler_available(&norm_to_sys_guid)) { + pr_debug("PRM not available\n"); + return -ENODEV; + } + + df_cfg.flags.prm_only = true; + return 0; } static int get_dram_hole_base(void) @@ -288,12 +296,18 @@ static void dump_df_cfg(void) int get_df_system_info(void) { - if (determine_df_rev()) { + int ret; + + ret = determine_df_rev(); + if (ret) { pr_warn("Failed to determine DF Revision"); df_cfg.rev = UNKNOWN; - return -EINVAL; + return ret; } + if (df_cfg.flags.prm_only) + return 0; + apply_node_id_shift(); get_num_maps(); diff --git a/drivers/ras/amd/atl/umc.c b/drivers/ras/amd/atl/umc.c index 6e072b7667e9..befc616d5e8a 100644 --- a/drivers/ras/amd/atl/umc.c +++ b/drivers/ras/amd/atl/umc.c @@ -49,17 +49,6 @@ static u8 get_coh_st_inst_id_mi300(struct atl_err *err) return i; } -/* XOR the bits in @val. */ -static u16 bitwise_xor_bits(u16 val) -{ - u16 tmp = 0; - u8 i; - - for (i = 0; i < 16; i++) - tmp ^= (val >> i) & 0x1; - - return tmp; -} struct xor_bits { bool xor_enable; @@ -250,17 +239,17 @@ static unsigned long convert_dram_to_norm_addr_mi300(unsigned long addr) if (!addr_hash.bank[i].xor_enable) continue; - temp = bitwise_xor_bits(col & addr_hash.bank[i].col_xor); - temp ^= bitwise_xor_bits(row & addr_hash.bank[i].row_xor); + temp = hweight16(col & addr_hash.bank[i].col_xor) & 1; + temp ^= hweight16(row & addr_hash.bank[i].row_xor) & 1; bank ^= temp << i; } /* Calculate hash for PC bit. */ if (addr_hash.pc.xor_enable) { - temp = bitwise_xor_bits(col & addr_hash.pc.col_xor); - temp ^= bitwise_xor_bits(row & addr_hash.pc.row_xor); + temp = hweight16(col & addr_hash.pc.col_xor) & 1; + temp ^= hweight16(row & addr_hash.pc.row_xor) & 1; /* Bits SID[1:0] act as Bank[5:4] for PC hash, so apply them here. */ - temp ^= bitwise_xor_bits((bank | sid << NUM_BANK_BITS) & addr_hash.bank_xor); + temp ^= hweight16((bank | sid << NUM_BANK_BITS) & addr_hash.bank_xor) & 1; pc ^= temp; } @@ -422,7 +411,7 @@ unsigned long convert_umc_mca_addr_to_sys_addr(struct atl_err *err) socket_id, die_id, coh_st_inst_id, addr); ret_addr = prm_umc_norm_to_sys_addr(socket_id, err->ipid, addr); - if (!IS_ERR_VALUE(ret_addr)) + if (!IS_ERR_VALUE(ret_addr) || df_cfg.flags.prm_only) return ret_addr; return norm_to_sys_addr(socket_id, die_id, coh_st_inst_id, addr); diff --git a/drivers/ras/cec.c b/drivers/ras/cec.c index e440b15fbabc..15f7f043c8ef 100644 --- a/drivers/ras/cec.c +++ b/drivers/ras/cec.c @@ -166,7 +166,7 @@ static void cec_mod_work(unsigned long interval) unsigned long iv; iv = interval * HZ; - mod_delayed_work(system_wq, &cec_work, round_jiffies(iv)); + mod_delayed_work(system_percpu_wq, &cec_work, round_jiffies(iv)); } static void cec_work_fn(struct work_struct *work) diff --git a/drivers/regulator/rtq2208-regulator.c b/drivers/regulator/rtq2208-regulator.c index 9cde7181b0f0..f669a562f036 100644 --- a/drivers/regulator/rtq2208-regulator.c +++ b/drivers/regulator/rtq2208-regulator.c @@ -53,7 +53,7 @@ #define RTQ2208_MASK_BUCKPH_GROUP1 GENMASK(6, 4) #define RTQ2208_MASK_BUCKPH_GROUP2 GENMASK(2, 0) #define RTQ2208_MASK_LDO2_OPT0 BIT(7) -#define RTQ2208_MASK_LDO2_OPT1 BIT(6) +#define RTQ2208_MASK_LDO2_OPT1 BIT(7) #define RTQ2208_MASK_LDO1_FIXED BIT(6) /* Size */ @@ -543,14 +543,14 @@ static int rtq2208_regulator_check(struct device *dev, int *num, int *regulator_ switch (FIELD_GET(RTQ2208_MASK_BUCKPH_GROUP2, buck_phase)) { case 2: - rtq2208_used_table[RTQ2208_BUCK_F] = true; + rtq2208_used_table[RTQ2208_BUCK_H] = true; fallthrough; case 1: rtq2208_used_table[RTQ2208_BUCK_E] = true; fallthrough; case 0: case 3: - rtq2208_used_table[RTQ2208_BUCK_H] = true; + rtq2208_used_table[RTQ2208_BUCK_F] = true; fallthrough; default: rtq2208_used_table[RTQ2208_BUCK_G] = true; diff --git a/drivers/resctrl/Kconfig b/drivers/resctrl/Kconfig new file mode 100644 index 000000000000..c808e0470394 --- /dev/null +++ b/drivers/resctrl/Kconfig @@ -0,0 +1,24 @@ +menuconfig ARM64_MPAM_DRIVER + bool "MPAM driver" + depends on ARM64 && ARM64_MPAM && EXPERT + help + Memory System Resource Partitioning and Monitoring (MPAM) driver for + System IP, e.g. caches and memory controllers. + +if ARM64_MPAM_DRIVER + +config ARM64_MPAM_DRIVER_DEBUG + bool "Enable debug messages from the MPAM driver" + help + Say yes here to enable debug messages from the MPAM driver. + +config MPAM_KUNIT_TEST + bool "KUnit tests for MPAM driver " if !KUNIT_ALL_TESTS + depends on KUNIT=y + default KUNIT_ALL_TESTS + help + Enable this option to run tests in the MPAM driver. + + If unsure, say N. + +endif diff --git a/drivers/resctrl/Makefile b/drivers/resctrl/Makefile new file mode 100644 index 000000000000..898199dcf80d --- /dev/null +++ b/drivers/resctrl/Makefile @@ -0,0 +1,4 @@ +obj-$(CONFIG_ARM64_MPAM_DRIVER) += mpam.o +mpam-y += mpam_devices.o + +ccflags-$(CONFIG_ARM64_MPAM_DRIVER_DEBUG) += -DDEBUG diff --git a/drivers/resctrl/mpam_devices.c b/drivers/resctrl/mpam_devices.c new file mode 100644 index 000000000000..0b5b158e1aaf --- /dev/null +++ b/drivers/resctrl/mpam_devices.c @@ -0,0 +1,2723 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Arm Ltd. + +#define pr_fmt(fmt) "%s:%s: " fmt, KBUILD_MODNAME, __func__ + +#include <linux/acpi.h> +#include <linux/atomic.h> +#include <linux/arm_mpam.h> +#include <linux/bitfield.h> +#include <linux/bitmap.h> +#include <linux/cacheinfo.h> +#include <linux/cpu.h> +#include <linux/cpumask.h> +#include <linux/device.h> +#include <linux/errno.h> +#include <linux/gfp.h> +#include <linux/interrupt.h> +#include <linux/irq.h> +#include <linux/irqdesc.h> +#include <linux/list.h> +#include <linux/lockdep.h> +#include <linux/mutex.h> +#include <linux/platform_device.h> +#include <linux/printk.h> +#include <linux/srcu.h> +#include <linux/spinlock.h> +#include <linux/types.h> +#include <linux/workqueue.h> + +#include "mpam_internal.h" + +DEFINE_STATIC_KEY_FALSE(mpam_enabled); /* This moves to arch code */ + +/* + * mpam_list_lock protects the SRCU lists when writing. Once the + * mpam_enabled key is enabled these lists are read-only, + * unless the error interrupt disables the driver. + */ +static DEFINE_MUTEX(mpam_list_lock); +static LIST_HEAD(mpam_all_msc); + +struct srcu_struct mpam_srcu; + +/* + * Number of MSCs that have been probed. Once all MSCs have been probed MPAM + * can be enabled. + */ +static atomic_t mpam_num_msc; + +static int mpam_cpuhp_state; +static DEFINE_MUTEX(mpam_cpuhp_state_lock); + +/* + * The smallest common values for any CPU or MSC in the system. + * Generating traffic outside this range will result in screaming interrupts. + */ +u16 mpam_partid_max; +u8 mpam_pmg_max; +static bool partid_max_init, partid_max_published; +static DEFINE_SPINLOCK(partid_max_lock); + +/* + * mpam is enabled once all devices have been probed from CPU online callbacks, + * scheduled via this work_struct. If access to an MSC depends on a CPU that + * was not brought online at boot, this can happen surprisingly late. + */ +static DECLARE_WORK(mpam_enable_work, &mpam_enable); + +/* + * All mpam error interrupts indicate a software bug. On receipt, disable the + * driver. + */ +static DECLARE_WORK(mpam_broken_work, &mpam_disable); + +/* When mpam is disabled, the printed reason to aid debugging */ +static char *mpam_disable_reason; + +/* + * An MSC is a physical container for controls and monitors, each identified by + * their RIS index. These share a base-address, interrupts and some MMIO + * registers. A vMSC is a virtual container for RIS in an MSC that control or + * monitor the same thing. Members of a vMSC are all RIS in the same MSC, but + * not all RIS in an MSC share a vMSC. + * + * Components are a group of vMSC that control or monitor the same thing but + * are from different MSC, so have different base-address, interrupts etc. + * Classes are the set components of the same type. + * + * The features of a vMSC is the union of the RIS it contains. + * The features of a Class and Component are the common subset of the vMSC + * they contain. + * + * e.g. The system cache may have bandwidth controls on multiple interfaces, + * for regulating traffic from devices independently of traffic from CPUs. + * If these are two RIS in one MSC, they will be treated as controlling + * different things, and will not share a vMSC/component/class. + * + * e.g. The L2 may have one MSC and two RIS, one for cache-controls another + * for bandwidth. These two RIS are members of the same vMSC. + * + * e.g. The set of RIS that make up the L2 are grouped as a component. These + * are sometimes termed slices. They should be configured the same, as if there + * were only one. + * + * e.g. The SoC probably has more than one L2, each attached to a distinct set + * of CPUs. All the L2 components are grouped as a class. + * + * When creating an MSC, struct mpam_msc is added to the all mpam_all_msc list, + * then linked via struct mpam_ris to a vmsc, component and class. + * The same MSC may exist under different class->component->vmsc paths, but the + * RIS index will be unique. + */ +LIST_HEAD(mpam_classes); + +/* List of all objects that can be free()d after synchronise_srcu() */ +static LLIST_HEAD(mpam_garbage); + +static inline void init_garbage(struct mpam_garbage *garbage) +{ + init_llist_node(&garbage->llist); +} + +#define add_to_garbage(x) \ +do { \ + __typeof__(x) _x = (x); \ + _x->garbage.to_free = _x; \ + llist_add(&_x->garbage.llist, &mpam_garbage); \ +} while (0) + +static void mpam_free_garbage(void) +{ + struct mpam_garbage *iter, *tmp; + struct llist_node *to_free = llist_del_all(&mpam_garbage); + + if (!to_free) + return; + + synchronize_srcu(&mpam_srcu); + + llist_for_each_entry_safe(iter, tmp, to_free, llist) { + if (iter->pdev) + devm_kfree(&iter->pdev->dev, iter->to_free); + else + kfree(iter->to_free); + } +} + +/* + * Once mpam is enabled, new requestors cannot further reduce the available + * partid. Assert that the size is fixed, and new requestors will be turned + * away. + */ +static void mpam_assert_partid_sizes_fixed(void) +{ + WARN_ON_ONCE(!partid_max_published); +} + +static u32 __mpam_read_reg(struct mpam_msc *msc, u16 reg) +{ + WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); + + return readl_relaxed(msc->mapped_hwpage + reg); +} + +static inline u32 _mpam_read_partsel_reg(struct mpam_msc *msc, u16 reg) +{ + lockdep_assert_held_once(&msc->part_sel_lock); + return __mpam_read_reg(msc, reg); +} + +#define mpam_read_partsel_reg(msc, reg) _mpam_read_partsel_reg(msc, MPAMF_##reg) + +static void __mpam_write_reg(struct mpam_msc *msc, u16 reg, u32 val) +{ + WARN_ON_ONCE(reg + sizeof(u32) > msc->mapped_hwpage_sz); + WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); + + writel_relaxed(val, msc->mapped_hwpage + reg); +} + +static inline void _mpam_write_partsel_reg(struct mpam_msc *msc, u16 reg, u32 val) +{ + lockdep_assert_held_once(&msc->part_sel_lock); + __mpam_write_reg(msc, reg, val); +} + +#define mpam_write_partsel_reg(msc, reg, val) _mpam_write_partsel_reg(msc, MPAMCFG_##reg, val) + +static inline u32 _mpam_read_monsel_reg(struct mpam_msc *msc, u16 reg) +{ + mpam_mon_sel_lock_held(msc); + return __mpam_read_reg(msc, reg); +} + +#define mpam_read_monsel_reg(msc, reg) _mpam_read_monsel_reg(msc, MSMON_##reg) + +static inline void _mpam_write_monsel_reg(struct mpam_msc *msc, u16 reg, u32 val) +{ + mpam_mon_sel_lock_held(msc); + __mpam_write_reg(msc, reg, val); +} + +#define mpam_write_monsel_reg(msc, reg, val) _mpam_write_monsel_reg(msc, MSMON_##reg, val) + +static u64 mpam_msc_read_idr(struct mpam_msc *msc) +{ + u64 idr_high = 0, idr_low; + + lockdep_assert_held(&msc->part_sel_lock); + + idr_low = mpam_read_partsel_reg(msc, IDR); + if (FIELD_GET(MPAMF_IDR_EXT, idr_low)) + idr_high = mpam_read_partsel_reg(msc, IDR + 4); + + return (idr_high << 32) | idr_low; +} + +static void mpam_msc_clear_esr(struct mpam_msc *msc) +{ + u64 esr_low = __mpam_read_reg(msc, MPAMF_ESR); + + if (!esr_low) + return; + + /* + * Clearing the high/low bits of MPAMF_ESR can not be atomic. + * Clear the top half first, so that the pending error bits in the + * lower half prevent hardware from updating either half of the + * register. + */ + if (msc->has_extd_esr) + __mpam_write_reg(msc, MPAMF_ESR + 4, 0); + __mpam_write_reg(msc, MPAMF_ESR, 0); +} + +static u64 mpam_msc_read_esr(struct mpam_msc *msc) +{ + u64 esr_high = 0, esr_low; + + esr_low = __mpam_read_reg(msc, MPAMF_ESR); + if (msc->has_extd_esr) + esr_high = __mpam_read_reg(msc, MPAMF_ESR + 4); + + return (esr_high << 32) | esr_low; +} + +static void __mpam_part_sel_raw(u32 partsel, struct mpam_msc *msc) +{ + lockdep_assert_held(&msc->part_sel_lock); + + mpam_write_partsel_reg(msc, PART_SEL, partsel); +} + +static void __mpam_part_sel(u8 ris_idx, u16 partid, struct mpam_msc *msc) +{ + u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) | + FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, partid); + + __mpam_part_sel_raw(partsel, msc); +} + +static void __mpam_intpart_sel(u8 ris_idx, u16 intpartid, struct mpam_msc *msc) +{ + u32 partsel = FIELD_PREP(MPAMCFG_PART_SEL_RIS, ris_idx) | + FIELD_PREP(MPAMCFG_PART_SEL_PARTID_SEL, intpartid) | + MPAMCFG_PART_SEL_INTERNAL; + + __mpam_part_sel_raw(partsel, msc); +} + +int mpam_register_requestor(u16 partid_max, u8 pmg_max) +{ + guard(spinlock)(&partid_max_lock); + if (!partid_max_init) { + mpam_partid_max = partid_max; + mpam_pmg_max = pmg_max; + partid_max_init = true; + } else if (!partid_max_published) { + mpam_partid_max = min(mpam_partid_max, partid_max); + mpam_pmg_max = min(mpam_pmg_max, pmg_max); + } else { + /* New requestors can't lower the values */ + if (partid_max < mpam_partid_max || pmg_max < mpam_pmg_max) + return -EBUSY; + } + + return 0; +} +EXPORT_SYMBOL(mpam_register_requestor); + +static struct mpam_class * +mpam_class_alloc(u8 level_idx, enum mpam_class_types type) +{ + struct mpam_class *class; + + lockdep_assert_held(&mpam_list_lock); + + class = kzalloc(sizeof(*class), GFP_KERNEL); + if (!class) + return ERR_PTR(-ENOMEM); + init_garbage(&class->garbage); + + INIT_LIST_HEAD_RCU(&class->components); + /* Affinity is updated when ris are added */ + class->level = level_idx; + class->type = type; + INIT_LIST_HEAD_RCU(&class->classes_list); + ida_init(&class->ida_csu_mon); + ida_init(&class->ida_mbwu_mon); + + list_add_rcu(&class->classes_list, &mpam_classes); + + return class; +} + +static void mpam_class_destroy(struct mpam_class *class) +{ + lockdep_assert_held(&mpam_list_lock); + + list_del_rcu(&class->classes_list); + add_to_garbage(class); +} + +static struct mpam_class * +mpam_class_find(u8 level_idx, enum mpam_class_types type) +{ + struct mpam_class *class; + + lockdep_assert_held(&mpam_list_lock); + + list_for_each_entry(class, &mpam_classes, classes_list) { + if (class->type == type && class->level == level_idx) + return class; + } + + return mpam_class_alloc(level_idx, type); +} + +static struct mpam_component * +mpam_component_alloc(struct mpam_class *class, int id) +{ + struct mpam_component *comp; + + lockdep_assert_held(&mpam_list_lock); + + comp = kzalloc(sizeof(*comp), GFP_KERNEL); + if (!comp) + return ERR_PTR(-ENOMEM); + init_garbage(&comp->garbage); + + comp->comp_id = id; + INIT_LIST_HEAD_RCU(&comp->vmsc); + /* Affinity is updated when RIS are added */ + INIT_LIST_HEAD_RCU(&comp->class_list); + comp->class = class; + + list_add_rcu(&comp->class_list, &class->components); + + return comp; +} + +static void __destroy_component_cfg(struct mpam_component *comp); + +static void mpam_component_destroy(struct mpam_component *comp) +{ + struct mpam_class *class = comp->class; + + lockdep_assert_held(&mpam_list_lock); + + __destroy_component_cfg(comp); + + list_del_rcu(&comp->class_list); + add_to_garbage(comp); + + if (list_empty(&class->components)) + mpam_class_destroy(class); +} + +static struct mpam_component * +mpam_component_find(struct mpam_class *class, int id) +{ + struct mpam_component *comp; + + lockdep_assert_held(&mpam_list_lock); + + list_for_each_entry(comp, &class->components, class_list) { + if (comp->comp_id == id) + return comp; + } + + return mpam_component_alloc(class, id); +} + +static struct mpam_vmsc * +mpam_vmsc_alloc(struct mpam_component *comp, struct mpam_msc *msc) +{ + struct mpam_vmsc *vmsc; + + lockdep_assert_held(&mpam_list_lock); + + vmsc = kzalloc(sizeof(*vmsc), GFP_KERNEL); + if (!vmsc) + return ERR_PTR(-ENOMEM); + init_garbage(&vmsc->garbage); + + INIT_LIST_HEAD_RCU(&vmsc->ris); + INIT_LIST_HEAD_RCU(&vmsc->comp_list); + vmsc->comp = comp; + vmsc->msc = msc; + + list_add_rcu(&vmsc->comp_list, &comp->vmsc); + + return vmsc; +} + +static void mpam_vmsc_destroy(struct mpam_vmsc *vmsc) +{ + struct mpam_component *comp = vmsc->comp; + + lockdep_assert_held(&mpam_list_lock); + + list_del_rcu(&vmsc->comp_list); + add_to_garbage(vmsc); + + if (list_empty(&comp->vmsc)) + mpam_component_destroy(comp); +} + +static struct mpam_vmsc * +mpam_vmsc_find(struct mpam_component *comp, struct mpam_msc *msc) +{ + struct mpam_vmsc *vmsc; + + lockdep_assert_held(&mpam_list_lock); + + list_for_each_entry(vmsc, &comp->vmsc, comp_list) { + if (vmsc->msc->id == msc->id) + return vmsc; + } + + return mpam_vmsc_alloc(comp, msc); +} + +/* + * The cacheinfo structures are only populated when CPUs are online. + * This helper walks the acpi tables to include offline CPUs too. + */ +int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level, + cpumask_t *affinity) +{ + return acpi_pptt_get_cpumask_from_cache_id(cache_id, affinity); +} + +/* + * cpumask_of_node() only knows about online CPUs. This can't tell us whether + * a class is represented on all possible CPUs. + */ +static void get_cpumask_from_node_id(u32 node_id, cpumask_t *affinity) +{ + int cpu; + + for_each_possible_cpu(cpu) { + if (node_id == cpu_to_node(cpu)) + cpumask_set_cpu(cpu, affinity); + } +} + +static int mpam_ris_get_affinity(struct mpam_msc *msc, cpumask_t *affinity, + enum mpam_class_types type, + struct mpam_class *class, + struct mpam_component *comp) +{ + int err; + + switch (type) { + case MPAM_CLASS_CACHE: + err = mpam_get_cpumask_from_cache_id(comp->comp_id, class->level, + affinity); + if (err) { + dev_warn_once(&msc->pdev->dev, + "Failed to determine CPU affinity\n"); + return err; + } + + if (cpumask_empty(affinity)) + dev_warn_once(&msc->pdev->dev, "no CPUs associated with cache node\n"); + + break; + case MPAM_CLASS_MEMORY: + get_cpumask_from_node_id(comp->comp_id, affinity); + /* affinity may be empty for CPU-less memory nodes */ + break; + case MPAM_CLASS_UNKNOWN: + return 0; + } + + cpumask_and(affinity, affinity, &msc->accessibility); + + return 0; +} + +static int mpam_ris_create_locked(struct mpam_msc *msc, u8 ris_idx, + enum mpam_class_types type, u8 class_id, + int component_id) +{ + int err; + struct mpam_vmsc *vmsc; + struct mpam_msc_ris *ris; + struct mpam_class *class; + struct mpam_component *comp; + struct platform_device *pdev = msc->pdev; + + lockdep_assert_held(&mpam_list_lock); + + if (ris_idx > MPAM_MSC_MAX_NUM_RIS) + return -EINVAL; + + if (test_and_set_bit(ris_idx, &msc->ris_idxs)) + return -EBUSY; + + ris = devm_kzalloc(&msc->pdev->dev, sizeof(*ris), GFP_KERNEL); + if (!ris) + return -ENOMEM; + init_garbage(&ris->garbage); + ris->garbage.pdev = pdev; + + class = mpam_class_find(class_id, type); + if (IS_ERR(class)) + return PTR_ERR(class); + + comp = mpam_component_find(class, component_id); + if (IS_ERR(comp)) { + if (list_empty(&class->components)) + mpam_class_destroy(class); + return PTR_ERR(comp); + } + + vmsc = mpam_vmsc_find(comp, msc); + if (IS_ERR(vmsc)) { + if (list_empty(&comp->vmsc)) + mpam_component_destroy(comp); + return PTR_ERR(vmsc); + } + + err = mpam_ris_get_affinity(msc, &ris->affinity, type, class, comp); + if (err) { + if (list_empty(&vmsc->ris)) + mpam_vmsc_destroy(vmsc); + return err; + } + + ris->ris_idx = ris_idx; + INIT_LIST_HEAD_RCU(&ris->msc_list); + INIT_LIST_HEAD_RCU(&ris->vmsc_list); + ris->vmsc = vmsc; + + cpumask_or(&comp->affinity, &comp->affinity, &ris->affinity); + cpumask_or(&class->affinity, &class->affinity, &ris->affinity); + list_add_rcu(&ris->vmsc_list, &vmsc->ris); + list_add_rcu(&ris->msc_list, &msc->ris); + + return 0; +} + +static void mpam_ris_destroy(struct mpam_msc_ris *ris) +{ + struct mpam_vmsc *vmsc = ris->vmsc; + struct mpam_msc *msc = vmsc->msc; + struct mpam_component *comp = vmsc->comp; + struct mpam_class *class = comp->class; + + lockdep_assert_held(&mpam_list_lock); + + /* + * It is assumed affinities don't overlap. If they do the class becomes + * unusable immediately. + */ + cpumask_andnot(&class->affinity, &class->affinity, &ris->affinity); + cpumask_andnot(&comp->affinity, &comp->affinity, &ris->affinity); + clear_bit(ris->ris_idx, &msc->ris_idxs); + list_del_rcu(&ris->msc_list); + list_del_rcu(&ris->vmsc_list); + add_to_garbage(ris); + + if (list_empty(&vmsc->ris)) + mpam_vmsc_destroy(vmsc); +} + +int mpam_ris_create(struct mpam_msc *msc, u8 ris_idx, + enum mpam_class_types type, u8 class_id, int component_id) +{ + int err; + + mutex_lock(&mpam_list_lock); + err = mpam_ris_create_locked(msc, ris_idx, type, class_id, + component_id); + mutex_unlock(&mpam_list_lock); + if (err) + mpam_free_garbage(); + + return err; +} + +static struct mpam_msc_ris *mpam_get_or_create_ris(struct mpam_msc *msc, + u8 ris_idx) +{ + int err; + struct mpam_msc_ris *ris; + + lockdep_assert_held(&mpam_list_lock); + + if (!test_bit(ris_idx, &msc->ris_idxs)) { + err = mpam_ris_create_locked(msc, ris_idx, MPAM_CLASS_UNKNOWN, + 0, 0); + if (err) + return ERR_PTR(err); + } + + list_for_each_entry(ris, &msc->ris, msc_list) { + if (ris->ris_idx == ris_idx) + return ris; + } + + return ERR_PTR(-ENOENT); +} + +/* + * IHI009A.a has this nugget: "If a monitor does not support automatic behaviour + * of NRDY, software can use this bit for any purpose" - so hardware might not + * implement this - but it isn't RES0. + * + * Try and see what values stick in this bit. If we can write either value, + * its probably not implemented by hardware. + */ +static bool _mpam_ris_hw_probe_hw_nrdy(struct mpam_msc_ris *ris, u32 mon_reg) +{ + u32 now; + u64 mon_sel; + bool can_set, can_clear; + struct mpam_msc *msc = ris->vmsc->msc; + + if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc))) + return false; + + mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, 0) | + FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); + _mpam_write_monsel_reg(msc, mon_reg, mon_sel); + + _mpam_write_monsel_reg(msc, mon_reg, MSMON___NRDY); + now = _mpam_read_monsel_reg(msc, mon_reg); + can_set = now & MSMON___NRDY; + + _mpam_write_monsel_reg(msc, mon_reg, 0); + now = _mpam_read_monsel_reg(msc, mon_reg); + can_clear = !(now & MSMON___NRDY); + mpam_mon_sel_unlock(msc); + + return (!can_set || !can_clear); +} + +#define mpam_ris_hw_probe_hw_nrdy(_ris, _mon_reg) \ + _mpam_ris_hw_probe_hw_nrdy(_ris, MSMON_##_mon_reg) + +static void mpam_ris_hw_probe(struct mpam_msc_ris *ris) +{ + int err; + struct mpam_msc *msc = ris->vmsc->msc; + struct device *dev = &msc->pdev->dev; + struct mpam_props *props = &ris->props; + struct mpam_class *class = ris->vmsc->comp->class; + + lockdep_assert_held(&msc->probe_lock); + lockdep_assert_held(&msc->part_sel_lock); + + /* Cache Capacity Partitioning */ + if (FIELD_GET(MPAMF_IDR_HAS_CCAP_PART, ris->idr)) { + u32 ccap_features = mpam_read_partsel_reg(msc, CCAP_IDR); + + props->cmax_wd = FIELD_GET(MPAMF_CCAP_IDR_CMAX_WD, ccap_features); + if (props->cmax_wd && + FIELD_GET(MPAMF_CCAP_IDR_HAS_CMAX_SOFTLIM, ccap_features)) + mpam_set_feature(mpam_feat_cmax_softlim, props); + + if (props->cmax_wd && + !FIELD_GET(MPAMF_CCAP_IDR_NO_CMAX, ccap_features)) + mpam_set_feature(mpam_feat_cmax_cmax, props); + + if (props->cmax_wd && + FIELD_GET(MPAMF_CCAP_IDR_HAS_CMIN, ccap_features)) + mpam_set_feature(mpam_feat_cmax_cmin, props); + + props->cassoc_wd = FIELD_GET(MPAMF_CCAP_IDR_CASSOC_WD, ccap_features); + if (props->cassoc_wd && + FIELD_GET(MPAMF_CCAP_IDR_HAS_CASSOC, ccap_features)) + mpam_set_feature(mpam_feat_cmax_cassoc, props); + } + + /* Cache Portion partitioning */ + if (FIELD_GET(MPAMF_IDR_HAS_CPOR_PART, ris->idr)) { + u32 cpor_features = mpam_read_partsel_reg(msc, CPOR_IDR); + + props->cpbm_wd = FIELD_GET(MPAMF_CPOR_IDR_CPBM_WD, cpor_features); + if (props->cpbm_wd) + mpam_set_feature(mpam_feat_cpor_part, props); + } + + /* Memory bandwidth partitioning */ + if (FIELD_GET(MPAMF_IDR_HAS_MBW_PART, ris->idr)) { + u32 mbw_features = mpam_read_partsel_reg(msc, MBW_IDR); + + /* portion bitmap resolution */ + props->mbw_pbm_bits = FIELD_GET(MPAMF_MBW_IDR_BWPBM_WD, mbw_features); + if (props->mbw_pbm_bits && + FIELD_GET(MPAMF_MBW_IDR_HAS_PBM, mbw_features)) + mpam_set_feature(mpam_feat_mbw_part, props); + + props->bwa_wd = FIELD_GET(MPAMF_MBW_IDR_BWA_WD, mbw_features); + if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MAX, mbw_features)) + mpam_set_feature(mpam_feat_mbw_max, props); + + if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_MIN, mbw_features)) + mpam_set_feature(mpam_feat_mbw_min, props); + + if (props->bwa_wd && FIELD_GET(MPAMF_MBW_IDR_HAS_PROP, mbw_features)) + mpam_set_feature(mpam_feat_mbw_prop, props); + } + + /* Priority partitioning */ + if (FIELD_GET(MPAMF_IDR_HAS_PRI_PART, ris->idr)) { + u32 pri_features = mpam_read_partsel_reg(msc, PRI_IDR); + + props->intpri_wd = FIELD_GET(MPAMF_PRI_IDR_INTPRI_WD, pri_features); + if (props->intpri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_INTPRI, pri_features)) { + mpam_set_feature(mpam_feat_intpri_part, props); + if (FIELD_GET(MPAMF_PRI_IDR_INTPRI_0_IS_LOW, pri_features)) + mpam_set_feature(mpam_feat_intpri_part_0_low, props); + } + + props->dspri_wd = FIELD_GET(MPAMF_PRI_IDR_DSPRI_WD, pri_features); + if (props->dspri_wd && FIELD_GET(MPAMF_PRI_IDR_HAS_DSPRI, pri_features)) { + mpam_set_feature(mpam_feat_dspri_part, props); + if (FIELD_GET(MPAMF_PRI_IDR_DSPRI_0_IS_LOW, pri_features)) + mpam_set_feature(mpam_feat_dspri_part_0_low, props); + } + } + + /* Performance Monitoring */ + if (FIELD_GET(MPAMF_IDR_HAS_MSMON, ris->idr)) { + u32 msmon_features = mpam_read_partsel_reg(msc, MSMON_IDR); + + /* + * If the firmware max-nrdy-us property is missing, the + * CSU counters can't be used. Should we wait forever? + */ + err = device_property_read_u32(&msc->pdev->dev, + "arm,not-ready-us", + &msc->nrdy_usec); + + if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_CSU, msmon_features)) { + u32 csumonidr; + + csumonidr = mpam_read_partsel_reg(msc, CSUMON_IDR); + props->num_csu_mon = FIELD_GET(MPAMF_CSUMON_IDR_NUM_MON, csumonidr); + if (props->num_csu_mon) { + bool hw_managed; + + mpam_set_feature(mpam_feat_msmon_csu, props); + + if (FIELD_GET(MPAMF_CSUMON_IDR_HAS_XCL, csumonidr)) + mpam_set_feature(mpam_feat_msmon_csu_xcl, props); + + /* Is NRDY hardware managed? */ + hw_managed = mpam_ris_hw_probe_hw_nrdy(ris, CSU); + if (hw_managed) + mpam_set_feature(mpam_feat_msmon_csu_hw_nrdy, props); + } + + /* + * Accept the missing firmware property if NRDY appears + * un-implemented. + */ + if (err && mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, props)) + dev_err_once(dev, "Counters are not usable because not-ready timeout was not provided by firmware."); + } + if (FIELD_GET(MPAMF_MSMON_IDR_MSMON_MBWU, msmon_features)) { + bool has_long, hw_managed; + u32 mbwumon_idr = mpam_read_partsel_reg(msc, MBWUMON_IDR); + + props->num_mbwu_mon = FIELD_GET(MPAMF_MBWUMON_IDR_NUM_MON, mbwumon_idr); + if (props->num_mbwu_mon) { + mpam_set_feature(mpam_feat_msmon_mbwu, props); + + if (FIELD_GET(MPAMF_MBWUMON_IDR_HAS_RWBW, mbwumon_idr)) + mpam_set_feature(mpam_feat_msmon_mbwu_rwbw, props); + + has_long = FIELD_GET(MPAMF_MBWUMON_IDR_HAS_LONG, mbwumon_idr); + if (has_long) { + if (FIELD_GET(MPAMF_MBWUMON_IDR_LWD, mbwumon_idr)) + mpam_set_feature(mpam_feat_msmon_mbwu_63counter, props); + else + mpam_set_feature(mpam_feat_msmon_mbwu_44counter, props); + } else { + mpam_set_feature(mpam_feat_msmon_mbwu_31counter, props); + } + + /* Is NRDY hardware managed? */ + hw_managed = mpam_ris_hw_probe_hw_nrdy(ris, MBWU); + if (hw_managed) + mpam_set_feature(mpam_feat_msmon_mbwu_hw_nrdy, props); + + /* + * Don't warn about any missing firmware property for + * MBWU NRDY - it doesn't make any sense! + */ + } + } + } + + /* + * RIS with PARTID narrowing don't have enough storage for one + * configuration per PARTID. If these are in a class we could use, + * reduce the supported partid_max to match the number of intpartid. + * If the class is unknown, just ignore it. + */ + if (FIELD_GET(MPAMF_IDR_HAS_PARTID_NRW, ris->idr) && + class->type != MPAM_CLASS_UNKNOWN) { + u32 nrwidr = mpam_read_partsel_reg(msc, PARTID_NRW_IDR); + u16 partid_max = FIELD_GET(MPAMF_PARTID_NRW_IDR_INTPARTID_MAX, nrwidr); + + mpam_set_feature(mpam_feat_partid_nrw, props); + msc->partid_max = min(msc->partid_max, partid_max); + } +} + +static int mpam_msc_hw_probe(struct mpam_msc *msc) +{ + u64 idr; + u16 partid_max; + u8 ris_idx, pmg_max; + struct mpam_msc_ris *ris; + struct device *dev = &msc->pdev->dev; + + lockdep_assert_held(&msc->probe_lock); + + idr = __mpam_read_reg(msc, MPAMF_AIDR); + if ((idr & MPAMF_AIDR_ARCH_MAJOR_REV) != MPAM_ARCHITECTURE_V1) { + dev_err_once(dev, "MSC does not match MPAM architecture v1.x\n"); + return -EIO; + } + + /* Grab an IDR value to find out how many RIS there are */ + mutex_lock(&msc->part_sel_lock); + idr = mpam_msc_read_idr(msc); + mutex_unlock(&msc->part_sel_lock); + + msc->ris_max = FIELD_GET(MPAMF_IDR_RIS_MAX, idr); + + /* Use these values so partid/pmg always starts with a valid value */ + msc->partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr); + msc->pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr); + + for (ris_idx = 0; ris_idx <= msc->ris_max; ris_idx++) { + mutex_lock(&msc->part_sel_lock); + __mpam_part_sel(ris_idx, 0, msc); + idr = mpam_msc_read_idr(msc); + mutex_unlock(&msc->part_sel_lock); + + partid_max = FIELD_GET(MPAMF_IDR_PARTID_MAX, idr); + pmg_max = FIELD_GET(MPAMF_IDR_PMG_MAX, idr); + msc->partid_max = min(msc->partid_max, partid_max); + msc->pmg_max = min(msc->pmg_max, pmg_max); + msc->has_extd_esr = FIELD_GET(MPAMF_IDR_HAS_EXTD_ESR, idr); + + mutex_lock(&mpam_list_lock); + ris = mpam_get_or_create_ris(msc, ris_idx); + mutex_unlock(&mpam_list_lock); + if (IS_ERR(ris)) + return PTR_ERR(ris); + ris->idr = idr; + + mutex_lock(&msc->part_sel_lock); + __mpam_part_sel(ris_idx, 0, msc); + mpam_ris_hw_probe(ris); + mutex_unlock(&msc->part_sel_lock); + } + + /* Clear any stale errors */ + mpam_msc_clear_esr(msc); + + spin_lock(&partid_max_lock); + mpam_partid_max = min(mpam_partid_max, msc->partid_max); + mpam_pmg_max = min(mpam_pmg_max, msc->pmg_max); + spin_unlock(&partid_max_lock); + + msc->probed = true; + + return 0; +} + +struct mon_read { + struct mpam_msc_ris *ris; + struct mon_cfg *ctx; + enum mpam_device_features type; + u64 *val; + int err; +}; + +static bool mpam_ris_has_mbwu_long_counter(struct mpam_msc_ris *ris) +{ + return (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, &ris->props) || + mpam_has_feature(mpam_feat_msmon_mbwu_44counter, &ris->props)); +} + +static u64 mpam_msc_read_mbwu_l(struct mpam_msc *msc) +{ + int retry = 3; + u32 mbwu_l_low; + u64 mbwu_l_high1, mbwu_l_high2; + + mpam_mon_sel_lock_held(msc); + + WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz); + WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); + + mbwu_l_high2 = __mpam_read_reg(msc, MSMON_MBWU_L + 4); + do { + mbwu_l_high1 = mbwu_l_high2; + mbwu_l_low = __mpam_read_reg(msc, MSMON_MBWU_L); + mbwu_l_high2 = __mpam_read_reg(msc, MSMON_MBWU_L + 4); + + retry--; + } while (mbwu_l_high1 != mbwu_l_high2 && retry > 0); + + if (mbwu_l_high1 == mbwu_l_high2) + return (mbwu_l_high1 << 32) | mbwu_l_low; + + pr_warn("Failed to read a stable value\n"); + return MSMON___L_NRDY; +} + +static void mpam_msc_zero_mbwu_l(struct mpam_msc *msc) +{ + mpam_mon_sel_lock_held(msc); + + WARN_ON_ONCE((MSMON_MBWU_L + sizeof(u64)) > msc->mapped_hwpage_sz); + WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), &msc->accessibility)); + + __mpam_write_reg(msc, MSMON_MBWU_L, 0); + __mpam_write_reg(msc, MSMON_MBWU_L + 4, 0); +} + +static void gen_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val, + u32 *flt_val) +{ + struct mon_cfg *ctx = m->ctx; + + /* + * For CSU counters its implementation-defined what happens when not + * filtering by partid. + */ + *ctl_val = MSMON_CFG_x_CTL_MATCH_PARTID; + + *flt_val = FIELD_PREP(MSMON_CFG_x_FLT_PARTID, ctx->partid); + + if (m->ctx->match_pmg) { + *ctl_val |= MSMON_CFG_x_CTL_MATCH_PMG; + *flt_val |= FIELD_PREP(MSMON_CFG_x_FLT_PMG, ctx->pmg); + } + + switch (m->type) { + case mpam_feat_msmon_csu: + *ctl_val |= MSMON_CFG_CSU_CTL_TYPE_CSU; + + if (mpam_has_feature(mpam_feat_msmon_csu_xcl, &m->ris->props)) + *flt_val |= FIELD_PREP(MSMON_CFG_CSU_FLT_XCL, ctx->csu_exclude_clean); + + break; + case mpam_feat_msmon_mbwu_31counter: + case mpam_feat_msmon_mbwu_44counter: + case mpam_feat_msmon_mbwu_63counter: + *ctl_val |= MSMON_CFG_MBWU_CTL_TYPE_MBWU; + + if (mpam_has_feature(mpam_feat_msmon_mbwu_rwbw, &m->ris->props)) + *flt_val |= FIELD_PREP(MSMON_CFG_MBWU_FLT_RWBW, ctx->opts); + + break; + default: + pr_warn("Unexpected monitor type %d\n", m->type); + } +} + +static void read_msmon_ctl_flt_vals(struct mon_read *m, u32 *ctl_val, + u32 *flt_val) +{ + struct mpam_msc *msc = m->ris->vmsc->msc; + + switch (m->type) { + case mpam_feat_msmon_csu: + *ctl_val = mpam_read_monsel_reg(msc, CFG_CSU_CTL); + *flt_val = mpam_read_monsel_reg(msc, CFG_CSU_FLT); + break; + case mpam_feat_msmon_mbwu_31counter: + case mpam_feat_msmon_mbwu_44counter: + case mpam_feat_msmon_mbwu_63counter: + *ctl_val = mpam_read_monsel_reg(msc, CFG_MBWU_CTL); + *flt_val = mpam_read_monsel_reg(msc, CFG_MBWU_FLT); + break; + default: + pr_warn("Unexpected monitor type %d\n", m->type); + } +} + +/* Remove values set by the hardware to prevent apparent mismatches. */ +static inline void clean_msmon_ctl_val(u32 *cur_ctl) +{ + *cur_ctl &= ~MSMON_CFG_x_CTL_OFLOW_STATUS; + + if (FIELD_GET(MSMON_CFG_x_CTL_TYPE, *cur_ctl) == MSMON_CFG_MBWU_CTL_TYPE_MBWU) + *cur_ctl &= ~MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L; +} + +static void write_msmon_ctl_flt_vals(struct mon_read *m, u32 ctl_val, + u32 flt_val) +{ + struct mpam_msc *msc = m->ris->vmsc->msc; + + /* + * Write the ctl_val with the enable bit cleared, reset the counter, + * then enable counter. + */ + switch (m->type) { + case mpam_feat_msmon_csu: + mpam_write_monsel_reg(msc, CFG_CSU_FLT, flt_val); + mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val); + mpam_write_monsel_reg(msc, CSU, 0); + mpam_write_monsel_reg(msc, CFG_CSU_CTL, ctl_val | MSMON_CFG_x_CTL_EN); + break; + case mpam_feat_msmon_mbwu_31counter: + case mpam_feat_msmon_mbwu_44counter: + case mpam_feat_msmon_mbwu_63counter: + mpam_write_monsel_reg(msc, CFG_MBWU_FLT, flt_val); + mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val); + mpam_write_monsel_reg(msc, CFG_MBWU_CTL, ctl_val | MSMON_CFG_x_CTL_EN); + /* Counting monitors require NRDY to be reset by software */ + if (m->type == mpam_feat_msmon_mbwu_31counter) + mpam_write_monsel_reg(msc, MBWU, 0); + else + mpam_msc_zero_mbwu_l(m->ris->vmsc->msc); + break; + default: + pr_warn("Unexpected monitor type %d\n", m->type); + } +} + +static u64 mpam_msmon_overflow_val(enum mpam_device_features type) +{ + /* TODO: implement scaling counters */ + switch (type) { + case mpam_feat_msmon_mbwu_63counter: + return BIT_ULL(hweight_long(MSMON___LWD_VALUE)); + case mpam_feat_msmon_mbwu_44counter: + return BIT_ULL(hweight_long(MSMON___L_VALUE)); + case mpam_feat_msmon_mbwu_31counter: + return BIT_ULL(hweight_long(MSMON___VALUE)); + default: + return 0; + } +} + +static void __ris_msmon_read(void *arg) +{ + u64 now; + bool nrdy = false; + bool config_mismatch; + bool overflow; + struct mon_read *m = arg; + struct mon_cfg *ctx = m->ctx; + bool reset_on_next_read = false; + struct mpam_msc_ris *ris = m->ris; + struct msmon_mbwu_state *mbwu_state; + struct mpam_props *rprops = &ris->props; + struct mpam_msc *msc = m->ris->vmsc->msc; + u32 mon_sel, ctl_val, flt_val, cur_ctl, cur_flt; + + if (!mpam_mon_sel_lock(msc)) { + m->err = -EIO; + return; + } + mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, ctx->mon) | + FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); + mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel); + + switch (m->type) { + case mpam_feat_msmon_mbwu_31counter: + case mpam_feat_msmon_mbwu_44counter: + case mpam_feat_msmon_mbwu_63counter: + mbwu_state = &ris->mbwu_state[ctx->mon]; + if (mbwu_state) { + reset_on_next_read = mbwu_state->reset_on_next_read; + mbwu_state->reset_on_next_read = false; + } + break; + default: + break; + } + + /* + * Read the existing configuration to avoid re-writing the same values. + * This saves waiting for 'nrdy' on subsequent reads. + */ + read_msmon_ctl_flt_vals(m, &cur_ctl, &cur_flt); + + if (mpam_feat_msmon_mbwu_31counter == m->type) + overflow = cur_ctl & MSMON_CFG_x_CTL_OFLOW_STATUS; + else if (mpam_feat_msmon_mbwu_44counter == m->type || + mpam_feat_msmon_mbwu_63counter == m->type) + overflow = cur_ctl & MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L; + + clean_msmon_ctl_val(&cur_ctl); + gen_msmon_ctl_flt_vals(m, &ctl_val, &flt_val); + config_mismatch = cur_flt != flt_val || + cur_ctl != (ctl_val | MSMON_CFG_x_CTL_EN); + + if (config_mismatch || reset_on_next_read) { + write_msmon_ctl_flt_vals(m, ctl_val, flt_val); + overflow = false; + } else if (overflow) { + mpam_write_monsel_reg(msc, CFG_MBWU_CTL, + cur_ctl & + ~(MSMON_CFG_x_CTL_OFLOW_STATUS | + MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L)); + } + + switch (m->type) { + case mpam_feat_msmon_csu: + now = mpam_read_monsel_reg(msc, CSU); + if (mpam_has_feature(mpam_feat_msmon_csu_hw_nrdy, rprops)) + nrdy = now & MSMON___NRDY; + now = FIELD_GET(MSMON___VALUE, now); + break; + case mpam_feat_msmon_mbwu_31counter: + case mpam_feat_msmon_mbwu_44counter: + case mpam_feat_msmon_mbwu_63counter: + if (m->type != mpam_feat_msmon_mbwu_31counter) { + now = mpam_msc_read_mbwu_l(msc); + if (mpam_has_feature(mpam_feat_msmon_mbwu_hw_nrdy, rprops)) + nrdy = now & MSMON___L_NRDY; + + if (m->type == mpam_feat_msmon_mbwu_63counter) + now = FIELD_GET(MSMON___LWD_VALUE, now); + else + now = FIELD_GET(MSMON___L_VALUE, now); + } else { + now = mpam_read_monsel_reg(msc, MBWU); + if (mpam_has_feature(mpam_feat_msmon_mbwu_hw_nrdy, rprops)) + nrdy = now & MSMON___NRDY; + now = FIELD_GET(MSMON___VALUE, now); + } + + if (nrdy) + break; + + mbwu_state = &ris->mbwu_state[ctx->mon]; + + if (overflow) + mbwu_state->correction += mpam_msmon_overflow_val(m->type); + + /* + * Include bandwidth consumed before the last hardware reset and + * a counter size increment for each overflow. + */ + now += mbwu_state->correction; + break; + default: + m->err = -EINVAL; + } + mpam_mon_sel_unlock(msc); + + if (nrdy) { + m->err = -EBUSY; + return; + } + + *m->val += now; +} + +static int _msmon_read(struct mpam_component *comp, struct mon_read *arg) +{ + int err, any_err = 0; + struct mpam_vmsc *vmsc; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, + srcu_read_lock_held(&mpam_srcu)) { + struct mpam_msc *msc = vmsc->msc; + struct mpam_msc_ris *ris; + + list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, + srcu_read_lock_held(&mpam_srcu)) { + arg->ris = ris; + + err = smp_call_function_any(&msc->accessibility, + __ris_msmon_read, arg, + true); + if (!err && arg->err) + err = arg->err; + + /* + * Save one error to be returned to the caller, but + * keep reading counters so that get reprogrammed. On + * platforms with NRDY this lets us wait once. + */ + if (err) + any_err = err; + } + } + + return any_err; +} + +static enum mpam_device_features mpam_msmon_choose_counter(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + if (mpam_has_feature(mpam_feat_msmon_mbwu_63counter, cprops)) + return mpam_feat_msmon_mbwu_63counter; + if (mpam_has_feature(mpam_feat_msmon_mbwu_44counter, cprops)) + return mpam_feat_msmon_mbwu_44counter; + + return mpam_feat_msmon_mbwu_31counter; +} + +int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx, + enum mpam_device_features type, u64 *val) +{ + int err; + struct mon_read arg; + u64 wait_jiffies = 0; + struct mpam_class *class = comp->class; + struct mpam_props *cprops = &class->props; + + might_sleep(); + + if (!mpam_is_enabled()) + return -EIO; + + if (!mpam_has_feature(type, cprops)) + return -EOPNOTSUPP; + + if (type == mpam_feat_msmon_mbwu) + type = mpam_msmon_choose_counter(class); + + arg = (struct mon_read) { + .ctx = ctx, + .type = type, + .val = val, + }; + *val = 0; + + err = _msmon_read(comp, &arg); + if (err == -EBUSY && class->nrdy_usec) + wait_jiffies = usecs_to_jiffies(class->nrdy_usec); + + while (wait_jiffies) + wait_jiffies = schedule_timeout_uninterruptible(wait_jiffies); + + if (err == -EBUSY) { + arg = (struct mon_read) { + .ctx = ctx, + .type = type, + .val = val, + }; + *val = 0; + + err = _msmon_read(comp, &arg); + } + + return err; +} + +void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx) +{ + struct mpam_msc *msc; + struct mpam_vmsc *vmsc; + struct mpam_msc_ris *ris; + + if (!mpam_is_enabled()) + return; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, + srcu_read_lock_held(&mpam_srcu)) { + if (!mpam_has_feature(mpam_feat_msmon_mbwu, &vmsc->props)) + continue; + + msc = vmsc->msc; + list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, + srcu_read_lock_held(&mpam_srcu)) { + if (!mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props)) + continue; + + if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc))) + continue; + + ris->mbwu_state[ctx->mon].correction = 0; + ris->mbwu_state[ctx->mon].reset_on_next_read = true; + mpam_mon_sel_unlock(msc); + } + } +} + +static void mpam_reset_msc_bitmap(struct mpam_msc *msc, u16 reg, u16 wd) +{ + u32 num_words, msb; + u32 bm = ~0; + int i; + + lockdep_assert_held(&msc->part_sel_lock); + + if (wd == 0) + return; + + /* + * Write all ~0 to all but the last 32bit-word, which may + * have fewer bits... + */ + num_words = DIV_ROUND_UP(wd, 32); + for (i = 0; i < num_words - 1; i++, reg += sizeof(bm)) + __mpam_write_reg(msc, reg, bm); + + /* + * ....and then the last (maybe) partial 32bit word. When wd is a + * multiple of 32, msb should be 31 to write a full 32bit word. + */ + msb = (wd - 1) % 32; + bm = GENMASK(msb, 0); + __mpam_write_reg(msc, reg, bm); +} + +/* Called via IPI. Call while holding an SRCU reference */ +static void mpam_reprogram_ris_partid(struct mpam_msc_ris *ris, u16 partid, + struct mpam_config *cfg) +{ + u32 pri_val = 0; + u16 cmax = MPAMCFG_CMAX_CMAX; + struct mpam_msc *msc = ris->vmsc->msc; + struct mpam_props *rprops = &ris->props; + u16 dspri = GENMASK(rprops->dspri_wd, 0); + u16 intpri = GENMASK(rprops->intpri_wd, 0); + + mutex_lock(&msc->part_sel_lock); + __mpam_part_sel(ris->ris_idx, partid, msc); + + if (mpam_has_feature(mpam_feat_partid_nrw, rprops)) { + /* Update the intpartid mapping */ + mpam_write_partsel_reg(msc, INTPARTID, + MPAMCFG_INTPARTID_INTERNAL | partid); + + /* + * Then switch to the 'internal' partid to update the + * configuration. + */ + __mpam_intpart_sel(ris->ris_idx, partid, msc); + } + + if (mpam_has_feature(mpam_feat_cpor_part, rprops) && + mpam_has_feature(mpam_feat_cpor_part, cfg)) { + if (cfg->reset_cpbm) + mpam_reset_msc_bitmap(msc, MPAMCFG_CPBM, rprops->cpbm_wd); + else + mpam_write_partsel_reg(msc, CPBM, cfg->cpbm); + } + + if (mpam_has_feature(mpam_feat_mbw_part, rprops) && + mpam_has_feature(mpam_feat_mbw_part, cfg)) { + if (cfg->reset_mbw_pbm) + mpam_reset_msc_bitmap(msc, MPAMCFG_MBW_PBM, rprops->mbw_pbm_bits); + else + mpam_write_partsel_reg(msc, MBW_PBM, cfg->mbw_pbm); + } + + if (mpam_has_feature(mpam_feat_mbw_min, rprops) && + mpam_has_feature(mpam_feat_mbw_min, cfg)) + mpam_write_partsel_reg(msc, MBW_MIN, 0); + + if (mpam_has_feature(mpam_feat_mbw_max, rprops) && + mpam_has_feature(mpam_feat_mbw_max, cfg)) { + if (cfg->reset_mbw_max) + mpam_write_partsel_reg(msc, MBW_MAX, MPAMCFG_MBW_MAX_MAX); + else + mpam_write_partsel_reg(msc, MBW_MAX, cfg->mbw_max); + } + + if (mpam_has_feature(mpam_feat_mbw_prop, rprops) && + mpam_has_feature(mpam_feat_mbw_prop, cfg)) + mpam_write_partsel_reg(msc, MBW_PROP, 0); + + if (mpam_has_feature(mpam_feat_cmax_cmax, rprops)) + mpam_write_partsel_reg(msc, CMAX, cmax); + + if (mpam_has_feature(mpam_feat_cmax_cmin, rprops)) + mpam_write_partsel_reg(msc, CMIN, 0); + + if (mpam_has_feature(mpam_feat_cmax_cassoc, rprops)) + mpam_write_partsel_reg(msc, CASSOC, MPAMCFG_CASSOC_CASSOC); + + if (mpam_has_feature(mpam_feat_intpri_part, rprops) || + mpam_has_feature(mpam_feat_dspri_part, rprops)) { + /* aces high? */ + if (!mpam_has_feature(mpam_feat_intpri_part_0_low, rprops)) + intpri = 0; + if (!mpam_has_feature(mpam_feat_dspri_part_0_low, rprops)) + dspri = 0; + + if (mpam_has_feature(mpam_feat_intpri_part, rprops)) + pri_val |= FIELD_PREP(MPAMCFG_PRI_INTPRI, intpri); + if (mpam_has_feature(mpam_feat_dspri_part, rprops)) + pri_val |= FIELD_PREP(MPAMCFG_PRI_DSPRI, dspri); + + mpam_write_partsel_reg(msc, PRI, pri_val); + } + + mutex_unlock(&msc->part_sel_lock); +} + +/* Call with msc cfg_lock held */ +static int mpam_restore_mbwu_state(void *_ris) +{ + int i; + struct mon_read mwbu_arg; + struct mpam_msc_ris *ris = _ris; + struct mpam_class *class = ris->vmsc->comp->class; + + for (i = 0; i < ris->props.num_mbwu_mon; i++) { + if (ris->mbwu_state[i].enabled) { + mwbu_arg.ris = ris; + mwbu_arg.ctx = &ris->mbwu_state[i].cfg; + mwbu_arg.type = mpam_msmon_choose_counter(class); + + __ris_msmon_read(&mwbu_arg); + } + } + + return 0; +} + +/* Call with MSC cfg_lock held */ +static int mpam_save_mbwu_state(void *arg) +{ + int i; + u64 val; + struct mon_cfg *cfg; + u32 cur_flt, cur_ctl, mon_sel; + struct mpam_msc_ris *ris = arg; + struct msmon_mbwu_state *mbwu_state; + struct mpam_msc *msc = ris->vmsc->msc; + + for (i = 0; i < ris->props.num_mbwu_mon; i++) { + mbwu_state = &ris->mbwu_state[i]; + cfg = &mbwu_state->cfg; + + if (WARN_ON_ONCE(!mpam_mon_sel_lock(msc))) + return -EIO; + + mon_sel = FIELD_PREP(MSMON_CFG_MON_SEL_MON_SEL, i) | + FIELD_PREP(MSMON_CFG_MON_SEL_RIS, ris->ris_idx); + mpam_write_monsel_reg(msc, CFG_MON_SEL, mon_sel); + + cur_flt = mpam_read_monsel_reg(msc, CFG_MBWU_FLT); + cur_ctl = mpam_read_monsel_reg(msc, CFG_MBWU_CTL); + mpam_write_monsel_reg(msc, CFG_MBWU_CTL, 0); + + if (mpam_ris_has_mbwu_long_counter(ris)) { + val = mpam_msc_read_mbwu_l(msc); + mpam_msc_zero_mbwu_l(msc); + } else { + val = mpam_read_monsel_reg(msc, MBWU); + mpam_write_monsel_reg(msc, MBWU, 0); + } + + cfg->mon = i; + cfg->pmg = FIELD_GET(MSMON_CFG_x_FLT_PMG, cur_flt); + cfg->match_pmg = FIELD_GET(MSMON_CFG_x_CTL_MATCH_PMG, cur_ctl); + cfg->partid = FIELD_GET(MSMON_CFG_x_FLT_PARTID, cur_flt); + mbwu_state->correction += val; + mbwu_state->enabled = FIELD_GET(MSMON_CFG_x_CTL_EN, cur_ctl); + mpam_mon_sel_unlock(msc); + } + + return 0; +} + +static void mpam_init_reset_cfg(struct mpam_config *reset_cfg) +{ + *reset_cfg = (struct mpam_config) { + .reset_cpbm = true, + .reset_mbw_pbm = true, + .reset_mbw_max = true, + }; + bitmap_fill(reset_cfg->features, MPAM_FEATURE_LAST); +} + +/* + * Called via smp_call_on_cpu() to prevent migration, while still being + * pre-emptible. Caller must hold mpam_srcu. + */ +static int mpam_reset_ris(void *arg) +{ + u16 partid, partid_max; + struct mpam_config reset_cfg; + struct mpam_msc_ris *ris = arg; + + if (ris->in_reset_state) + return 0; + + mpam_init_reset_cfg(&reset_cfg); + + spin_lock(&partid_max_lock); + partid_max = mpam_partid_max; + spin_unlock(&partid_max_lock); + for (partid = 0; partid <= partid_max; partid++) + mpam_reprogram_ris_partid(ris, partid, &reset_cfg); + + return 0; +} + +/* + * Get the preferred CPU for this MSC. If it is accessible from this CPU, + * this CPU is preferred. This can be preempted/migrated, it will only result + * in more work. + */ +static int mpam_get_msc_preferred_cpu(struct mpam_msc *msc) +{ + int cpu = raw_smp_processor_id(); + + if (cpumask_test_cpu(cpu, &msc->accessibility)) + return cpu; + + return cpumask_first_and(&msc->accessibility, cpu_online_mask); +} + +static int mpam_touch_msc(struct mpam_msc *msc, int (*fn)(void *a), void *arg) +{ + lockdep_assert_irqs_enabled(); + lockdep_assert_cpus_held(); + WARN_ON_ONCE(!srcu_read_lock_held((&mpam_srcu))); + + return smp_call_on_cpu(mpam_get_msc_preferred_cpu(msc), fn, arg, true); +} + +struct mpam_write_config_arg { + struct mpam_msc_ris *ris; + struct mpam_component *comp; + u16 partid; +}; + +static int __write_config(void *arg) +{ + struct mpam_write_config_arg *c = arg; + + mpam_reprogram_ris_partid(c->ris, c->partid, &c->comp->cfg[c->partid]); + + return 0; +} + +static void mpam_reprogram_msc(struct mpam_msc *msc) +{ + u16 partid; + bool reset; + struct mpam_config *cfg; + struct mpam_msc_ris *ris; + struct mpam_write_config_arg arg; + + /* + * No lock for mpam_partid_max as partid_max_published has been + * set by mpam_enabled(), so the values can no longer change. + */ + mpam_assert_partid_sizes_fixed(); + + mutex_lock(&msc->cfg_lock); + list_for_each_entry_srcu(ris, &msc->ris, msc_list, + srcu_read_lock_held(&mpam_srcu)) { + if (!mpam_is_enabled() && !ris->in_reset_state) { + mpam_touch_msc(msc, &mpam_reset_ris, ris); + ris->in_reset_state = true; + continue; + } + + arg.comp = ris->vmsc->comp; + arg.ris = ris; + reset = true; + for (partid = 0; partid <= mpam_partid_max; partid++) { + cfg = &ris->vmsc->comp->cfg[partid]; + if (!bitmap_empty(cfg->features, MPAM_FEATURE_LAST)) + reset = false; + + arg.partid = partid; + mpam_touch_msc(msc, __write_config, &arg); + } + ris->in_reset_state = reset; + + if (mpam_has_feature(mpam_feat_msmon_mbwu, &ris->props)) + mpam_touch_msc(msc, &mpam_restore_mbwu_state, ris); + } + mutex_unlock(&msc->cfg_lock); +} + +static void _enable_percpu_irq(void *_irq) +{ + int *irq = _irq; + + enable_percpu_irq(*irq, IRQ_TYPE_NONE); +} + +static int mpam_cpu_online(unsigned int cpu) +{ + struct mpam_msc *msc; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, + srcu_read_lock_held(&mpam_srcu)) { + if (!cpumask_test_cpu(cpu, &msc->accessibility)) + continue; + + if (msc->reenable_error_ppi) + _enable_percpu_irq(&msc->reenable_error_ppi); + + if (atomic_fetch_inc(&msc->online_refs) == 0) + mpam_reprogram_msc(msc); + } + + return 0; +} + +/* Before mpam is enabled, try to probe new MSC */ +static int mpam_discovery_cpu_online(unsigned int cpu) +{ + int err = 0; + struct mpam_msc *msc; + bool new_device_probed = false; + + if (mpam_is_enabled()) + return 0; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, + srcu_read_lock_held(&mpam_srcu)) { + if (!cpumask_test_cpu(cpu, &msc->accessibility)) + continue; + + mutex_lock(&msc->probe_lock); + if (!msc->probed) + err = mpam_msc_hw_probe(msc); + mutex_unlock(&msc->probe_lock); + + if (err) + break; + new_device_probed = true; + } + + if (new_device_probed && !err) + schedule_work(&mpam_enable_work); + if (err) { + mpam_disable_reason = "error during probing"; + schedule_work(&mpam_broken_work); + } + + return err; +} + +static int mpam_cpu_offline(unsigned int cpu) +{ + struct mpam_msc *msc; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, + srcu_read_lock_held(&mpam_srcu)) { + if (!cpumask_test_cpu(cpu, &msc->accessibility)) + continue; + + if (msc->reenable_error_ppi) + disable_percpu_irq(msc->reenable_error_ppi); + + if (atomic_dec_and_test(&msc->online_refs)) { + struct mpam_msc_ris *ris; + + mutex_lock(&msc->cfg_lock); + list_for_each_entry_srcu(ris, &msc->ris, msc_list, + srcu_read_lock_held(&mpam_srcu)) { + mpam_touch_msc(msc, &mpam_reset_ris, ris); + + /* + * The reset state for non-zero partid may be + * lost while the CPUs are offline. + */ + ris->in_reset_state = false; + + if (mpam_is_enabled()) + mpam_touch_msc(msc, &mpam_save_mbwu_state, ris); + } + mutex_unlock(&msc->cfg_lock); + } + } + + return 0; +} + +static void mpam_register_cpuhp_callbacks(int (*online)(unsigned int online), + int (*offline)(unsigned int offline), + char *name) +{ + mutex_lock(&mpam_cpuhp_state_lock); + if (mpam_cpuhp_state) { + cpuhp_remove_state(mpam_cpuhp_state); + mpam_cpuhp_state = 0; + } + + mpam_cpuhp_state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, name, online, + offline); + if (mpam_cpuhp_state <= 0) { + pr_err("Failed to register cpuhp callbacks"); + mpam_cpuhp_state = 0; + } + mutex_unlock(&mpam_cpuhp_state_lock); +} + +static int __setup_ppi(struct mpam_msc *msc) +{ + int cpu; + + msc->error_dev_id = alloc_percpu(struct mpam_msc *); + if (!msc->error_dev_id) + return -ENOMEM; + + for_each_cpu(cpu, &msc->accessibility) + *per_cpu_ptr(msc->error_dev_id, cpu) = msc; + + return 0; +} + +static int mpam_msc_setup_error_irq(struct mpam_msc *msc) +{ + int irq; + + irq = platform_get_irq_byname_optional(msc->pdev, "error"); + if (irq <= 0) + return 0; + + /* Allocate and initialise the percpu device pointer for PPI */ + if (irq_is_percpu(irq)) + return __setup_ppi(msc); + + /* sanity check: shared interrupts can be routed anywhere? */ + if (!cpumask_equal(&msc->accessibility, cpu_possible_mask)) { + pr_err_once("msc:%u is a private resource with a shared error interrupt", + msc->id); + return -EINVAL; + } + + return 0; +} + +/* + * An MSC can control traffic from a set of CPUs, but may only be accessible + * from a (hopefully wider) set of CPUs. The common reason for this is power + * management. If all the CPUs in a cluster are in PSCI:CPU_SUSPEND, the + * corresponding cache may also be powered off. By making accesses from + * one of those CPUs, we ensure we don't access a cache that's powered off. + */ +static void update_msc_accessibility(struct mpam_msc *msc) +{ + u32 affinity_id; + int err; + + err = device_property_read_u32(&msc->pdev->dev, "cpu_affinity", + &affinity_id); + if (err) + cpumask_copy(&msc->accessibility, cpu_possible_mask); + else + acpi_pptt_get_cpus_from_container(affinity_id, &msc->accessibility); +} + +/* + * There are two ways of reaching a struct mpam_msc_ris. Via the + * class->component->vmsc->ris, or via the msc. + * When destroying the msc, the other side needs unlinking and cleaning up too. + */ +static void mpam_msc_destroy(struct mpam_msc *msc) +{ + struct platform_device *pdev = msc->pdev; + struct mpam_msc_ris *ris, *tmp; + + lockdep_assert_held(&mpam_list_lock); + + list_for_each_entry_safe(ris, tmp, &msc->ris, msc_list) + mpam_ris_destroy(ris); + + list_del_rcu(&msc->all_msc_list); + platform_set_drvdata(pdev, NULL); + + add_to_garbage(msc); +} + +static void mpam_msc_drv_remove(struct platform_device *pdev) +{ + struct mpam_msc *msc = platform_get_drvdata(pdev); + + mutex_lock(&mpam_list_lock); + mpam_msc_destroy(msc); + mutex_unlock(&mpam_list_lock); + + mpam_free_garbage(); +} + +static struct mpam_msc *do_mpam_msc_drv_probe(struct platform_device *pdev) +{ + int err; + u32 tmp; + struct mpam_msc *msc; + struct resource *msc_res; + struct device *dev = &pdev->dev; + + lockdep_assert_held(&mpam_list_lock); + + msc = devm_kzalloc(&pdev->dev, sizeof(*msc), GFP_KERNEL); + if (!msc) + return ERR_PTR(-ENOMEM); + init_garbage(&msc->garbage); + msc->garbage.pdev = pdev; + + err = devm_mutex_init(dev, &msc->probe_lock); + if (err) + return ERR_PTR(err); + + err = devm_mutex_init(dev, &msc->part_sel_lock); + if (err) + return ERR_PTR(err); + + err = devm_mutex_init(dev, &msc->error_irq_lock); + if (err) + return ERR_PTR(err); + + err = devm_mutex_init(dev, &msc->cfg_lock); + if (err) + return ERR_PTR(err); + + mpam_mon_sel_lock_init(msc); + msc->id = pdev->id; + msc->pdev = pdev; + INIT_LIST_HEAD_RCU(&msc->all_msc_list); + INIT_LIST_HEAD_RCU(&msc->ris); + + update_msc_accessibility(msc); + if (cpumask_empty(&msc->accessibility)) { + dev_err_once(dev, "MSC is not accessible from any CPU!"); + return ERR_PTR(-EINVAL); + } + + err = mpam_msc_setup_error_irq(msc); + if (err) + return ERR_PTR(err); + + if (device_property_read_u32(&pdev->dev, "pcc-channel", &tmp)) + msc->iface = MPAM_IFACE_MMIO; + else + msc->iface = MPAM_IFACE_PCC; + + if (msc->iface == MPAM_IFACE_MMIO) { + void __iomem *io; + + io = devm_platform_get_and_ioremap_resource(pdev, 0, + &msc_res); + if (IS_ERR(io)) { + dev_err_once(dev, "Failed to map MSC base address\n"); + return ERR_CAST(io); + } + msc->mapped_hwpage_sz = msc_res->end - msc_res->start; + msc->mapped_hwpage = io; + } else { + return ERR_PTR(-EINVAL); + } + + list_add_rcu(&msc->all_msc_list, &mpam_all_msc); + platform_set_drvdata(pdev, msc); + + return msc; +} + +static int fw_num_msc; + +static int mpam_msc_drv_probe(struct platform_device *pdev) +{ + int err; + struct mpam_msc *msc = NULL; + void *plat_data = pdev->dev.platform_data; + + mutex_lock(&mpam_list_lock); + msc = do_mpam_msc_drv_probe(pdev); + mutex_unlock(&mpam_list_lock); + + if (IS_ERR(msc)) + return PTR_ERR(msc); + + /* Create RIS entries described by firmware */ + err = acpi_mpam_parse_resources(msc, plat_data); + if (err) { + mpam_msc_drv_remove(pdev); + return err; + } + + if (atomic_add_return(1, &mpam_num_msc) == fw_num_msc) + mpam_register_cpuhp_callbacks(mpam_discovery_cpu_online, NULL, + "mpam:drv_probe"); + + return 0; +} + +static struct platform_driver mpam_msc_driver = { + .driver = { + .name = "mpam_msc", + }, + .probe = mpam_msc_drv_probe, + .remove = mpam_msc_drv_remove, +}; + +/* Any of these features mean the BWA_WD field is valid. */ +static bool mpam_has_bwa_wd_feature(struct mpam_props *props) +{ + if (mpam_has_feature(mpam_feat_mbw_min, props)) + return true; + if (mpam_has_feature(mpam_feat_mbw_max, props)) + return true; + if (mpam_has_feature(mpam_feat_mbw_prop, props)) + return true; + return false; +} + +/* Any of these features mean the CMAX_WD field is valid. */ +static bool mpam_has_cmax_wd_feature(struct mpam_props *props) +{ + if (mpam_has_feature(mpam_feat_cmax_cmax, props)) + return true; + if (mpam_has_feature(mpam_feat_cmax_cmin, props)) + return true; + return false; +} + +#define MISMATCHED_HELPER(parent, child, helper, field, alias) \ + helper(parent) && \ + ((helper(child) && (parent)->field != (child)->field) || \ + (!helper(child) && !(alias))) + +#define MISMATCHED_FEAT(parent, child, feat, field, alias) \ + mpam_has_feature((feat), (parent)) && \ + ((mpam_has_feature((feat), (child)) && (parent)->field != (child)->field) || \ + (!mpam_has_feature((feat), (child)) && !(alias))) + +#define CAN_MERGE_FEAT(parent, child, feat, alias) \ + (alias) && !mpam_has_feature((feat), (parent)) && \ + mpam_has_feature((feat), (child)) + +/* + * Combine two props fields. + * If this is for controls that alias the same resource, it is safe to just + * copy the values over. If two aliasing controls implement the same scheme + * a safe value must be picked. + * For non-aliasing controls, these control different resources, and the + * resulting safe value must be compatible with both. When merging values in + * the tree, all the aliasing resources must be handled first. + * On mismatch, parent is modified. + */ +static void __props_mismatch(struct mpam_props *parent, + struct mpam_props *child, bool alias) +{ + if (CAN_MERGE_FEAT(parent, child, mpam_feat_cpor_part, alias)) { + parent->cpbm_wd = child->cpbm_wd; + } else if (MISMATCHED_FEAT(parent, child, mpam_feat_cpor_part, + cpbm_wd, alias)) { + pr_debug("cleared cpor_part\n"); + mpam_clear_feature(mpam_feat_cpor_part, parent); + parent->cpbm_wd = 0; + } + + if (CAN_MERGE_FEAT(parent, child, mpam_feat_mbw_part, alias)) { + parent->mbw_pbm_bits = child->mbw_pbm_bits; + } else if (MISMATCHED_FEAT(parent, child, mpam_feat_mbw_part, + mbw_pbm_bits, alias)) { + pr_debug("cleared mbw_part\n"); + mpam_clear_feature(mpam_feat_mbw_part, parent); + parent->mbw_pbm_bits = 0; + } + + /* bwa_wd is a count of bits, fewer bits means less precision */ + if (alias && !mpam_has_bwa_wd_feature(parent) && + mpam_has_bwa_wd_feature(child)) { + parent->bwa_wd = child->bwa_wd; + } else if (MISMATCHED_HELPER(parent, child, mpam_has_bwa_wd_feature, + bwa_wd, alias)) { + pr_debug("took the min bwa_wd\n"); + parent->bwa_wd = min(parent->bwa_wd, child->bwa_wd); + } + + if (alias && !mpam_has_cmax_wd_feature(parent) && mpam_has_cmax_wd_feature(child)) { + parent->cmax_wd = child->cmax_wd; + } else if (MISMATCHED_HELPER(parent, child, mpam_has_cmax_wd_feature, + cmax_wd, alias)) { + pr_debug("%s took the min cmax_wd\n", __func__); + parent->cmax_wd = min(parent->cmax_wd, child->cmax_wd); + } + + if (CAN_MERGE_FEAT(parent, child, mpam_feat_cmax_cassoc, alias)) { + parent->cassoc_wd = child->cassoc_wd; + } else if (MISMATCHED_FEAT(parent, child, mpam_feat_cmax_cassoc, + cassoc_wd, alias)) { + pr_debug("%s cleared cassoc_wd\n", __func__); + mpam_clear_feature(mpam_feat_cmax_cassoc, parent); + parent->cassoc_wd = 0; + } + + /* For num properties, take the minimum */ + if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_csu, alias)) { + parent->num_csu_mon = child->num_csu_mon; + } else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_csu, + num_csu_mon, alias)) { + pr_debug("took the min num_csu_mon\n"); + parent->num_csu_mon = min(parent->num_csu_mon, + child->num_csu_mon); + } + + if (CAN_MERGE_FEAT(parent, child, mpam_feat_msmon_mbwu, alias)) { + parent->num_mbwu_mon = child->num_mbwu_mon; + } else if (MISMATCHED_FEAT(parent, child, mpam_feat_msmon_mbwu, + num_mbwu_mon, alias)) { + pr_debug("took the min num_mbwu_mon\n"); + parent->num_mbwu_mon = min(parent->num_mbwu_mon, + child->num_mbwu_mon); + } + + if (CAN_MERGE_FEAT(parent, child, mpam_feat_intpri_part, alias)) { + parent->intpri_wd = child->intpri_wd; + } else if (MISMATCHED_FEAT(parent, child, mpam_feat_intpri_part, + intpri_wd, alias)) { + pr_debug("%s took the min intpri_wd\n", __func__); + parent->intpri_wd = min(parent->intpri_wd, child->intpri_wd); + } + + if (CAN_MERGE_FEAT(parent, child, mpam_feat_dspri_part, alias)) { + parent->dspri_wd = child->dspri_wd; + } else if (MISMATCHED_FEAT(parent, child, mpam_feat_dspri_part, + dspri_wd, alias)) { + pr_debug("%s took the min dspri_wd\n", __func__); + parent->dspri_wd = min(parent->dspri_wd, child->dspri_wd); + } + + /* TODO: alias support for these two */ + /* {int,ds}pri may not have differing 0-low behaviour */ + if (mpam_has_feature(mpam_feat_intpri_part, parent) && + (!mpam_has_feature(mpam_feat_intpri_part, child) || + mpam_has_feature(mpam_feat_intpri_part_0_low, parent) != + mpam_has_feature(mpam_feat_intpri_part_0_low, child))) { + pr_debug("%s cleared intpri_part\n", __func__); + mpam_clear_feature(mpam_feat_intpri_part, parent); + mpam_clear_feature(mpam_feat_intpri_part_0_low, parent); + } + if (mpam_has_feature(mpam_feat_dspri_part, parent) && + (!mpam_has_feature(mpam_feat_dspri_part, child) || + mpam_has_feature(mpam_feat_dspri_part_0_low, parent) != + mpam_has_feature(mpam_feat_dspri_part_0_low, child))) { + pr_debug("%s cleared dspri_part\n", __func__); + mpam_clear_feature(mpam_feat_dspri_part, parent); + mpam_clear_feature(mpam_feat_dspri_part_0_low, parent); + } + + if (alias) { + /* Merge features for aliased resources */ + bitmap_or(parent->features, parent->features, child->features, MPAM_FEATURE_LAST); + } else { + /* Clear missing features for non aliasing */ + bitmap_and(parent->features, parent->features, child->features, MPAM_FEATURE_LAST); + } +} + +/* + * If a vmsc doesn't match class feature/configuration, do the right thing(tm). + * For 'num' properties we can just take the minimum. + * For properties where the mismatched unused bits would make a difference, we + * nobble the class feature, as we can't configure all the resources. + * e.g. The L3 cache is composed of two resources with 13 and 17 portion + * bitmaps respectively. + */ +static void +__class_props_mismatch(struct mpam_class *class, struct mpam_vmsc *vmsc) +{ + struct mpam_props *cprops = &class->props; + struct mpam_props *vprops = &vmsc->props; + struct device *dev = &vmsc->msc->pdev->dev; + + lockdep_assert_held(&mpam_list_lock); /* we modify class */ + + dev_dbg(dev, "Merging features for class:0x%lx &= vmsc:0x%lx\n", + (long)cprops->features, (long)vprops->features); + + /* Take the safe value for any common features */ + __props_mismatch(cprops, vprops, false); +} + +static void +__vmsc_props_mismatch(struct mpam_vmsc *vmsc, struct mpam_msc_ris *ris) +{ + struct mpam_props *rprops = &ris->props; + struct mpam_props *vprops = &vmsc->props; + struct device *dev = &vmsc->msc->pdev->dev; + + lockdep_assert_held(&mpam_list_lock); /* we modify vmsc */ + + dev_dbg(dev, "Merging features for vmsc:0x%lx |= ris:0x%lx\n", + (long)vprops->features, (long)rprops->features); + + /* + * Merge mismatched features - Copy any features that aren't common, + * but take the safe value for any common features. + */ + __props_mismatch(vprops, rprops, true); +} + +/* + * Copy the first component's first vMSC's properties and features to the + * class. __class_props_mismatch() will remove conflicts. + * It is not possible to have a class with no components, or a component with + * no resources. The vMSC properties have already been built. + */ +static void mpam_enable_init_class_features(struct mpam_class *class) +{ + struct mpam_vmsc *vmsc; + struct mpam_component *comp; + + comp = list_first_entry(&class->components, + struct mpam_component, class_list); + vmsc = list_first_entry(&comp->vmsc, + struct mpam_vmsc, comp_list); + + class->props = vmsc->props; +} + +static void mpam_enable_merge_vmsc_features(struct mpam_component *comp) +{ + struct mpam_vmsc *vmsc; + struct mpam_msc_ris *ris; + struct mpam_class *class = comp->class; + + list_for_each_entry(vmsc, &comp->vmsc, comp_list) { + list_for_each_entry(ris, &vmsc->ris, vmsc_list) { + __vmsc_props_mismatch(vmsc, ris); + class->nrdy_usec = max(class->nrdy_usec, + vmsc->msc->nrdy_usec); + } + } +} + +static void mpam_enable_merge_class_features(struct mpam_component *comp) +{ + struct mpam_vmsc *vmsc; + struct mpam_class *class = comp->class; + + list_for_each_entry(vmsc, &comp->vmsc, comp_list) + __class_props_mismatch(class, vmsc); +} + +/* + * Merge all the common resource features into class. + * vmsc features are bitwise-or'd together by mpam_enable_merge_vmsc_features() + * as the first step so that mpam_enable_init_class_features() can initialise + * the class with a representative set of features. + * Next the mpam_enable_merge_class_features() bitwise-and's all the vmsc + * features to form the class features. + * Other features are the min/max as appropriate. + * + * To avoid walking the whole tree twice, the class->nrdy_usec property is + * updated when working with the vmsc as it is a max(), and doesn't need + * initialising first. + */ +static void mpam_enable_merge_features(struct list_head *all_classes_list) +{ + struct mpam_class *class; + struct mpam_component *comp; + + lockdep_assert_held(&mpam_list_lock); + + list_for_each_entry(class, all_classes_list, classes_list) { + list_for_each_entry(comp, &class->components, class_list) + mpam_enable_merge_vmsc_features(comp); + + mpam_enable_init_class_features(class); + + list_for_each_entry(comp, &class->components, class_list) + mpam_enable_merge_class_features(comp); + } +} + +static char *mpam_errcode_names[16] = { + [MPAM_ERRCODE_NONE] = "No error", + [MPAM_ERRCODE_PARTID_SEL_RANGE] = "PARTID_SEL_Range", + [MPAM_ERRCODE_REQ_PARTID_RANGE] = "Req_PARTID_Range", + [MPAM_ERRCODE_MSMONCFG_ID_RANGE] = "MSMONCFG_ID_RANGE", + [MPAM_ERRCODE_REQ_PMG_RANGE] = "Req_PMG_Range", + [MPAM_ERRCODE_MONITOR_RANGE] = "Monitor_Range", + [MPAM_ERRCODE_INTPARTID_RANGE] = "intPARTID_Range", + [MPAM_ERRCODE_UNEXPECTED_INTERNAL] = "Unexpected_INTERNAL", + [MPAM_ERRCODE_UNDEFINED_RIS_PART_SEL] = "Undefined_RIS_PART_SEL", + [MPAM_ERRCODE_RIS_NO_CONTROL] = "RIS_No_Control", + [MPAM_ERRCODE_UNDEFINED_RIS_MON_SEL] = "Undefined_RIS_MON_SEL", + [MPAM_ERRCODE_RIS_NO_MONITOR] = "RIS_No_Monitor", + [12 ... 15] = "Reserved" +}; + +static int mpam_enable_msc_ecr(void *_msc) +{ + struct mpam_msc *msc = _msc; + + __mpam_write_reg(msc, MPAMF_ECR, MPAMF_ECR_INTEN); + + return 0; +} + +/* This can run in mpam_disable(), and the interrupt handler on the same CPU */ +static int mpam_disable_msc_ecr(void *_msc) +{ + struct mpam_msc *msc = _msc; + + __mpam_write_reg(msc, MPAMF_ECR, 0); + + return 0; +} + +static irqreturn_t __mpam_irq_handler(int irq, struct mpam_msc *msc) +{ + u64 reg; + u16 partid; + u8 errcode, pmg, ris; + + if (WARN_ON_ONCE(!msc) || + WARN_ON_ONCE(!cpumask_test_cpu(smp_processor_id(), + &msc->accessibility))) + return IRQ_NONE; + + reg = mpam_msc_read_esr(msc); + + errcode = FIELD_GET(MPAMF_ESR_ERRCODE, reg); + if (!errcode) + return IRQ_NONE; + + /* Clear level triggered irq */ + mpam_msc_clear_esr(msc); + + partid = FIELD_GET(MPAMF_ESR_PARTID_MON, reg); + pmg = FIELD_GET(MPAMF_ESR_PMG, reg); + ris = FIELD_GET(MPAMF_ESR_RIS, reg); + + pr_err_ratelimited("error irq from msc:%u '%s', partid:%u, pmg: %u, ris: %u\n", + msc->id, mpam_errcode_names[errcode], partid, pmg, + ris); + + /* Disable this interrupt. */ + mpam_disable_msc_ecr(msc); + + /* Are we racing with the thread disabling MPAM? */ + if (!mpam_is_enabled()) + return IRQ_HANDLED; + + /* + * Schedule the teardown work. Don't use a threaded IRQ as we can't + * unregister the interrupt from the threaded part of the handler. + */ + mpam_disable_reason = "hardware error interrupt"; + schedule_work(&mpam_broken_work); + + return IRQ_HANDLED; +} + +static irqreturn_t mpam_ppi_handler(int irq, void *dev_id) +{ + struct mpam_msc *msc = *(struct mpam_msc **)dev_id; + + return __mpam_irq_handler(irq, msc); +} + +static irqreturn_t mpam_spi_handler(int irq, void *dev_id) +{ + struct mpam_msc *msc = dev_id; + + return __mpam_irq_handler(irq, msc); +} + +static int mpam_register_irqs(void) +{ + int err, irq; + struct mpam_msc *msc; + + lockdep_assert_cpus_held(); + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, + srcu_read_lock_held(&mpam_srcu)) { + irq = platform_get_irq_byname_optional(msc->pdev, "error"); + if (irq <= 0) + continue; + + /* The MPAM spec says the interrupt can be SPI, PPI or LPI */ + /* We anticipate sharing the interrupt with other MSCs */ + if (irq_is_percpu(irq)) { + err = request_percpu_irq(irq, &mpam_ppi_handler, + "mpam:msc:error", + msc->error_dev_id); + if (err) + return err; + + msc->reenable_error_ppi = irq; + smp_call_function_many(&msc->accessibility, + &_enable_percpu_irq, &irq, + true); + } else { + err = devm_request_irq(&msc->pdev->dev, irq, + &mpam_spi_handler, IRQF_SHARED, + "mpam:msc:error", msc); + if (err) + return err; + } + + mutex_lock(&msc->error_irq_lock); + msc->error_irq_req = true; + mpam_touch_msc(msc, mpam_enable_msc_ecr, msc); + msc->error_irq_hw_enabled = true; + mutex_unlock(&msc->error_irq_lock); + } + + return 0; +} + +static void mpam_unregister_irqs(void) +{ + int irq; + struct mpam_msc *msc; + + guard(cpus_read_lock)(); + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, + srcu_read_lock_held(&mpam_srcu)) { + irq = platform_get_irq_byname_optional(msc->pdev, "error"); + if (irq <= 0) + continue; + + mutex_lock(&msc->error_irq_lock); + if (msc->error_irq_hw_enabled) { + mpam_touch_msc(msc, mpam_disable_msc_ecr, msc); + msc->error_irq_hw_enabled = false; + } + + if (msc->error_irq_req) { + if (irq_is_percpu(irq)) { + msc->reenable_error_ppi = 0; + free_percpu_irq(irq, msc->error_dev_id); + } else { + devm_free_irq(&msc->pdev->dev, irq, msc); + } + msc->error_irq_req = false; + } + mutex_unlock(&msc->error_irq_lock); + } +} + +static void __destroy_component_cfg(struct mpam_component *comp) +{ + struct mpam_msc *msc; + struct mpam_vmsc *vmsc; + struct mpam_msc_ris *ris; + + lockdep_assert_held(&mpam_list_lock); + + add_to_garbage(comp->cfg); + list_for_each_entry(vmsc, &comp->vmsc, comp_list) { + msc = vmsc->msc; + + if (mpam_mon_sel_lock(msc)) { + list_for_each_entry(ris, &vmsc->ris, vmsc_list) + add_to_garbage(ris->mbwu_state); + mpam_mon_sel_unlock(msc); + } + } +} + +static void mpam_reset_component_cfg(struct mpam_component *comp) +{ + int i; + struct mpam_props *cprops = &comp->class->props; + + mpam_assert_partid_sizes_fixed(); + + if (!comp->cfg) + return; + + for (i = 0; i <= mpam_partid_max; i++) { + comp->cfg[i] = (struct mpam_config) {}; + if (cprops->cpbm_wd) + comp->cfg[i].cpbm = GENMASK(cprops->cpbm_wd - 1, 0); + if (cprops->mbw_pbm_bits) + comp->cfg[i].mbw_pbm = GENMASK(cprops->mbw_pbm_bits - 1, 0); + if (cprops->bwa_wd) + comp->cfg[i].mbw_max = GENMASK(15, 16 - cprops->bwa_wd); + } +} + +static int __allocate_component_cfg(struct mpam_component *comp) +{ + struct mpam_vmsc *vmsc; + + mpam_assert_partid_sizes_fixed(); + + if (comp->cfg) + return 0; + + comp->cfg = kcalloc(mpam_partid_max + 1, sizeof(*comp->cfg), GFP_KERNEL); + if (!comp->cfg) + return -ENOMEM; + + /* + * The array is free()d in one go, so only cfg[0]'s structure needs + * to be initialised. + */ + init_garbage(&comp->cfg[0].garbage); + + mpam_reset_component_cfg(comp); + + list_for_each_entry(vmsc, &comp->vmsc, comp_list) { + struct mpam_msc *msc; + struct mpam_msc_ris *ris; + struct msmon_mbwu_state *mbwu_state; + + if (!vmsc->props.num_mbwu_mon) + continue; + + msc = vmsc->msc; + list_for_each_entry(ris, &vmsc->ris, vmsc_list) { + if (!ris->props.num_mbwu_mon) + continue; + + mbwu_state = kcalloc(ris->props.num_mbwu_mon, + sizeof(*ris->mbwu_state), + GFP_KERNEL); + if (!mbwu_state) { + __destroy_component_cfg(comp); + return -ENOMEM; + } + + init_garbage(&mbwu_state[0].garbage); + + if (mpam_mon_sel_lock(msc)) { + ris->mbwu_state = mbwu_state; + mpam_mon_sel_unlock(msc); + } + } + } + + return 0; +} + +static int mpam_allocate_config(void) +{ + struct mpam_class *class; + struct mpam_component *comp; + + lockdep_assert_held(&mpam_list_lock); + + list_for_each_entry(class, &mpam_classes, classes_list) { + list_for_each_entry(comp, &class->components, class_list) { + int err = __allocate_component_cfg(comp); + if (err) + return err; + } + } + + return 0; +} + +static void mpam_enable_once(void) +{ + int err; + + /* + * Once the cpuhp callbacks have been changed, mpam_partid_max can no + * longer change. + */ + spin_lock(&partid_max_lock); + partid_max_published = true; + spin_unlock(&partid_max_lock); + + /* + * If all the MSC have been probed, enabling the IRQs happens next. + * That involves cross-calling to a CPU that can reach the MSC, and + * the locks must be taken in this order: + */ + cpus_read_lock(); + mutex_lock(&mpam_list_lock); + do { + mpam_enable_merge_features(&mpam_classes); + + err = mpam_register_irqs(); + if (err) { + pr_warn("Failed to register irqs: %d\n", err); + break; + } + + err = mpam_allocate_config(); + if (err) { + pr_err("Failed to allocate configuration arrays.\n"); + break; + } + } while (0); + mutex_unlock(&mpam_list_lock); + cpus_read_unlock(); + + if (err) { + mpam_disable_reason = "Failed to enable."; + schedule_work(&mpam_broken_work); + return; + } + + static_branch_enable(&mpam_enabled); + mpam_register_cpuhp_callbacks(mpam_cpu_online, mpam_cpu_offline, + "mpam:online"); + + /* Use printk() to avoid the pr_fmt adding the function name. */ + printk(KERN_INFO "MPAM enabled with %u PARTIDs and %u PMGs\n", + mpam_partid_max + 1, mpam_pmg_max + 1); +} + +static void mpam_reset_component_locked(struct mpam_component *comp) +{ + struct mpam_vmsc *vmsc; + + lockdep_assert_cpus_held(); + mpam_assert_partid_sizes_fixed(); + + mpam_reset_component_cfg(comp); + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, + srcu_read_lock_held(&mpam_srcu)) { + struct mpam_msc *msc = vmsc->msc; + struct mpam_msc_ris *ris; + + list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, + srcu_read_lock_held(&mpam_srcu)) { + if (!ris->in_reset_state) + mpam_touch_msc(msc, mpam_reset_ris, ris); + ris->in_reset_state = true; + } + } +} + +static void mpam_reset_class_locked(struct mpam_class *class) +{ + struct mpam_component *comp; + + lockdep_assert_cpus_held(); + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(comp, &class->components, class_list, + srcu_read_lock_held(&mpam_srcu)) + mpam_reset_component_locked(comp); +} + +static void mpam_reset_class(struct mpam_class *class) +{ + cpus_read_lock(); + mpam_reset_class_locked(class); + cpus_read_unlock(); +} + +/* + * Called in response to an error IRQ. + * All of MPAMs errors indicate a software bug, restore any modified + * controls to their reset values. + */ +void mpam_disable(struct work_struct *ignored) +{ + int idx; + struct mpam_class *class; + struct mpam_msc *msc, *tmp; + + mutex_lock(&mpam_cpuhp_state_lock); + if (mpam_cpuhp_state) { + cpuhp_remove_state(mpam_cpuhp_state); + mpam_cpuhp_state = 0; + } + mutex_unlock(&mpam_cpuhp_state_lock); + + static_branch_disable(&mpam_enabled); + + mpam_unregister_irqs(); + + idx = srcu_read_lock(&mpam_srcu); + list_for_each_entry_srcu(class, &mpam_classes, classes_list, + srcu_read_lock_held(&mpam_srcu)) + mpam_reset_class(class); + srcu_read_unlock(&mpam_srcu, idx); + + mutex_lock(&mpam_list_lock); + list_for_each_entry_safe(msc, tmp, &mpam_all_msc, all_msc_list) + mpam_msc_destroy(msc); + mutex_unlock(&mpam_list_lock); + mpam_free_garbage(); + + pr_err_once("MPAM disabled due to %s\n", mpam_disable_reason); +} + +/* + * Enable mpam once all devices have been probed. + * Scheduled by mpam_discovery_cpu_online() once all devices have been created. + * Also scheduled when new devices are probed when new CPUs come online. + */ +void mpam_enable(struct work_struct *work) +{ + static atomic_t once; + struct mpam_msc *msc; + bool all_devices_probed = true; + + /* Have we probed all the hw devices? */ + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(msc, &mpam_all_msc, all_msc_list, + srcu_read_lock_held(&mpam_srcu)) { + mutex_lock(&msc->probe_lock); + if (!msc->probed) + all_devices_probed = false; + mutex_unlock(&msc->probe_lock); + + if (!all_devices_probed) + break; + } + + if (all_devices_probed && !atomic_fetch_inc(&once)) + mpam_enable_once(); +} + +#define maybe_update_config(cfg, feature, newcfg, member, changes) do { \ + if (mpam_has_feature(feature, newcfg) && \ + (newcfg)->member != (cfg)->member) { \ + (cfg)->member = (newcfg)->member; \ + mpam_set_feature(feature, cfg); \ + \ + (changes) = true; \ + } \ +} while (0) + +static bool mpam_update_config(struct mpam_config *cfg, + const struct mpam_config *newcfg) +{ + bool has_changes = false; + + maybe_update_config(cfg, mpam_feat_cpor_part, newcfg, cpbm, has_changes); + maybe_update_config(cfg, mpam_feat_mbw_part, newcfg, mbw_pbm, has_changes); + maybe_update_config(cfg, mpam_feat_mbw_max, newcfg, mbw_max, has_changes); + + return has_changes; +} + +int mpam_apply_config(struct mpam_component *comp, u16 partid, + struct mpam_config *cfg) +{ + struct mpam_write_config_arg arg; + struct mpam_msc_ris *ris; + struct mpam_vmsc *vmsc; + struct mpam_msc *msc; + + lockdep_assert_cpus_held(); + + /* Don't pass in the current config! */ + WARN_ON_ONCE(&comp->cfg[partid] == cfg); + + if (!mpam_update_config(&comp->cfg[partid], cfg)) + return 0; + + arg.comp = comp; + arg.partid = partid; + + guard(srcu)(&mpam_srcu); + list_for_each_entry_srcu(vmsc, &comp->vmsc, comp_list, + srcu_read_lock_held(&mpam_srcu)) { + msc = vmsc->msc; + + mutex_lock(&msc->cfg_lock); + list_for_each_entry_srcu(ris, &vmsc->ris, vmsc_list, + srcu_read_lock_held(&mpam_srcu)) { + arg.ris = ris; + mpam_touch_msc(msc, __write_config, &arg); + } + mutex_unlock(&msc->cfg_lock); + } + + return 0; +} + +static int __init mpam_msc_driver_init(void) +{ + if (!system_supports_mpam()) + return -EOPNOTSUPP; + + init_srcu_struct(&mpam_srcu); + + fw_num_msc = acpi_mpam_count_msc(); + if (fw_num_msc <= 0) { + pr_err("No MSC devices found in firmware\n"); + return -EINVAL; + } + + return platform_driver_register(&mpam_msc_driver); +} + +/* Must occur after arm64_mpam_register_cpus() from arch_initcall() */ +subsys_initcall(mpam_msc_driver_init); + +#ifdef CONFIG_MPAM_KUNIT_TEST +#include "test_mpam_devices.c" +#endif diff --git a/drivers/resctrl/mpam_internal.h b/drivers/resctrl/mpam_internal.h new file mode 100644 index 000000000000..e79c3c47259c --- /dev/null +++ b/drivers/resctrl/mpam_internal.h @@ -0,0 +1,658 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +// Copyright (C) 2025 Arm Ltd. + +#ifndef MPAM_INTERNAL_H +#define MPAM_INTERNAL_H + +#include <linux/arm_mpam.h> +#include <linux/atomic.h> +#include <linux/bitmap.h> +#include <linux/cpumask.h> +#include <linux/io.h> +#include <linux/jump_label.h> +#include <linux/llist.h> +#include <linux/mutex.h> +#include <linux/srcu.h> +#include <linux/spinlock.h> +#include <linux/srcu.h> +#include <linux/types.h> + +#define MPAM_MSC_MAX_NUM_RIS 16 + +struct platform_device; + +DECLARE_STATIC_KEY_FALSE(mpam_enabled); + +#ifdef CONFIG_MPAM_KUNIT_TEST +#define PACKED_FOR_KUNIT __packed +#else +#define PACKED_FOR_KUNIT +#endif + +static inline bool mpam_is_enabled(void) +{ + return static_branch_likely(&mpam_enabled); +} + +/* + * Structures protected by SRCU may not be freed for a surprising amount of + * time (especially if perf is running). To ensure the MPAM error interrupt can + * tear down all the structures, build a list of objects that can be garbage + * collected once synchronize_srcu() has returned. + * If pdev is non-NULL, use devm_kfree(). + */ +struct mpam_garbage { + /* member of mpam_garbage */ + struct llist_node llist; + + void *to_free; + struct platform_device *pdev; +}; + +struct mpam_msc { + /* member of mpam_all_msc */ + struct list_head all_msc_list; + + int id; + struct platform_device *pdev; + + /* Not modified after mpam_is_enabled() becomes true */ + enum mpam_msc_iface iface; + u32 nrdy_usec; + cpumask_t accessibility; + bool has_extd_esr; + + int reenable_error_ppi; + struct mpam_msc * __percpu *error_dev_id; + + atomic_t online_refs; + + /* + * probe_lock is only taken during discovery. After discovery these + * properties become read-only and the lists are protected by SRCU. + */ + struct mutex probe_lock; + bool probed; + u16 partid_max; + u8 pmg_max; + unsigned long ris_idxs; + u32 ris_max; + + /* + * error_irq_lock is taken when registering/unregistering the error + * interrupt and maniupulating the below flags. + */ + struct mutex error_irq_lock; + bool error_irq_req; + bool error_irq_hw_enabled; + + /* mpam_msc_ris of this component */ + struct list_head ris; + + /* + * part_sel_lock protects access to the MSC hardware registers that are + * affected by MPAMCFG_PART_SEL. (including the ID registers that vary + * by RIS). + * If needed, take msc->probe_lock first. + */ + struct mutex part_sel_lock; + + /* + * cfg_lock protects the msc configuration and guards against mbwu_state + * save and restore racing. + */ + struct mutex cfg_lock; + + /* + * mon_sel_lock protects access to the MSC hardware registers that are + * affected by MPAMCFG_MON_SEL, and the mbwu_state. + * Access to mon_sel is needed from both process and interrupt contexts, + * but is complicated by firmware-backed platforms that can't make any + * access unless they can sleep. + * Always use the mpam_mon_sel_lock() helpers. + * Accesses to mon_sel need to be able to fail if they occur in the wrong + * context. + * If needed, take msc->probe_lock first. + */ + raw_spinlock_t _mon_sel_lock; + unsigned long _mon_sel_flags; + + void __iomem *mapped_hwpage; + size_t mapped_hwpage_sz; + + struct mpam_garbage garbage; +}; + +/* Returning false here means accesses to mon_sel must fail and report an error. */ +static inline bool __must_check mpam_mon_sel_lock(struct mpam_msc *msc) +{ + /* Locking will require updating to support a firmware backed interface */ + if (WARN_ON_ONCE(msc->iface != MPAM_IFACE_MMIO)) + return false; + + raw_spin_lock_irqsave(&msc->_mon_sel_lock, msc->_mon_sel_flags); + return true; +} + +static inline void mpam_mon_sel_unlock(struct mpam_msc *msc) +{ + raw_spin_unlock_irqrestore(&msc->_mon_sel_lock, msc->_mon_sel_flags); +} + +static inline void mpam_mon_sel_lock_held(struct mpam_msc *msc) +{ + lockdep_assert_held_once(&msc->_mon_sel_lock); +} + +static inline void mpam_mon_sel_lock_init(struct mpam_msc *msc) +{ + raw_spin_lock_init(&msc->_mon_sel_lock); +} + +/* Bits for mpam features bitmaps */ +enum mpam_device_features { + mpam_feat_cpor_part, + mpam_feat_cmax_softlim, + mpam_feat_cmax_cmax, + mpam_feat_cmax_cmin, + mpam_feat_cmax_cassoc, + mpam_feat_mbw_part, + mpam_feat_mbw_min, + mpam_feat_mbw_max, + mpam_feat_mbw_prop, + mpam_feat_intpri_part, + mpam_feat_intpri_part_0_low, + mpam_feat_dspri_part, + mpam_feat_dspri_part_0_low, + mpam_feat_msmon, + mpam_feat_msmon_csu, + mpam_feat_msmon_csu_capture, + mpam_feat_msmon_csu_xcl, + mpam_feat_msmon_csu_hw_nrdy, + mpam_feat_msmon_mbwu, + mpam_feat_msmon_mbwu_31counter, + mpam_feat_msmon_mbwu_44counter, + mpam_feat_msmon_mbwu_63counter, + mpam_feat_msmon_mbwu_capture, + mpam_feat_msmon_mbwu_rwbw, + mpam_feat_msmon_mbwu_hw_nrdy, + mpam_feat_partid_nrw, + MPAM_FEATURE_LAST +}; + +struct mpam_props { + DECLARE_BITMAP(features, MPAM_FEATURE_LAST); + + u16 cpbm_wd; + u16 mbw_pbm_bits; + u16 bwa_wd; + u16 cmax_wd; + u16 cassoc_wd; + u16 intpri_wd; + u16 dspri_wd; + u16 num_csu_mon; + u16 num_mbwu_mon; + +/* + * Kunit tests use memset() to set up feature combinations that should be + * removed, and will false-positive if the compiler introduces padding that + * isn't cleared during sanitisation. + */ +} PACKED_FOR_KUNIT; + +#define mpam_has_feature(_feat, x) test_bit(_feat, (x)->features) +#define mpam_set_feature(_feat, x) set_bit(_feat, (x)->features) +#define mpam_clear_feature(_feat, x) clear_bit(_feat, (x)->features) + +/* The values for MSMON_CFG_MBWU_FLT.RWBW */ +enum mon_filter_options { + COUNT_BOTH = 0, + COUNT_WRITE = 1, + COUNT_READ = 2, +}; + +struct mon_cfg { + u16 mon; + u8 pmg; + bool match_pmg; + bool csu_exclude_clean; + u32 partid; + enum mon_filter_options opts; +}; + +/* Changes to msmon_mbwu_state are protected by the msc's mon_sel_lock. */ +struct msmon_mbwu_state { + bool enabled; + bool reset_on_next_read; + struct mon_cfg cfg; + + /* + * The value to add to the new reading to account for power management, + * and overflow. + */ + u64 correction; + + struct mpam_garbage garbage; +}; + +struct mpam_class { + /* mpam_components in this class */ + struct list_head components; + + cpumask_t affinity; + + struct mpam_props props; + u32 nrdy_usec; + u8 level; + enum mpam_class_types type; + + /* member of mpam_classes */ + struct list_head classes_list; + + struct ida ida_csu_mon; + struct ida ida_mbwu_mon; + + struct mpam_garbage garbage; +}; + +struct mpam_config { + /* Which configuration values are valid. */ + DECLARE_BITMAP(features, MPAM_FEATURE_LAST); + + u32 cpbm; + u32 mbw_pbm; + u16 mbw_max; + + bool reset_cpbm; + bool reset_mbw_pbm; + bool reset_mbw_max; + + struct mpam_garbage garbage; +}; + +struct mpam_component { + u32 comp_id; + + /* mpam_vmsc in this component */ + struct list_head vmsc; + + cpumask_t affinity; + + /* + * Array of configuration values, indexed by partid. + * Read from cpuhp callbacks, hold the cpuhp lock when writing. + */ + struct mpam_config *cfg; + + /* member of mpam_class:components */ + struct list_head class_list; + + /* parent: */ + struct mpam_class *class; + + struct mpam_garbage garbage; +}; + +struct mpam_vmsc { + /* member of mpam_component:vmsc_list */ + struct list_head comp_list; + + /* mpam_msc_ris in this vmsc */ + struct list_head ris; + + struct mpam_props props; + + /* All RIS in this vMSC are members of this MSC */ + struct mpam_msc *msc; + + /* parent: */ + struct mpam_component *comp; + + struct mpam_garbage garbage; +}; + +struct mpam_msc_ris { + u8 ris_idx; + u64 idr; + struct mpam_props props; + bool in_reset_state; + + cpumask_t affinity; + + /* member of mpam_vmsc:ris */ + struct list_head vmsc_list; + + /* member of mpam_msc:ris */ + struct list_head msc_list; + + /* parent: */ + struct mpam_vmsc *vmsc; + + /* msmon mbwu configuration is preserved over reset */ + struct msmon_mbwu_state *mbwu_state; + + struct mpam_garbage garbage; +}; + +static inline int mpam_alloc_csu_mon(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_msmon_csu, cprops)) + return -EOPNOTSUPP; + + return ida_alloc_max(&class->ida_csu_mon, cprops->num_csu_mon - 1, + GFP_KERNEL); +} + +static inline void mpam_free_csu_mon(struct mpam_class *class, int csu_mon) +{ + ida_free(&class->ida_csu_mon, csu_mon); +} + +static inline int mpam_alloc_mbwu_mon(struct mpam_class *class) +{ + struct mpam_props *cprops = &class->props; + + if (!mpam_has_feature(mpam_feat_msmon_mbwu, cprops)) + return -EOPNOTSUPP; + + return ida_alloc_max(&class->ida_mbwu_mon, cprops->num_mbwu_mon - 1, + GFP_KERNEL); +} + +static inline void mpam_free_mbwu_mon(struct mpam_class *class, int mbwu_mon) +{ + ida_free(&class->ida_mbwu_mon, mbwu_mon); +} + +/* List of all classes - protected by srcu*/ +extern struct srcu_struct mpam_srcu; +extern struct list_head mpam_classes; + +/* System wide partid/pmg values */ +extern u16 mpam_partid_max; +extern u8 mpam_pmg_max; + +/* Scheduled work callback to enable mpam once all MSC have been probed */ +void mpam_enable(struct work_struct *work); +void mpam_disable(struct work_struct *work); + +int mpam_apply_config(struct mpam_component *comp, u16 partid, + struct mpam_config *cfg); + +int mpam_msmon_read(struct mpam_component *comp, struct mon_cfg *ctx, + enum mpam_device_features, u64 *val); +void mpam_msmon_reset_mbwu(struct mpam_component *comp, struct mon_cfg *ctx); + +int mpam_get_cpumask_from_cache_id(unsigned long cache_id, u32 cache_level, + cpumask_t *affinity); + +/* + * MPAM MSCs have the following register layout. See: + * Arm Memory System Resource Partitioning and Monitoring (MPAM) System + * Component Specification. + * https://developer.arm.com/documentation/ihi0099/aa/ + */ +#define MPAM_ARCHITECTURE_V1 0x10 + +/* Memory mapped control pages */ +/* ID Register offsets in the memory mapped page */ +#define MPAMF_IDR 0x0000 /* features id register */ +#define MPAMF_IIDR 0x0018 /* implementer id register */ +#define MPAMF_AIDR 0x0020 /* architectural id register */ +#define MPAMF_IMPL_IDR 0x0028 /* imp-def partitioning */ +#define MPAMF_CPOR_IDR 0x0030 /* cache-portion partitioning */ +#define MPAMF_CCAP_IDR 0x0038 /* cache-capacity partitioning */ +#define MPAMF_MBW_IDR 0x0040 /* mem-bw partitioning */ +#define MPAMF_PRI_IDR 0x0048 /* priority partitioning */ +#define MPAMF_MSMON_IDR 0x0080 /* performance monitoring features */ +#define MPAMF_CSUMON_IDR 0x0088 /* cache-usage monitor */ +#define MPAMF_MBWUMON_IDR 0x0090 /* mem-bw usage monitor */ +#define MPAMF_PARTID_NRW_IDR 0x0050 /* partid-narrowing */ + +/* Configuration and Status Register offsets in the memory mapped page */ +#define MPAMCFG_PART_SEL 0x0100 /* partid to configure */ +#define MPAMCFG_CPBM 0x1000 /* cache-portion config */ +#define MPAMCFG_CMAX 0x0108 /* cache-capacity config */ +#define MPAMCFG_CMIN 0x0110 /* cache-capacity config */ +#define MPAMCFG_CASSOC 0x0118 /* cache-associativity config */ +#define MPAMCFG_MBW_MIN 0x0200 /* min mem-bw config */ +#define MPAMCFG_MBW_MAX 0x0208 /* max mem-bw config */ +#define MPAMCFG_MBW_WINWD 0x0220 /* mem-bw accounting window config */ +#define MPAMCFG_MBW_PBM 0x2000 /* mem-bw portion bitmap config */ +#define MPAMCFG_PRI 0x0400 /* priority partitioning config */ +#define MPAMCFG_MBW_PROP 0x0500 /* mem-bw stride config */ +#define MPAMCFG_INTPARTID 0x0600 /* partid-narrowing config */ + +#define MSMON_CFG_MON_SEL 0x0800 /* monitor selector */ +#define MSMON_CFG_CSU_FLT 0x0810 /* cache-usage monitor filter */ +#define MSMON_CFG_CSU_CTL 0x0818 /* cache-usage monitor config */ +#define MSMON_CFG_MBWU_FLT 0x0820 /* mem-bw monitor filter */ +#define MSMON_CFG_MBWU_CTL 0x0828 /* mem-bw monitor config */ +#define MSMON_CSU 0x0840 /* current cache-usage */ +#define MSMON_CSU_CAPTURE 0x0848 /* last cache-usage value captured */ +#define MSMON_MBWU 0x0860 /* current mem-bw usage value */ +#define MSMON_MBWU_CAPTURE 0x0868 /* last mem-bw value captured */ +#define MSMON_MBWU_L 0x0880 /* current long mem-bw usage value */ +#define MSMON_MBWU_L_CAPTURE 0x0890 /* last long mem-bw value captured */ +#define MSMON_CAPT_EVNT 0x0808 /* signal a capture event */ +#define MPAMF_ESR 0x00F8 /* error status register */ +#define MPAMF_ECR 0x00F0 /* error control register */ + +/* MPAMF_IDR - MPAM features ID register */ +#define MPAMF_IDR_PARTID_MAX GENMASK(15, 0) +#define MPAMF_IDR_PMG_MAX GENMASK(23, 16) +#define MPAMF_IDR_HAS_CCAP_PART BIT(24) +#define MPAMF_IDR_HAS_CPOR_PART BIT(25) +#define MPAMF_IDR_HAS_MBW_PART BIT(26) +#define MPAMF_IDR_HAS_PRI_PART BIT(27) +#define MPAMF_IDR_EXT BIT(28) +#define MPAMF_IDR_HAS_IMPL_IDR BIT(29) +#define MPAMF_IDR_HAS_MSMON BIT(30) +#define MPAMF_IDR_HAS_PARTID_NRW BIT(31) +#define MPAMF_IDR_HAS_RIS BIT(32) +#define MPAMF_IDR_HAS_EXTD_ESR BIT(38) +#define MPAMF_IDR_HAS_ESR BIT(39) +#define MPAMF_IDR_RIS_MAX GENMASK(59, 56) + +/* MPAMF_MSMON_IDR - MPAM performance monitoring ID register */ +#define MPAMF_MSMON_IDR_MSMON_CSU BIT(16) +#define MPAMF_MSMON_IDR_MSMON_MBWU BIT(17) +#define MPAMF_MSMON_IDR_HAS_LOCAL_CAPT_EVNT BIT(31) + +/* MPAMF_CPOR_IDR - MPAM features cache portion partitioning ID register */ +#define MPAMF_CPOR_IDR_CPBM_WD GENMASK(15, 0) + +/* MPAMF_CCAP_IDR - MPAM features cache capacity partitioning ID register */ +#define MPAMF_CCAP_IDR_CMAX_WD GENMASK(5, 0) +#define MPAMF_CCAP_IDR_CASSOC_WD GENMASK(12, 8) +#define MPAMF_CCAP_IDR_HAS_CASSOC BIT(28) +#define MPAMF_CCAP_IDR_HAS_CMIN BIT(29) +#define MPAMF_CCAP_IDR_NO_CMAX BIT(30) +#define MPAMF_CCAP_IDR_HAS_CMAX_SOFTLIM BIT(31) + +/* MPAMF_MBW_IDR - MPAM features memory bandwidth partitioning ID register */ +#define MPAMF_MBW_IDR_BWA_WD GENMASK(5, 0) +#define MPAMF_MBW_IDR_HAS_MIN BIT(10) +#define MPAMF_MBW_IDR_HAS_MAX BIT(11) +#define MPAMF_MBW_IDR_HAS_PBM BIT(12) +#define MPAMF_MBW_IDR_HAS_PROP BIT(13) +#define MPAMF_MBW_IDR_WINDWR BIT(14) +#define MPAMF_MBW_IDR_BWPBM_WD GENMASK(28, 16) + +/* MPAMF_PRI_IDR - MPAM features priority partitioning ID register */ +#define MPAMF_PRI_IDR_HAS_INTPRI BIT(0) +#define MPAMF_PRI_IDR_INTPRI_0_IS_LOW BIT(1) +#define MPAMF_PRI_IDR_INTPRI_WD GENMASK(9, 4) +#define MPAMF_PRI_IDR_HAS_DSPRI BIT(16) +#define MPAMF_PRI_IDR_DSPRI_0_IS_LOW BIT(17) +#define MPAMF_PRI_IDR_DSPRI_WD GENMASK(25, 20) + +/* MPAMF_CSUMON_IDR - MPAM cache storage usage monitor ID register */ +#define MPAMF_CSUMON_IDR_NUM_MON GENMASK(15, 0) +#define MPAMF_CSUMON_IDR_HAS_OFLOW_CAPT BIT(24) +#define MPAMF_CSUMON_IDR_HAS_CEVNT_OFLW BIT(25) +#define MPAMF_CSUMON_IDR_HAS_OFSR BIT(26) +#define MPAMF_CSUMON_IDR_HAS_OFLOW_LNKG BIT(27) +#define MPAMF_CSUMON_IDR_HAS_XCL BIT(29) +#define MPAMF_CSUMON_IDR_CSU_RO BIT(30) +#define MPAMF_CSUMON_IDR_HAS_CAPTURE BIT(31) + +/* MPAMF_MBWUMON_IDR - MPAM memory bandwidth usage monitor ID register */ +#define MPAMF_MBWUMON_IDR_NUM_MON GENMASK(15, 0) +#define MPAMF_MBWUMON_IDR_HAS_RWBW BIT(28) +#define MPAMF_MBWUMON_IDR_LWD BIT(29) +#define MPAMF_MBWUMON_IDR_HAS_LONG BIT(30) +#define MPAMF_MBWUMON_IDR_HAS_CAPTURE BIT(31) + +/* MPAMF_PARTID_NRW_IDR - MPAM PARTID narrowing ID register */ +#define MPAMF_PARTID_NRW_IDR_INTPARTID_MAX GENMASK(15, 0) + +/* MPAMF_IIDR - MPAM implementation ID register */ +#define MPAMF_IIDR_IMPLEMENTER GENMASK(11, 0) +#define MPAMF_IIDR_REVISION GENMASK(15, 12) +#define MPAMF_IIDR_VARIANT GENMASK(19, 16) +#define MPAMF_IIDR_PRODUCTID GENMASK(31, 20) + +/* MPAMF_AIDR - MPAM architecture ID register */ +#define MPAMF_AIDR_ARCH_MINOR_REV GENMASK(3, 0) +#define MPAMF_AIDR_ARCH_MAJOR_REV GENMASK(7, 4) + +/* MPAMCFG_PART_SEL - MPAM partition configuration selection register */ +#define MPAMCFG_PART_SEL_PARTID_SEL GENMASK(15, 0) +#define MPAMCFG_PART_SEL_INTERNAL BIT(16) +#define MPAMCFG_PART_SEL_RIS GENMASK(27, 24) + +/* MPAMCFG_CASSOC - MPAM cache maximum associativity partition configuration register */ +#define MPAMCFG_CASSOC_CASSOC GENMASK(15, 0) + +/* MPAMCFG_CMAX - MPAM cache capacity configuration register */ +#define MPAMCFG_CMAX_SOFTLIM BIT(31) +#define MPAMCFG_CMAX_CMAX GENMASK(15, 0) + +/* MPAMCFG_CMIN - MPAM cache capacity configuration register */ +#define MPAMCFG_CMIN_CMIN GENMASK(15, 0) + +/* + * MPAMCFG_MBW_MIN - MPAM memory minimum bandwidth partitioning configuration + * register + */ +#define MPAMCFG_MBW_MIN_MIN GENMASK(15, 0) + +/* + * MPAMCFG_MBW_MAX - MPAM memory maximum bandwidth partitioning configuration + * register + */ +#define MPAMCFG_MBW_MAX_MAX GENMASK(15, 0) +#define MPAMCFG_MBW_MAX_HARDLIM BIT(31) + +/* + * MPAMCFG_MBW_WINWD - MPAM memory bandwidth partitioning window width + * register + */ +#define MPAMCFG_MBW_WINWD_US_FRAC GENMASK(7, 0) +#define MPAMCFG_MBW_WINWD_US_INT GENMASK(23, 8) + +/* MPAMCFG_PRI - MPAM priority partitioning configuration register */ +#define MPAMCFG_PRI_INTPRI GENMASK(15, 0) +#define MPAMCFG_PRI_DSPRI GENMASK(31, 16) + +/* + * MPAMCFG_MBW_PROP - Memory bandwidth proportional stride partitioning + * configuration register + */ +#define MPAMCFG_MBW_PROP_STRIDEM1 GENMASK(15, 0) +#define MPAMCFG_MBW_PROP_EN BIT(31) + +/* + * MPAMCFG_INTPARTID - MPAM internal partition narrowing configuration register + */ +#define MPAMCFG_INTPARTID_INTPARTID GENMASK(15, 0) +#define MPAMCFG_INTPARTID_INTERNAL BIT(16) + +/* MSMON_CFG_MON_SEL - Memory system performance monitor selection register */ +#define MSMON_CFG_MON_SEL_MON_SEL GENMASK(15, 0) +#define MSMON_CFG_MON_SEL_RIS GENMASK(27, 24) + +/* MPAMF_ESR - MPAM Error Status Register */ +#define MPAMF_ESR_PARTID_MON GENMASK(15, 0) +#define MPAMF_ESR_PMG GENMASK(23, 16) +#define MPAMF_ESR_ERRCODE GENMASK(27, 24) +#define MPAMF_ESR_OVRWR BIT(31) +#define MPAMF_ESR_RIS GENMASK(35, 32) + +/* MPAMF_ECR - MPAM Error Control Register */ +#define MPAMF_ECR_INTEN BIT(0) + +/* Error conditions in accessing memory mapped registers */ +#define MPAM_ERRCODE_NONE 0 +#define MPAM_ERRCODE_PARTID_SEL_RANGE 1 +#define MPAM_ERRCODE_REQ_PARTID_RANGE 2 +#define MPAM_ERRCODE_MSMONCFG_ID_RANGE 3 +#define MPAM_ERRCODE_REQ_PMG_RANGE 4 +#define MPAM_ERRCODE_MONITOR_RANGE 5 +#define MPAM_ERRCODE_INTPARTID_RANGE 6 +#define MPAM_ERRCODE_UNEXPECTED_INTERNAL 7 +#define MPAM_ERRCODE_UNDEFINED_RIS_PART_SEL 8 +#define MPAM_ERRCODE_RIS_NO_CONTROL 9 +#define MPAM_ERRCODE_UNDEFINED_RIS_MON_SEL 10 +#define MPAM_ERRCODE_RIS_NO_MONITOR 11 + +/* + * MSMON_CFG_CSU_CTL - Memory system performance monitor configure cache storage + * usage monitor control register + * MSMON_CFG_MBWU_CTL - Memory system performance monitor configure memory + * bandwidth usage monitor control register + */ +#define MSMON_CFG_x_CTL_TYPE GENMASK(7, 0) +#define MSMON_CFG_MBWU_CTL_OFLOW_STATUS_L BIT(15) +#define MSMON_CFG_x_CTL_MATCH_PARTID BIT(16) +#define MSMON_CFG_x_CTL_MATCH_PMG BIT(17) +#define MSMON_CFG_MBWU_CTL_SCLEN BIT(19) +#define MSMON_CFG_x_CTL_SUBTYPE GENMASK(22, 20) +#define MSMON_CFG_x_CTL_OFLOW_FRZ BIT(24) +#define MSMON_CFG_x_CTL_OFLOW_INTR BIT(25) +#define MSMON_CFG_x_CTL_OFLOW_STATUS BIT(26) +#define MSMON_CFG_x_CTL_CAPT_RESET BIT(27) +#define MSMON_CFG_x_CTL_CAPT_EVNT GENMASK(30, 28) +#define MSMON_CFG_x_CTL_EN BIT(31) + +#define MSMON_CFG_MBWU_CTL_TYPE_MBWU 0x42 +#define MSMON_CFG_CSU_CTL_TYPE_CSU 0x43 + +/* + * MSMON_CFG_CSU_FLT - Memory system performance monitor configure cache storage + * usage monitor filter register + * MSMON_CFG_MBWU_FLT - Memory system performance monitor configure memory + * bandwidth usage monitor filter register + */ +#define MSMON_CFG_x_FLT_PARTID GENMASK(15, 0) +#define MSMON_CFG_x_FLT_PMG GENMASK(23, 16) + +#define MSMON_CFG_MBWU_FLT_RWBW GENMASK(31, 30) +#define MSMON_CFG_CSU_FLT_XCL BIT(31) + +/* + * MSMON_CSU - Memory system performance monitor cache storage usage monitor + * register + * MSMON_CSU_CAPTURE - Memory system performance monitor cache storage usage + * capture register + * MSMON_MBWU - Memory system performance monitor memory bandwidth usage + * monitor register + * MSMON_MBWU_CAPTURE - Memory system performance monitor memory bandwidth usage + * capture register + */ +#define MSMON___VALUE GENMASK(30, 0) +#define MSMON___NRDY BIT(31) +#define MSMON___L_NRDY BIT(63) +#define MSMON___L_VALUE GENMASK(43, 0) +#define MSMON___LWD_VALUE GENMASK(62, 0) + +/* + * MSMON_CAPT_EVNT - Memory system performance monitoring capture event + * generation register + */ +#define MSMON_CAPT_EVNT_NOW BIT(0) + +#endif /* MPAM_INTERNAL_H */ diff --git a/drivers/resctrl/test_mpam_devices.c b/drivers/resctrl/test_mpam_devices.c new file mode 100644 index 000000000000..3e8d564a0c64 --- /dev/null +++ b/drivers/resctrl/test_mpam_devices.c @@ -0,0 +1,389 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2025 Arm Ltd. +/* This file is intended to be included into mpam_devices.c */ + +#include <kunit/test.h> + +/* + * This test catches fields that aren't being sanitised - but can't tell you + * which one... + */ +static void test__props_mismatch(struct kunit *test) +{ + struct mpam_props parent = { 0 }; + struct mpam_props child; + + memset(&child, 0xff, sizeof(child)); + __props_mismatch(&parent, &child, false); + + memset(&child, 0, sizeof(child)); + KUNIT_EXPECT_EQ(test, memcmp(&parent, &child, sizeof(child)), 0); + + memset(&child, 0xff, sizeof(child)); + __props_mismatch(&parent, &child, true); + + KUNIT_EXPECT_EQ(test, memcmp(&parent, &child, sizeof(child)), 0); +} + +static struct list_head fake_classes_list; +static struct mpam_class fake_class = { 0 }; +static struct mpam_component fake_comp1 = { 0 }; +static struct mpam_component fake_comp2 = { 0 }; +static struct mpam_vmsc fake_vmsc1 = { 0 }; +static struct mpam_vmsc fake_vmsc2 = { 0 }; +static struct mpam_msc fake_msc1 = { 0 }; +static struct mpam_msc fake_msc2 = { 0 }; +static struct mpam_msc_ris fake_ris1 = { 0 }; +static struct mpam_msc_ris fake_ris2 = { 0 }; +static struct platform_device fake_pdev = { 0 }; + +static inline void reset_fake_hierarchy(void) +{ + INIT_LIST_HEAD(&fake_classes_list); + + memset(&fake_class, 0, sizeof(fake_class)); + fake_class.level = 3; + fake_class.type = MPAM_CLASS_CACHE; + INIT_LIST_HEAD_RCU(&fake_class.components); + INIT_LIST_HEAD(&fake_class.classes_list); + + memset(&fake_comp1, 0, sizeof(fake_comp1)); + memset(&fake_comp2, 0, sizeof(fake_comp2)); + fake_comp1.comp_id = 1; + fake_comp2.comp_id = 2; + INIT_LIST_HEAD(&fake_comp1.vmsc); + INIT_LIST_HEAD(&fake_comp1.class_list); + INIT_LIST_HEAD(&fake_comp2.vmsc); + INIT_LIST_HEAD(&fake_comp2.class_list); + + memset(&fake_vmsc1, 0, sizeof(fake_vmsc1)); + memset(&fake_vmsc2, 0, sizeof(fake_vmsc2)); + INIT_LIST_HEAD(&fake_vmsc1.ris); + INIT_LIST_HEAD(&fake_vmsc1.comp_list); + fake_vmsc1.msc = &fake_msc1; + INIT_LIST_HEAD(&fake_vmsc2.ris); + INIT_LIST_HEAD(&fake_vmsc2.comp_list); + fake_vmsc2.msc = &fake_msc2; + + memset(&fake_ris1, 0, sizeof(fake_ris1)); + memset(&fake_ris2, 0, sizeof(fake_ris2)); + fake_ris1.ris_idx = 1; + INIT_LIST_HEAD(&fake_ris1.msc_list); + fake_ris2.ris_idx = 2; + INIT_LIST_HEAD(&fake_ris2.msc_list); + + fake_msc1.pdev = &fake_pdev; + fake_msc2.pdev = &fake_pdev; + + list_add(&fake_class.classes_list, &fake_classes_list); +} + +static void test_mpam_enable_merge_features(struct kunit *test) +{ + reset_fake_hierarchy(); + + mutex_lock(&mpam_list_lock); + + /* One Class+Comp, two RIS in one vMSC with common features */ + fake_comp1.class = &fake_class; + list_add(&fake_comp1.class_list, &fake_class.components); + fake_comp2.class = NULL; + fake_vmsc1.comp = &fake_comp1; + list_add(&fake_vmsc1.comp_list, &fake_comp1.vmsc); + fake_vmsc2.comp = NULL; + fake_ris1.vmsc = &fake_vmsc1; + list_add(&fake_ris1.vmsc_list, &fake_vmsc1.ris); + fake_ris2.vmsc = &fake_vmsc1; + list_add(&fake_ris2.vmsc_list, &fake_vmsc1.ris); + + mpam_set_feature(mpam_feat_cpor_part, &fake_ris1.props); + mpam_set_feature(mpam_feat_cpor_part, &fake_ris2.props); + fake_ris1.props.cpbm_wd = 4; + fake_ris2.props.cpbm_wd = 4; + + mpam_enable_merge_features(&fake_classes_list); + + KUNIT_EXPECT_TRUE(test, mpam_has_feature(mpam_feat_cpor_part, &fake_class.props)); + KUNIT_EXPECT_EQ(test, fake_class.props.cpbm_wd, 4); + + reset_fake_hierarchy(); + + /* One Class+Comp, two RIS in one vMSC with non-overlapping features */ + fake_comp1.class = &fake_class; + list_add(&fake_comp1.class_list, &fake_class.components); + fake_comp2.class = NULL; + fake_vmsc1.comp = &fake_comp1; + list_add(&fake_vmsc1.comp_list, &fake_comp1.vmsc); + fake_vmsc2.comp = NULL; + fake_ris1.vmsc = &fake_vmsc1; + list_add(&fake_ris1.vmsc_list, &fake_vmsc1.ris); + fake_ris2.vmsc = &fake_vmsc1; + list_add(&fake_ris2.vmsc_list, &fake_vmsc1.ris); + + mpam_set_feature(mpam_feat_cpor_part, &fake_ris1.props); + mpam_set_feature(mpam_feat_cmax_cmin, &fake_ris2.props); + fake_ris1.props.cpbm_wd = 4; + fake_ris2.props.cmax_wd = 4; + + mpam_enable_merge_features(&fake_classes_list); + + /* Multiple RIS within one MSC controlling the same resource can be mismatched */ + KUNIT_EXPECT_TRUE(test, mpam_has_feature(mpam_feat_cpor_part, &fake_class.props)); + KUNIT_EXPECT_TRUE(test, mpam_has_feature(mpam_feat_cmax_cmin, &fake_class.props)); + KUNIT_EXPECT_TRUE(test, mpam_has_feature(mpam_feat_cmax_cmin, &fake_vmsc1.props)); + KUNIT_EXPECT_EQ(test, fake_class.props.cpbm_wd, 4); + KUNIT_EXPECT_EQ(test, fake_vmsc1.props.cmax_wd, 4); + KUNIT_EXPECT_EQ(test, fake_class.props.cmax_wd, 4); + + reset_fake_hierarchy(); + + /* One Class+Comp, two MSC with overlapping features */ + fake_comp1.class = &fake_class; + list_add(&fake_comp1.class_list, &fake_class.components); + fake_comp2.class = NULL; + fake_vmsc1.comp = &fake_comp1; + list_add(&fake_vmsc1.comp_list, &fake_comp1.vmsc); + fake_vmsc2.comp = &fake_comp1; + list_add(&fake_vmsc2.comp_list, &fake_comp1.vmsc); + fake_ris1.vmsc = &fake_vmsc1; + list_add(&fake_ris1.vmsc_list, &fake_vmsc1.ris); + fake_ris2.vmsc = &fake_vmsc2; + list_add(&fake_ris2.vmsc_list, &fake_vmsc2.ris); + + mpam_set_feature(mpam_feat_cpor_part, &fake_ris1.props); + mpam_set_feature(mpam_feat_cpor_part, &fake_ris2.props); + fake_ris1.props.cpbm_wd = 4; + fake_ris2.props.cpbm_wd = 4; + + mpam_enable_merge_features(&fake_classes_list); + + KUNIT_EXPECT_TRUE(test, mpam_has_feature(mpam_feat_cpor_part, &fake_class.props)); + KUNIT_EXPECT_EQ(test, fake_class.props.cpbm_wd, 4); + + reset_fake_hierarchy(); + + /* One Class+Comp, two MSC with non-overlapping features */ + fake_comp1.class = &fake_class; + list_add(&fake_comp1.class_list, &fake_class.components); + fake_comp2.class = NULL; + fake_vmsc1.comp = &fake_comp1; + list_add(&fake_vmsc1.comp_list, &fake_comp1.vmsc); + fake_vmsc2.comp = &fake_comp1; + list_add(&fake_vmsc2.comp_list, &fake_comp1.vmsc); + fake_ris1.vmsc = &fake_vmsc1; + list_add(&fake_ris1.vmsc_list, &fake_vmsc1.ris); + fake_ris2.vmsc = &fake_vmsc2; + list_add(&fake_ris2.vmsc_list, &fake_vmsc2.ris); + + mpam_set_feature(mpam_feat_cpor_part, &fake_ris1.props); + mpam_set_feature(mpam_feat_cmax_cmin, &fake_ris2.props); + fake_ris1.props.cpbm_wd = 4; + fake_ris2.props.cmax_wd = 4; + + mpam_enable_merge_features(&fake_classes_list); + + /* + * Multiple RIS in different MSC can't control the same resource, + * mismatched features can not be supported. + */ + KUNIT_EXPECT_FALSE(test, mpam_has_feature(mpam_feat_cpor_part, &fake_class.props)); + KUNIT_EXPECT_FALSE(test, mpam_has_feature(mpam_feat_cmax_cmin, &fake_class.props)); + KUNIT_EXPECT_EQ(test, fake_class.props.cpbm_wd, 0); + KUNIT_EXPECT_EQ(test, fake_class.props.cmax_wd, 0); + + reset_fake_hierarchy(); + + /* One Class+Comp, two MSC with incompatible overlapping features */ + fake_comp1.class = &fake_class; + list_add(&fake_comp1.class_list, &fake_class.components); + fake_comp2.class = NULL; + fake_vmsc1.comp = &fake_comp1; + list_add(&fake_vmsc1.comp_list, &fake_comp1.vmsc); + fake_vmsc2.comp = &fake_comp1; + list_add(&fake_vmsc2.comp_list, &fake_comp1.vmsc); + fake_ris1.vmsc = &fake_vmsc1; + list_add(&fake_ris1.vmsc_list, &fake_vmsc1.ris); + fake_ris2.vmsc = &fake_vmsc2; + list_add(&fake_ris2.vmsc_list, &fake_vmsc2.ris); + + mpam_set_feature(mpam_feat_cpor_part, &fake_ris1.props); + mpam_set_feature(mpam_feat_cpor_part, &fake_ris2.props); + mpam_set_feature(mpam_feat_mbw_part, &fake_ris1.props); + mpam_set_feature(mpam_feat_mbw_part, &fake_ris2.props); + fake_ris1.props.cpbm_wd = 5; + fake_ris2.props.cpbm_wd = 3; + fake_ris1.props.mbw_pbm_bits = 5; + fake_ris2.props.mbw_pbm_bits = 3; + + mpam_enable_merge_features(&fake_classes_list); + + /* + * Multiple RIS in different MSC can't control the same resource, + * mismatched features can not be supported. + */ + KUNIT_EXPECT_FALSE(test, mpam_has_feature(mpam_feat_cpor_part, &fake_class.props)); + KUNIT_EXPECT_FALSE(test, mpam_has_feature(mpam_feat_mbw_part, &fake_class.props)); + KUNIT_EXPECT_EQ(test, fake_class.props.cpbm_wd, 0); + KUNIT_EXPECT_EQ(test, fake_class.props.mbw_pbm_bits, 0); + + reset_fake_hierarchy(); + + /* One Class+Comp, two MSC with overlapping features that need tweaking */ + fake_comp1.class = &fake_class; + list_add(&fake_comp1.class_list, &fake_class.components); + fake_comp2.class = NULL; + fake_vmsc1.comp = &fake_comp1; + list_add(&fake_vmsc1.comp_list, &fake_comp1.vmsc); + fake_vmsc2.comp = &fake_comp1; + list_add(&fake_vmsc2.comp_list, &fake_comp1.vmsc); + fake_ris1.vmsc = &fake_vmsc1; + list_add(&fake_ris1.vmsc_list, &fake_vmsc1.ris); + fake_ris2.vmsc = &fake_vmsc2; + list_add(&fake_ris2.vmsc_list, &fake_vmsc2.ris); + + mpam_set_feature(mpam_feat_mbw_min, &fake_ris1.props); + mpam_set_feature(mpam_feat_mbw_min, &fake_ris2.props); + mpam_set_feature(mpam_feat_cmax_cmax, &fake_ris1.props); + mpam_set_feature(mpam_feat_cmax_cmax, &fake_ris2.props); + fake_ris1.props.bwa_wd = 5; + fake_ris2.props.bwa_wd = 3; + fake_ris1.props.cmax_wd = 5; + fake_ris2.props.cmax_wd = 3; + + mpam_enable_merge_features(&fake_classes_list); + + /* + * RIS with different control properties need to be sanitised so the + * class has the common set of properties. + */ + KUNIT_EXPECT_TRUE(test, mpam_has_feature(mpam_feat_mbw_min, &fake_class.props)); + KUNIT_EXPECT_TRUE(test, mpam_has_feature(mpam_feat_cmax_cmax, &fake_class.props)); + KUNIT_EXPECT_EQ(test, fake_class.props.bwa_wd, 3); + KUNIT_EXPECT_EQ(test, fake_class.props.cmax_wd, 3); + + reset_fake_hierarchy(); + + /* One Class Two Comp with overlapping features */ + fake_comp1.class = &fake_class; + list_add(&fake_comp1.class_list, &fake_class.components); + fake_comp2.class = &fake_class; + list_add(&fake_comp2.class_list, &fake_class.components); + fake_vmsc1.comp = &fake_comp1; + list_add(&fake_vmsc1.comp_list, &fake_comp1.vmsc); + fake_vmsc2.comp = &fake_comp2; + list_add(&fake_vmsc2.comp_list, &fake_comp2.vmsc); + fake_ris1.vmsc = &fake_vmsc1; + list_add(&fake_ris1.vmsc_list, &fake_vmsc1.ris); + fake_ris2.vmsc = &fake_vmsc2; + list_add(&fake_ris2.vmsc_list, &fake_vmsc2.ris); + + mpam_set_feature(mpam_feat_cpor_part, &fake_ris1.props); + mpam_set_feature(mpam_feat_cpor_part, &fake_ris2.props); + fake_ris1.props.cpbm_wd = 4; + fake_ris2.props.cpbm_wd = 4; + + mpam_enable_merge_features(&fake_classes_list); + + KUNIT_EXPECT_TRUE(test, mpam_has_feature(mpam_feat_cpor_part, &fake_class.props)); + KUNIT_EXPECT_EQ(test, fake_class.props.cpbm_wd, 4); + + reset_fake_hierarchy(); + + /* One Class Two Comp with non-overlapping features */ + fake_comp1.class = &fake_class; + list_add(&fake_comp1.class_list, &fake_class.components); + fake_comp2.class = &fake_class; + list_add(&fake_comp2.class_list, &fake_class.components); + fake_vmsc1.comp = &fake_comp1; + list_add(&fake_vmsc1.comp_list, &fake_comp1.vmsc); + fake_vmsc2.comp = &fake_comp2; + list_add(&fake_vmsc2.comp_list, &fake_comp2.vmsc); + fake_ris1.vmsc = &fake_vmsc1; + list_add(&fake_ris1.vmsc_list, &fake_vmsc1.ris); + fake_ris2.vmsc = &fake_vmsc2; + list_add(&fake_ris2.vmsc_list, &fake_vmsc2.ris); + + mpam_set_feature(mpam_feat_cpor_part, &fake_ris1.props); + mpam_set_feature(mpam_feat_cmax_cmin, &fake_ris2.props); + fake_ris1.props.cpbm_wd = 4; + fake_ris2.props.cmax_wd = 4; + + mpam_enable_merge_features(&fake_classes_list); + + /* + * Multiple components can't control the same resource, mismatched features can + * not be supported. + */ + KUNIT_EXPECT_FALSE(test, mpam_has_feature(mpam_feat_cpor_part, &fake_class.props)); + KUNIT_EXPECT_FALSE(test, mpam_has_feature(mpam_feat_cmax_cmin, &fake_class.props)); + KUNIT_EXPECT_EQ(test, fake_class.props.cpbm_wd, 0); + KUNIT_EXPECT_EQ(test, fake_class.props.cmax_wd, 0); + + mutex_unlock(&mpam_list_lock); +} + +static void test_mpam_reset_msc_bitmap(struct kunit *test) +{ + char __iomem *buf = kunit_kzalloc(test, SZ_16K, GFP_KERNEL); + struct mpam_msc fake_msc = {}; + u32 *test_result; + + if (!buf) + return; + + fake_msc.mapped_hwpage = buf; + fake_msc.mapped_hwpage_sz = SZ_16K; + cpumask_copy(&fake_msc.accessibility, cpu_possible_mask); + + /* Satisfy lockdep checks */ + mutex_init(&fake_msc.part_sel_lock); + mutex_lock(&fake_msc.part_sel_lock); + + test_result = (u32 *)(buf + MPAMCFG_CPBM); + + mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 0); + KUNIT_EXPECT_EQ(test, test_result[0], 0); + KUNIT_EXPECT_EQ(test, test_result[1], 0); + test_result[0] = 0; + test_result[1] = 0; + + mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 1); + KUNIT_EXPECT_EQ(test, test_result[0], 1); + KUNIT_EXPECT_EQ(test, test_result[1], 0); + test_result[0] = 0; + test_result[1] = 0; + + mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 16); + KUNIT_EXPECT_EQ(test, test_result[0], 0xffff); + KUNIT_EXPECT_EQ(test, test_result[1], 0); + test_result[0] = 0; + test_result[1] = 0; + + mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 32); + KUNIT_EXPECT_EQ(test, test_result[0], 0xffffffff); + KUNIT_EXPECT_EQ(test, test_result[1], 0); + test_result[0] = 0; + test_result[1] = 0; + + mpam_reset_msc_bitmap(&fake_msc, MPAMCFG_CPBM, 33); + KUNIT_EXPECT_EQ(test, test_result[0], 0xffffffff); + KUNIT_EXPECT_EQ(test, test_result[1], 1); + test_result[0] = 0; + test_result[1] = 0; + + mutex_unlock(&fake_msc.part_sel_lock); +} + +static struct kunit_case mpam_devices_test_cases[] = { + KUNIT_CASE(test_mpam_reset_msc_bitmap), + KUNIT_CASE(test_mpam_enable_merge_features), + KUNIT_CASE(test__props_mismatch), + {} +}; + +static struct kunit_suite mpam_devices_test_suite = { + .name = "mpam_devices_test_suite", + .test_cases = mpam_devices_test_cases, +}; + +kunit_test_suites(&mpam_devices_test_suite); diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index 7765e40f7cea..35031357ac4d 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -207,19 +207,6 @@ static int dasd_state_known_to_new(struct dasd_device *device) return 0; } -static struct dentry *dasd_debugfs_setup(const char *name, - struct dentry *base_dentry) -{ - struct dentry *pde; - - if (!base_dentry) - return NULL; - pde = debugfs_create_dir(name, base_dentry); - if (!pde || IS_ERR(pde)) - return NULL; - return pde; -} - /* * Request the irq line for the device. */ @@ -234,14 +221,14 @@ static int dasd_state_known_to_basic(struct dasd_device *device) if (rc) return rc; block->debugfs_dentry = - dasd_debugfs_setup(block->gdp->disk_name, + debugfs_create_dir(block->gdp->disk_name, dasd_debugfs_root_entry); dasd_profile_init(&block->profile, block->debugfs_dentry); if (dasd_global_profile_level == DASD_PROFILE_ON) dasd_profile_on(&device->block->profile); } device->debugfs_dentry = - dasd_debugfs_setup(dev_name(&device->cdev->dev), + debugfs_create_dir(dev_name(&device->cdev->dev), dasd_debugfs_root_entry); dasd_profile_init(&device->profile, device->debugfs_dentry); dasd_hosts_init(device->debugfs_dentry, device); @@ -974,8 +961,7 @@ static void dasd_stats_array(struct seq_file *m, unsigned int *array) static void dasd_stats_seq_print(struct seq_file *m, struct dasd_profile_info *data) { - seq_printf(m, "start_time %lld.%09ld\n", - (s64)data->starttod.tv_sec, data->starttod.tv_nsec); + seq_printf(m, "start_time %ptSp\n", &data->starttod); seq_printf(m, "total_requests %u\n", data->dasd_io_reqs); seq_printf(m, "total_sectors %u\n", data->dasd_io_sects); seq_printf(m, "total_pav %u\n", data->dasd_io_alias); @@ -1058,19 +1044,9 @@ static const struct file_operations dasd_stats_raw_fops = { static void dasd_profile_init(struct dasd_profile *profile, struct dentry *base_dentry) { - umode_t mode; - struct dentry *pde; - - if (!base_dentry) - return; - profile->dentry = NULL; profile->data = NULL; - mode = (S_IRUSR | S_IWUSR | S_IFREG); - pde = debugfs_create_file("statistics", mode, base_dentry, - profile, &dasd_stats_raw_fops); - if (pde && !IS_ERR(pde)) - profile->dentry = pde; - return; + profile->dentry = debugfs_create_file("statistics", 0600, base_dentry, + profile, &dasd_stats_raw_fops); } static void dasd_profile_exit(struct dasd_profile *profile) @@ -1090,25 +1066,9 @@ static void dasd_statistics_removeroot(void) static void dasd_statistics_createroot(void) { - struct dentry *pde; - - dasd_debugfs_root_entry = NULL; - pde = debugfs_create_dir("dasd", NULL); - if (!pde || IS_ERR(pde)) - goto error; - dasd_debugfs_root_entry = pde; - pde = debugfs_create_dir("global", dasd_debugfs_root_entry); - if (!pde || IS_ERR(pde)) - goto error; - dasd_debugfs_global_entry = pde; + dasd_debugfs_root_entry = debugfs_create_dir("dasd", NULL); + dasd_debugfs_global_entry = debugfs_create_dir("global", dasd_debugfs_root_entry); dasd_profile_init(&dasd_global_profile, dasd_debugfs_global_entry); - return; - -error: - DBF_EVENT(DBF_ERR, "%s", - "Creation of the dasd debugfs interface failed"); - dasd_statistics_removeroot(); - return; } #else @@ -1169,17 +1129,8 @@ static void dasd_hosts_exit(struct dasd_device *device) static void dasd_hosts_init(struct dentry *base_dentry, struct dasd_device *device) { - struct dentry *pde; - umode_t mode; - - if (!base_dentry) - return; - - mode = S_IRUSR | S_IFREG; - pde = debugfs_create_file("host_access_list", mode, base_dentry, - device, &dasd_hosts_fops); - if (pde && !IS_ERR(pde)) - device->hosts_dentry = pde; + device->hosts_dentry = debugfs_create_file("host_access_list", 0400, base_dentry, + device, &dasd_hosts_fops); } struct dasd_ccw_req *dasd_smalloc_request(int magic, int cplength, int datasize, @@ -3350,7 +3301,6 @@ dasd_device_operations = { .open = dasd_open, .release = dasd_release, .ioctl = dasd_ioctl, - .compat_ioctl = dasd_ioctl, .getgeo = dasd_getgeo, .set_read_only = dasd_set_read_only, }; diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c index ddbdf1f85d44..73972900fc55 100644 --- a/drivers/s390/block/dasd_devmap.c +++ b/drivers/s390/block/dasd_devmap.c @@ -355,7 +355,8 @@ static int __init dasd_parse_range(const char *range) /* each device in dasd= parameter should be set initially online */ features |= DASD_FEATURE_INITIAL_ONLINE; while (from <= to) { - sprintf(bus_id, "%01x.%01x.%04x", from_id0, from_id1, from++); + scnprintf(bus_id, sizeof(bus_id), + "%01x.%01x.%04x", from_id0, from_id1, from++); devmap = dasd_add_busid(bus_id, features); if (IS_ERR(devmap)) { rc = PTR_ERR(devmap); diff --git a/drivers/s390/block/dasd_eckd.c b/drivers/s390/block/dasd_eckd.c index 88fa17aea2ec..b08e900687f3 100644 --- a/drivers/s390/block/dasd_eckd.c +++ b/drivers/s390/block/dasd_eckd.c @@ -16,7 +16,6 @@ #include <linux/hdreg.h> /* HDIO_GETGEO */ #include <linux/bio.h> #include <linux/module.h> -#include <linux/compat.h> #include <linux/init.h> #include <linux/seq_file.h> #include <linux/uaccess.h> @@ -5389,16 +5388,6 @@ static int dasd_symm_io(struct dasd_device *device, void __user *argp) rc = -EFAULT; if (copy_from_user(&usrparm, argp, sizeof(usrparm))) goto out; - if (is_compat_task()) { - /* Make sure pointers are sane even on 31 bit. */ - rc = -EINVAL; - if ((usrparm.psf_data >> 32) != 0) - goto out; - if ((usrparm.rssd_result >> 32) != 0) - goto out; - usrparm.psf_data &= 0x7fffffffULL; - usrparm.rssd_result &= 0x7fffffffULL; - } /* at least 2 bytes are accessed and should be allocated */ if (usrparm.psf_data_len < 2) { DBF_DEV_EVENT(DBF_WARNING, device, @@ -6150,6 +6139,7 @@ static int dasd_eckd_copy_pair_swap(struct dasd_device *device, char *prim_busid struct dasd_copy_relation *copy; struct dasd_block *block; struct gendisk *gdp; + int rc; copy = device->copy; if (!copy) @@ -6184,6 +6174,13 @@ static int dasd_eckd_copy_pair_swap(struct dasd_device *device, char *prim_busid /* swap blocklayer device link */ gdp = block->gdp; dasd_add_link_to_gendisk(gdp, secondary); + rc = device_move(disk_to_dev(gdp), &secondary->cdev->dev, DPM_ORDER_NONE); + if (rc) { + dev_err(&primary->cdev->dev, + "copy_pair_swap: moving blockdevice parent %s->%s failed (%d)\n", + dev_name(&primary->cdev->dev), + dev_name(&secondary->cdev->dev), rc); + } /* re-enable device */ dasd_device_remove_stop_bits(primary, DASD_STOPPED_PPRC); diff --git a/drivers/s390/block/dasd_fba.c b/drivers/s390/block/dasd_fba.c index a2216795591d..c2a87201c153 100644 --- a/drivers/s390/block/dasd_fba.c +++ b/drivers/s390/block/dasd_fba.c @@ -5,7 +5,6 @@ * Copyright IBM Corp. 1999, 2009 */ -#define KMSG_COMPONENT "dasd-fba" #include <linux/stddef.h> #include <linux/kernel.h> diff --git a/drivers/s390/block/dasd_genhd.c b/drivers/s390/block/dasd_genhd.c index 28e92fad0ca1..6ee3d952412e 100644 --- a/drivers/s390/block/dasd_genhd.c +++ b/drivers/s390/block/dasd_genhd.c @@ -22,6 +22,7 @@ static unsigned int queue_depth = 32; static unsigned int nr_hw_queues = 4; +static void dasd_gd_free(struct gendisk *gdp); module_param(queue_depth, uint, 0444); MODULE_PARM_DESC(queue_depth, "Default queue depth for new DASD devices"); @@ -30,6 +31,37 @@ module_param(nr_hw_queues, uint, 0444); MODULE_PARM_DESC(nr_hw_queues, "Default number of hardware queues for new DASD devices"); /* + * Set device name. + * dasda - dasdz : 26 devices + * dasdaa - dasdzz : 676 devices, added up = 702 + * dasdaaa - dasdzzz : 17576 devices, added up = 18278 + * dasdaaaa - dasdzzzz : 456976 devices, added up = 475252 + */ +static int dasd_name_format(char *prefix, int index, char *buf, int buflen) +{ + const int base = 'z' - 'a' + 1; + char *begin = buf + strlen(prefix); + char *end = buf + buflen; + char *p; + int unit; + + p = end - 1; + *p = '\0'; + unit = base; + do { + if (p == begin) + return -EINVAL; + *--p = 'a' + (index % unit); + index = (index / unit) - 1; + } while (index >= 0); + + memmove(begin, p, end - p); + memcpy(buf, prefix, strlen(prefix)); + + return 0; +} + +/* * Allocate and register gendisk structure for device. */ int dasd_gendisk_alloc(struct dasd_block *block) @@ -45,11 +77,13 @@ int dasd_gendisk_alloc(struct dasd_block *block) }; struct gendisk *gdp; struct dasd_device *base; - int len, rc; + unsigned int devindex; + int rc; /* Make sure the minor for this device exists. */ base = block->base; - if (base->devindex >= DASD_PER_MAJOR) + devindex = base->devindex; + if (devindex >= DASD_PER_MAJOR) return -EBUSY; block->tag_set.ops = &dasd_mq_ops; @@ -69,31 +103,17 @@ int dasd_gendisk_alloc(struct dasd_block *block) /* Initialize gendisk structure. */ gdp->major = DASD_MAJOR; - gdp->first_minor = base->devindex << DASD_PARTN_BITS; + gdp->first_minor = devindex << DASD_PARTN_BITS; gdp->minors = 1 << DASD_PARTN_BITS; gdp->fops = &dasd_device_operations; - /* - * Set device name. - * dasda - dasdz : 26 devices - * dasdaa - dasdzz : 676 devices, added up = 702 - * dasdaaa - dasdzzz : 17576 devices, added up = 18278 - * dasdaaaa - dasdzzzz : 456976 devices, added up = 475252 - */ - len = sprintf(gdp->disk_name, "dasd"); - if (base->devindex > 25) { - if (base->devindex > 701) { - if (base->devindex > 18277) - len += sprintf(gdp->disk_name + len, "%c", - 'a'+(((base->devindex-18278) - /17576)%26)); - len += sprintf(gdp->disk_name + len, "%c", - 'a'+(((base->devindex-702)/676)%26)); - } - len += sprintf(gdp->disk_name + len, "%c", - 'a'+(((base->devindex-26)/26)%26)); + rc = dasd_name_format("dasd", devindex, gdp->disk_name, sizeof(gdp->disk_name)); + if (rc) { + DBF_DEV_EVENT(DBF_ERR, block->base, + "setting disk name failed, rc %d", rc); + dasd_gd_free(gdp); + return rc; } - len += sprintf(gdp->disk_name + len, "%c", 'a'+(base->devindex%26)); if (base->features & DASD_FEATURE_READONLY || test_bit(DASD_FLAG_DEVICE_RO, &base->flags)) @@ -112,14 +132,22 @@ int dasd_gendisk_alloc(struct dasd_block *block) } /* + * Free gendisk structure + */ +static void dasd_gd_free(struct gendisk *gd) +{ + del_gendisk(gd); + gd->private_data = NULL; + put_disk(gd); +} + +/* * Unregister and free gendisk structure for device. */ void dasd_gendisk_free(struct dasd_block *block) { if (block->gdp) { - del_gendisk(block->gdp); - block->gdp->private_data = NULL; - put_disk(block->gdp); + dasd_gd_free(block->gdp); block->gdp = NULL; blk_mq_free_tag_set(&block->tag_set); } diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c index 8308046a9f8f..f883990be626 100644 --- a/drivers/s390/block/dasd_ioctl.c +++ b/drivers/s390/block/dasd_ioctl.c @@ -11,7 +11,6 @@ */ #include <linux/interrupt.h> -#include <linux/compat.h> #include <linux/export.h> #include <linux/major.h> #include <linux/fs.h> @@ -616,10 +615,7 @@ int dasd_ioctl(struct block_device *bdev, blk_mode_t mode, void __user *argp; int rc; - if (is_compat_task()) - argp = compat_ptr(arg); - else - argp = (void __user *)arg; + argp = (void __user *)arg; if ((_IOC_DIR(cmd) != _IOC_NONE) && !arg) return -EINVAL; diff --git a/drivers/s390/block/dcssblk.c b/drivers/s390/block/dcssblk.c index 86fef4b15015..38e1df8f8a82 100644 --- a/drivers/s390/block/dcssblk.c +++ b/drivers/s390/block/dcssblk.c @@ -5,8 +5,7 @@ * Authors: Carsten Otte, Stefan Weinhuber, Gerald Schaefer */ -#define KMSG_COMPONENT "dcssblk" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "dcssblk: " fmt #include <linux/module.h> #include <linux/moduleparam.h> @@ -674,8 +673,8 @@ dcssblk_add_store(struct device *dev, struct device_attribute *attr, const char rc = dcssblk_assign_free_minor(dev_info); if (rc) goto release_gd; - sprintf(dev_info->gd->disk_name, "dcssblk%d", - dev_info->gd->first_minor); + scnprintf(dev_info->gd->disk_name, sizeof(dev_info->gd->disk_name), + "dcssblk%d", dev_info->gd->first_minor); list_add_tail(&dev_info->lh, &dcssblk_devices); if (!try_module_get(THIS_MODULE)) { diff --git a/drivers/s390/block/scm_blk.c b/drivers/s390/block/scm_blk.c index 91bbe9d2e5ac..04e84f45dcc9 100644 --- a/drivers/s390/block/scm_blk.c +++ b/drivers/s390/block/scm_blk.c @@ -6,8 +6,7 @@ * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com> */ -#define KMSG_COMPONENT "scm_block" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "scm_block: " fmt #include <linux/interrupt.h> #include <linux/spinlock.h> diff --git a/drivers/s390/block/scm_drv.c b/drivers/s390/block/scm_drv.c index 69a845eb8b1f..6cffbbe83f89 100644 --- a/drivers/s390/block/scm_drv.c +++ b/drivers/s390/block/scm_drv.c @@ -6,8 +6,7 @@ * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com> */ -#define KMSG_COMPONENT "scm_block" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "scm_block: " fmt #include <linux/module.h> #include <linux/slab.h> diff --git a/drivers/s390/char/con3270.c b/drivers/s390/char/con3270.c index a367f95c7c53..4a7c084e68a6 100644 --- a/drivers/s390/char/con3270.c +++ b/drivers/s390/char/con3270.c @@ -21,7 +21,6 @@ #include <linux/reboot.h> #include <linux/slab.h> #include <linux/memblock.h> -#include <linux/compat.h> #include <asm/machine.h> #include <asm/ccwdev.h> @@ -1662,7 +1661,7 @@ static void tty3270_escape_sequence(struct tty3270 *tp, u8 ch) else if (tp->esc_par[0] == 6) { /* Cursor report. */ char buf[40]; - sprintf(buf, "\033[%d;%dR", tp->cy + 1, tp->cx + 1); + scnprintf(buf, sizeof(buf), "\033[%d;%dR", tp->cy + 1, tp->cx + 1); kbd_puts_queue(&tp->port, buf); } return; @@ -1947,21 +1946,6 @@ static int tty3270_ioctl(struct tty_struct *tty, unsigned int cmd, return kbd_ioctl(tp->kbd, cmd, arg); } -#ifdef CONFIG_COMPAT -static long tty3270_compat_ioctl(struct tty_struct *tty, - unsigned int cmd, unsigned long arg) -{ - struct tty3270 *tp; - - tp = tty->driver_data; - if (!tp) - return -ENODEV; - if (tty_io_error(tty)) - return -EIO; - return kbd_ioctl(tp->kbd, cmd, (unsigned long)compat_ptr(arg)); -} -#endif - static const struct tty_operations tty3270_ops = { .install = tty3270_install, .cleanup = tty3270_cleanup, @@ -1976,9 +1960,6 @@ static const struct tty_operations tty3270_ops = { .hangup = tty3270_hangup, .wait_until_sent = tty3270_wait_until_sent, .ioctl = tty3270_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = tty3270_compat_ioctl, -#endif .set_termios = tty3270_set_termios }; diff --git a/drivers/s390/char/diag_ftp.c b/drivers/s390/char/diag_ftp.c index f41b39c9d267..a1e110c96f74 100644 --- a/drivers/s390/char/diag_ftp.c +++ b/drivers/s390/char/diag_ftp.c @@ -7,8 +7,7 @@ * */ -#define KMSG_COMPONENT "hmcdrv" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "hmcdrv: " fmt #include <linux/kernel.h> #include <linux/mm.h> diff --git a/drivers/s390/char/fs3270.c b/drivers/s390/char/fs3270.c index cfe7efd5b5da..73555dbe30d0 100644 --- a/drivers/s390/char/fs3270.c +++ b/drivers/s390/char/fs3270.c @@ -12,7 +12,6 @@ #include <linux/console.h> #include <linux/init.h> #include <linux/interrupt.h> -#include <linux/compat.h> #include <linux/sched/signal.h> #include <linux/module.h> #include <linux/list.h> @@ -330,10 +329,7 @@ static long fs3270_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) fp = filp->private_data; if (!fp) return -ENODEV; - if (is_compat_task()) - argp = compat_ptr(arg); - else - argp = (char __user *)arg; + argp = (char __user *)arg; rc = 0; mutex_lock(&fs3270_mutex); switch (cmd) { @@ -512,7 +508,6 @@ static const struct file_operations fs3270_fops = { .read = fs3270_read, /* read */ .write = fs3270_write, /* write */ .unlocked_ioctl = fs3270_ioctl, /* ioctl */ - .compat_ioctl = fs3270_ioctl, /* ioctl */ .open = fs3270_open, /* open */ .release = fs3270_close, /* release */ }; diff --git a/drivers/s390/char/hmcdrv_cache.c b/drivers/s390/char/hmcdrv_cache.c index 43df27ceec11..85fb689594ca 100644 --- a/drivers/s390/char/hmcdrv_cache.c +++ b/drivers/s390/char/hmcdrv_cache.c @@ -7,8 +7,7 @@ * */ -#define KMSG_COMPONENT "hmcdrv" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "hmcdrv: " fmt #include <linux/kernel.h> #include <linux/mm.h> diff --git a/drivers/s390/char/hmcdrv_dev.c b/drivers/s390/char/hmcdrv_dev.c index b26fcf6849f2..04b938c5357f 100644 --- a/drivers/s390/char/hmcdrv_dev.c +++ b/drivers/s390/char/hmcdrv_dev.c @@ -14,8 +14,7 @@ * end read() the response. */ -#define KMSG_COMPONENT "hmcdrv" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "hmcdrv: " fmt #include <linux/kernel.h> #include <linux/module.h> diff --git a/drivers/s390/char/hmcdrv_ftp.c b/drivers/s390/char/hmcdrv_ftp.c index 4e3c7ec6749b..3312b2ac00a9 100644 --- a/drivers/s390/char/hmcdrv_ftp.c +++ b/drivers/s390/char/hmcdrv_ftp.c @@ -6,8 +6,7 @@ * Author(s): Ralf Hoppe (rhoppe@de.ibm.com) */ -#define KMSG_COMPONENT "hmcdrv" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "hmcdrv: " fmt #include <linux/kernel.h> #include <linux/slab.h> diff --git a/drivers/s390/char/hmcdrv_mod.c b/drivers/s390/char/hmcdrv_mod.c index 1447d0887225..b1cc5ba9fed8 100644 --- a/drivers/s390/char/hmcdrv_mod.c +++ b/drivers/s390/char/hmcdrv_mod.c @@ -6,8 +6,7 @@ * Author(s): Ralf Hoppe (rhoppe@de.ibm.com) */ -#define KMSG_COMPONENT "hmcdrv" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "hmcdrv: " fmt #include <linux/kernel.h> #include <linux/module.h> diff --git a/drivers/s390/char/monreader.c b/drivers/s390/char/monreader.c index 2d9886651d9b..3d84f84b4cbd 100644 --- a/drivers/s390/char/monreader.c +++ b/drivers/s390/char/monreader.c @@ -7,8 +7,7 @@ * Author: Gerald Schaefer <gerald.schaefer@de.ibm.com> */ -#define KMSG_COMPONENT "monreader" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "monreader: " fmt #include <linux/module.h> #include <linux/moduleparam.h> diff --git a/drivers/s390/char/monwriter.c b/drivers/s390/char/monwriter.c index 0fab1f025a94..cf2e51061422 100644 --- a/drivers/s390/char/monwriter.c +++ b/drivers/s390/char/monwriter.c @@ -7,8 +7,7 @@ * Author(s): Melissa Howland <Melissa.Howland@us.ibm.com> */ -#define KMSG_COMPONENT "monwriter" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "monwriter: " fmt #include <linux/module.h> #include <linux/moduleparam.h> diff --git a/drivers/s390/char/sclp_ap.c b/drivers/s390/char/sclp_ap.c index 0dd1ca712795..18bb018b4e0c 100644 --- a/drivers/s390/char/sclp_ap.c +++ b/drivers/s390/char/sclp_ap.c @@ -4,8 +4,7 @@ * * Copyright IBM Corp. 2020 */ -#define KMSG_COMPONENT "sclp_cmd" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_cmd: " fmt #include <linux/export.h> #include <linux/slab.h> diff --git a/drivers/s390/char/sclp_cmd.c b/drivers/s390/char/sclp_cmd.c index 3480198eac02..be4730936f5c 100644 --- a/drivers/s390/char/sclp_cmd.c +++ b/drivers/s390/char/sclp_cmd.c @@ -5,8 +5,7 @@ * Author(s): Peter Oberparleiter <peter.oberparleiter@de.ibm.com> */ -#define KMSG_COMPONENT "sclp_cmd" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_cmd: " fmt #include <linux/completion.h> #include <linux/err.h> diff --git a/drivers/s390/char/sclp_config.c b/drivers/s390/char/sclp_config.c index 356d26a09af0..9cfbe3fc3dca 100644 --- a/drivers/s390/char/sclp_config.c +++ b/drivers/s390/char/sclp_config.c @@ -3,8 +3,7 @@ * Copyright IBM Corp. 2007 */ -#define KMSG_COMPONENT "sclp_config" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_config: " fmt #include <linux/init.h> #include <linux/errno.h> diff --git a/drivers/s390/char/sclp_cpi_sys.c b/drivers/s390/char/sclp_cpi_sys.c index d8f91aab11e8..8e1636bcf8b5 100644 --- a/drivers/s390/char/sclp_cpi_sys.c +++ b/drivers/s390/char/sclp_cpi_sys.c @@ -7,8 +7,7 @@ * Michael Ernst <mernst@de.ibm.com> */ -#define KMSG_COMPONENT "sclp_cpi" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_cpi: " fmt #include <linux/kernel.h> #include <linux/init.h> diff --git a/drivers/s390/char/sclp_ctl.c b/drivers/s390/char/sclp_ctl.c index dd6051602070..e23a97359286 100644 --- a/drivers/s390/char/sclp_ctl.c +++ b/drivers/s390/char/sclp_ctl.c @@ -7,7 +7,6 @@ * Author: Michael Holzheu <holzheu@linux.vnet.ibm.com> */ -#include <linux/compat.h> #include <linux/uaccess.h> #include <linux/miscdevice.h> #include <linux/gfp.h> @@ -43,10 +42,7 @@ static int sclp_ctl_cmdw_supported(unsigned int cmdw) static void __user *u64_to_uptr(u64 value) { - if (is_compat_task()) - return compat_ptr(value); - else - return (void __user *)(unsigned long)value; + return (void __user *)(unsigned long)value; } /* @@ -95,10 +91,7 @@ static long sclp_ctl_ioctl(struct file *filp, unsigned int cmd, { void __user *argp; - if (is_compat_task()) - argp = compat_ptr(arg); - else - argp = (void __user *) arg; + argp = (void __user *)arg; switch (cmd) { case SCLP_CTL_SCCB: return sclp_ctl_ioctl_sccb(argp); @@ -114,7 +107,6 @@ static const struct file_operations sclp_ctl_fops = { .owner = THIS_MODULE, .open = nonseekable_open, .unlocked_ioctl = sclp_ctl_ioctl, - .compat_ioctl = sclp_ctl_ioctl, }; /* diff --git a/drivers/s390/char/sclp_early.c b/drivers/s390/char/sclp_early.c index bd5e5ba50c0a..6bf501ad8ff0 100644 --- a/drivers/s390/char/sclp_early.c +++ b/drivers/s390/char/sclp_early.c @@ -5,8 +5,7 @@ * Copyright IBM Corp. 2013 */ -#define KMSG_COMPONENT "sclp_early" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_early: " fmt #include <linux/export.h> #include <linux/errno.h> diff --git a/drivers/s390/char/sclp_ftp.c b/drivers/s390/char/sclp_ftp.c index d27e2cbfbccb..2a1c4b2cafc8 100644 --- a/drivers/s390/char/sclp_ftp.c +++ b/drivers/s390/char/sclp_ftp.c @@ -7,8 +7,7 @@ * */ -#define KMSG_COMPONENT "hmcdrv" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "hmcdrv: " fmt #include <linux/kernel.h> #include <linux/mm.h> diff --git a/drivers/s390/char/sclp_mem.c b/drivers/s390/char/sclp_mem.c index 27f49f5fd358..676c085b4f8a 100644 --- a/drivers/s390/char/sclp_mem.c +++ b/drivers/s390/char/sclp_mem.c @@ -5,13 +5,15 @@ * Copyright IBM Corp. 2025 */ -#define KMSG_COMPONENT "sclp_mem" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_mem: " fmt #include <linux/cpufeature.h> +#include <linux/container_of.h> #include <linux/err.h> #include <linux/errno.h> #include <linux/init.h> +#include <linux/kobject.h> +#include <linux/kstrtox.h> #include <linux/memory.h> #include <linux/memory_hotplug.h> #include <linux/mm.h> @@ -27,7 +29,6 @@ #define SCLP_CMDW_ASSIGN_STORAGE 0x000d0001 #define SCLP_CMDW_UNASSIGN_STORAGE 0x000c0001 -static DEFINE_MUTEX(sclp_mem_mutex); static LIST_HEAD(sclp_mem_list); static u8 sclp_max_storage_id; static DECLARE_BITMAP(sclp_storage_ids, 256); @@ -38,6 +39,18 @@ struct memory_increment { int standby; }; +struct sclp_mem { + struct kobject kobj; + unsigned int id; + unsigned int memmap_on_memory; + unsigned int config; +}; + +struct sclp_mem_arg { + struct sclp_mem *sclp_mems; + struct kset *kset; +}; + struct assign_storage_sccb { struct sccb_header header; u16 rn; @@ -163,92 +176,168 @@ static int sclp_mem_change_state(unsigned long start, unsigned long size, return rc ? -EIO : 0; } -static bool contains_standby_increment(unsigned long start, unsigned long end) +static ssize_t sclp_config_mem_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) { - struct memory_increment *incr; - unsigned long istart; + struct sclp_mem *sclp_mem = container_of(kobj, struct sclp_mem, kobj); - list_for_each_entry(incr, &sclp_mem_list, list) { - istart = rn2addr(incr->rn); - if (end - 1 < istart) - continue; - if (start > istart + sclp.rzm - 1) - continue; - if (incr->standby) - return true; - } - return false; + return sysfs_emit(buf, "%u\n", READ_ONCE(sclp_mem->config)); } -static int sclp_mem_notifier(struct notifier_block *nb, - unsigned long action, void *data) +static ssize_t sclp_config_mem_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) { - unsigned long start, size; - struct memory_notify *arg; + unsigned long addr, block_size; + struct sclp_mem *sclp_mem; + struct memory_block *mem; unsigned char id; - int rc = 0; + bool value; + int rc; - arg = data; - start = arg->start_pfn << PAGE_SHIFT; - size = arg->nr_pages << PAGE_SHIFT; - mutex_lock(&sclp_mem_mutex); + rc = kstrtobool(buf, &value); + if (rc) + return rc; + sclp_mem = container_of(kobj, struct sclp_mem, kobj); + block_size = memory_block_size_bytes(); + addr = sclp_mem->id * block_size; + /* + * Hold device_hotplug_lock when adding/removing memory blocks. + * Additionally, also protect calls to find_memory_block() and + * sclp_attach_storage(). + */ + rc = lock_device_hotplug_sysfs(); + if (rc) + goto out; for_each_clear_bit(id, sclp_storage_ids, sclp_max_storage_id + 1) sclp_attach_storage(id); - switch (action) { - case MEM_GOING_OFFLINE: - /* - * Do not allow to set memory blocks offline that contain - * standby memory. This is done to simplify the "memory online" - * case. - */ - if (contains_standby_increment(start, start + size)) - rc = -EPERM; - break; - case MEM_PREPARE_ONLINE: + if (value) { + if (sclp_mem->config) + goto out_unlock; + rc = sclp_mem_change_state(addr, block_size, 1); + if (rc) + goto out_unlock; /* - * Access the altmap_start_pfn and altmap_nr_pages fields - * within the struct memory_notify specifically when dealing - * with only MEM_PREPARE_ONLINE/MEM_FINISH_OFFLINE notifiers. - * - * When altmap is in use, take the specified memory range - * online, which includes the altmap. + * Set entire memory block CMMA state to nodat. Later, when + * page tables pages are allocated via __add_memory(), those + * regions are marked __arch_set_page_dat(). */ - if (arg->altmap_nr_pages) { - start = PFN_PHYS(arg->altmap_start_pfn); - size += PFN_PHYS(arg->altmap_nr_pages); + __arch_set_page_nodat((void *)__va(addr), block_size >> PAGE_SHIFT); + rc = __add_memory(0, addr, block_size, + sclp_mem->memmap_on_memory ? + MHP_MEMMAP_ON_MEMORY : MHP_NONE); + if (rc) { + sclp_mem_change_state(addr, block_size, 0); + goto out_unlock; } - rc = sclp_mem_change_state(start, size, 1); - if (rc || !arg->altmap_nr_pages) - break; - /* - * Set CMMA state to nodat here, since the struct page memory - * at the beginning of the memory block will not go through the - * buddy allocator later. - */ - __arch_set_page_nodat((void *)__va(start), arg->altmap_nr_pages); - break; - case MEM_FINISH_OFFLINE: - /* - * When altmap is in use, take the specified memory range - * offline, which includes the altmap. - */ - if (arg->altmap_nr_pages) { - start = PFN_PHYS(arg->altmap_start_pfn); - size += PFN_PHYS(arg->altmap_nr_pages); + mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(addr))); + put_device(&mem->dev); + WRITE_ONCE(sclp_mem->config, 1); + } else { + if (!sclp_mem->config) + goto out_unlock; + mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(addr))); + if (mem->state != MEM_OFFLINE) { + put_device(&mem->dev); + rc = -EBUSY; + goto out_unlock; } - sclp_mem_change_state(start, size, 0); - break; - default: - break; + /* drop the ref just got via find_memory_block() */ + put_device(&mem->dev); + sclp_mem_change_state(addr, block_size, 0); + __remove_memory(addr, block_size); + WRITE_ONCE(sclp_mem->config, 0); + } +out_unlock: + unlock_device_hotplug(); +out: + return rc ? rc : count; +} + +static struct kobj_attribute sclp_config_mem_attr = + __ATTR(config, 0644, sclp_config_mem_show, sclp_config_mem_store); + +static ssize_t sclp_memmap_on_memory_show(struct kobject *kobj, struct kobj_attribute *attr, char *buf) +{ + struct sclp_mem *sclp_mem = container_of(kobj, struct sclp_mem, kobj); + + return sysfs_emit(buf, "%u\n", READ_ONCE(sclp_mem->memmap_on_memory)); +} + +static ssize_t sclp_memmap_on_memory_store(struct kobject *kobj, struct kobj_attribute *attr, + const char *buf, size_t count) +{ + struct sclp_mem *sclp_mem; + unsigned long block_size; + struct memory_block *mem; + bool value; + int rc; + + rc = kstrtobool(buf, &value); + if (rc) + return rc; + if (value && !mhp_supports_memmap_on_memory()) + return -EOPNOTSUPP; + rc = lock_device_hotplug_sysfs(); + if (rc) + return rc; + block_size = memory_block_size_bytes(); + sclp_mem = container_of(kobj, struct sclp_mem, kobj); + mem = find_memory_block(pfn_to_section_nr(PFN_DOWN(sclp_mem->id * block_size))); + if (!mem) { + WRITE_ONCE(sclp_mem->memmap_on_memory, value); + } else { + put_device(&mem->dev); + rc = -EBUSY; } - mutex_unlock(&sclp_mem_mutex); - return rc ? NOTIFY_BAD : NOTIFY_OK; + unlock_device_hotplug(); + return rc ? rc : count; } -static struct notifier_block sclp_mem_nb = { - .notifier_call = sclp_mem_notifier, +static const struct kobj_type ktype = { + .sysfs_ops = &kobj_sysfs_ops, +}; + +static struct kobj_attribute sclp_memmap_attr = + __ATTR(memmap_on_memory, 0644, sclp_memmap_on_memory_show, sclp_memmap_on_memory_store); + +static struct attribute *sclp_mem_attrs[] = { + &sclp_config_mem_attr.attr, + &sclp_memmap_attr.attr, + NULL, }; +static struct attribute_group sclp_mem_attr_group = { + .attrs = sclp_mem_attrs, +}; + +static int sclp_create_mem(struct sclp_mem *sclp_mem, struct kset *kset, + unsigned int id, bool config, bool memmap_on_memory) +{ + int rc; + + sclp_mem->memmap_on_memory = memmap_on_memory; + sclp_mem->config = config; + sclp_mem->id = id; + kobject_init(&sclp_mem->kobj, &ktype); + rc = kobject_add(&sclp_mem->kobj, &kset->kobj, "memory%d", id); + if (rc) + return rc; + return sysfs_create_group(&sclp_mem->kobj, &sclp_mem_attr_group); +} + +static int sclp_create_configured_mem(struct memory_block *mem, void *argument) +{ + struct sclp_mem *sclp_mems; + struct sclp_mem_arg *arg; + struct kset *kset; + unsigned int id; + + id = mem->dev.id; + arg = (struct sclp_mem_arg *)argument; + sclp_mems = arg->sclp_mems; + kset = arg->kset; + return sclp_create_mem(&sclp_mems[id], kset, id, true, false); +} + static void __init align_to_block_size(unsigned long *start, unsigned long *size, unsigned long alignment) @@ -264,14 +353,17 @@ static void __init align_to_block_size(unsigned long *start, *size = size_align; } -static void __init add_memory_merged(u16 rn) +static int __init sclp_create_standby_mems_merged(struct sclp_mem *sclp_mems, + struct kset *kset, u16 rn) { unsigned long start, size, addr, block_size; static u16 first_rn, num; + unsigned int id; + int rc = 0; if (rn && first_rn && (first_rn + num == rn)) { num++; - return; + return rc; } if (!first_rn) goto skip_add; @@ -286,24 +378,57 @@ static void __init add_memory_merged(u16 rn) if (!size) goto skip_add; for (addr = start; addr < start + size; addr += block_size) { - add_memory(0, addr, block_size, - cpu_has_edat1() ? - MHP_MEMMAP_ON_MEMORY | MHP_OFFLINE_INACCESSIBLE : MHP_NONE); + id = addr / block_size; + rc = sclp_create_mem(&sclp_mems[id], kset, id, false, + mhp_supports_memmap_on_memory()); + if (rc) + break; } skip_add: first_rn = rn; num = 1; + return rc; } -static void __init sclp_add_standby_memory(void) +static int __init sclp_create_standby_mems(struct sclp_mem *sclp_mems, struct kset *kset) { struct memory_increment *incr; + int rc = 0; list_for_each_entry(incr, &sclp_mem_list, list) { if (incr->standby) - add_memory_merged(incr->rn); + rc = sclp_create_standby_mems_merged(sclp_mems, kset, incr->rn); + if (rc) + return rc; } - add_memory_merged(0); + return sclp_create_standby_mems_merged(sclp_mems, kset, 0); +} + +static int __init sclp_init_mem(void) +{ + const unsigned long block_size = memory_block_size_bytes(); + unsigned int max_sclp_mems; + struct sclp_mem *sclp_mems; + struct sclp_mem_arg arg; + struct kset *kset; + int rc; + + max_sclp_mems = roundup(sclp.rnmax * sclp.rzm, block_size) / block_size; + /* Allocate memory for all blocks ahead of time. */ + sclp_mems = kcalloc(max_sclp_mems, sizeof(struct sclp_mem), GFP_KERNEL); + if (!sclp_mems) + return -ENOMEM; + kset = kset_create_and_add("memory", NULL, firmware_kobj); + if (!kset) + return -ENOMEM; + /* Initial memory is in the "configured" state already. */ + arg.sclp_mems = sclp_mems; + arg.kset = kset; + rc = for_each_memory_block(&arg, sclp_create_configured_mem); + if (rc) + return rc; + /* Standby memory is "deconfigured". */ + return sclp_create_standby_mems(sclp_mems, kset); } static void __init insert_increment(u16 rn, int standby, int assigned) @@ -336,7 +461,7 @@ static void __init insert_increment(u16 rn, int standby, int assigned) list_add(&new_incr->list, prev); } -static int __init sclp_detect_standby_memory(void) +static int __init sclp_setup_memory(void) { struct read_storage_sccb *sccb; int i, id, assigned, rc; @@ -388,12 +513,9 @@ static int __init sclp_detect_standby_memory(void) goto out; for (i = 1; i <= sclp.rnmax - assigned; i++) insert_increment(0, 1, 0); - rc = register_memory_notifier(&sclp_mem_nb); - if (rc) - goto out; - sclp_add_standby_memory(); + rc = sclp_init_mem(); out: free_page((unsigned long)sccb); return rc; } -__initcall(sclp_detect_standby_memory); +__initcall(sclp_setup_memory); diff --git a/drivers/s390/char/sclp_ocf.c b/drivers/s390/char/sclp_ocf.c index ae2479b804d8..35f3a4a08b12 100644 --- a/drivers/s390/char/sclp_ocf.c +++ b/drivers/s390/char/sclp_ocf.c @@ -6,8 +6,7 @@ * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> */ -#define KMSG_COMPONENT "sclp_ocf" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_ocf: " fmt #include <linux/export.h> #include <linux/kernel.h> diff --git a/drivers/s390/char/sclp_pci.c b/drivers/s390/char/sclp_pci.c index 56400886f7fc..899063e64aef 100644 --- a/drivers/s390/char/sclp_pci.c +++ b/drivers/s390/char/sclp_pci.c @@ -4,8 +4,7 @@ * * Copyright IBM Corp. 2016 */ -#define KMSG_COMPONENT "sclp_cmd" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_cmd: " fmt #include <linux/completion.h> #include <linux/export.h> diff --git a/drivers/s390/char/sclp_sd.c b/drivers/s390/char/sclp_sd.c index 129b89fe40a3..bb1bce70ec00 100644 --- a/drivers/s390/char/sclp_sd.c +++ b/drivers/s390/char/sclp_sd.c @@ -5,8 +5,7 @@ * Copyright IBM Corp. 2017 */ -#define KMSG_COMPONENT "sclp_sd" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_sd: " fmt #include <linux/completion.h> #include <linux/jiffies.h> diff --git a/drivers/s390/char/sclp_sdias.c b/drivers/s390/char/sclp_sdias.c index e915a343fcf5..ab8f1b758a1a 100644 --- a/drivers/s390/char/sclp_sdias.c +++ b/drivers/s390/char/sclp_sdias.c @@ -6,8 +6,7 @@ * Author(s): Michael Holzheu */ -#define KMSG_COMPONENT "sclp_sdias" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "sclp_sdias: " fmt #include <linux/completion.h> #include <linux/sched.h> diff --git a/drivers/s390/char/tape.h b/drivers/s390/char/tape.h index 0aba30efb483..3953b31b0c55 100644 --- a/drivers/s390/char/tape.h +++ b/drivers/s390/char/tape.h @@ -130,6 +130,7 @@ struct tape_request { int retries; /* retry counter for error recovery. */ int rescnt; /* residual count from devstat. */ struct timer_list timer; /* timer for std_assign_timeout(). */ + struct irb irb; /* device status */ /* Callback for delivering final status. */ void (*callback)(struct tape_request *, void *); @@ -151,8 +152,8 @@ struct tape_discipline { int (*setup_device)(struct tape_device *); void (*cleanup_device)(struct tape_device *); int (*irq)(struct tape_device *, struct tape_request *, struct irb *); - struct tape_request *(*read_block)(struct tape_device *, size_t); - struct tape_request *(*write_block)(struct tape_device *, size_t); + struct tape_request *(*read_block)(struct tape_device *); + struct tape_request *(*write_block)(struct tape_device *); void (*process_eov)(struct tape_device*); /* ioctl function for additional ioctls. */ int (*ioctl_fn)(struct tape_device *, unsigned int, unsigned long); @@ -172,7 +173,7 @@ struct tape_discipline { /* Char Frontend Data */ struct tape_char_data { - struct idal_buffer *idal_buf; /* idal buffer for user char data */ + struct idal_buffer **ibs; /* idal buffer array for user char data */ int block_size; /* of size block_size. */ }; @@ -234,6 +235,7 @@ struct tape_device { /* Externals from tape_core.c */ extern struct tape_request *tape_alloc_request(int cplength, int datasize); extern void tape_free_request(struct tape_request *); +extern int tape_check_idalbuffer(struct tape_device *device, size_t size); extern int tape_do_io(struct tape_device *, struct tape_request *); extern int tape_do_io_async(struct tape_device *, struct tape_request *); extern int tape_do_io_interruptible(struct tape_device *, struct tape_request *); @@ -347,12 +349,21 @@ tape_ccw_repeat(struct ccw1 *ccw, __u8 cmd_code, int count) } static inline struct ccw1 * +tape_ccw_dc_idal(struct ccw1 *ccw, __u8 cmd_code, struct idal_buffer *idal) +{ + ccw->cmd_code = cmd_code; + ccw->flags = CCW_FLAG_DC; + idal_buffer_set_cda(idal, ccw); + return ccw + 1; +} + +static inline struct ccw1 * tape_ccw_cc_idal(struct ccw1 *ccw, __u8 cmd_code, struct idal_buffer *idal) { ccw->cmd_code = cmd_code; ccw->flags = CCW_FLAG_CC; idal_buffer_set_cda(idal, ccw); - return ccw++; + return ccw + 1; } static inline struct ccw1 * @@ -361,7 +372,7 @@ tape_ccw_end_idal(struct ccw1 *ccw, __u8 cmd_code, struct idal_buffer *idal) ccw->cmd_code = cmd_code; ccw->flags = 0; idal_buffer_set_cda(idal, ccw); - return ccw++; + return ccw + 1; } /* Global vars */ diff --git a/drivers/s390/char/tape_34xx.c b/drivers/s390/char/tape_34xx.c index 1e4984acb648..a13e0ac1a4e2 100644 --- a/drivers/s390/char/tape_34xx.c +++ b/drivers/s390/char/tape_34xx.c @@ -8,8 +8,7 @@ * Martin Schwidefsky <schwidefsky@de.ibm.com> */ -#define KMSG_COMPONENT "tape_34xx" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "tape_34xx: " fmt #include <linux/export.h> #include <linux/module.h> @@ -234,31 +233,6 @@ tape_34xx_unsolicited_irq(struct tape_device *device, struct irb *irb) return TAPE_IO_SUCCESS; } -/* - * Read Opposite Error Recovery Function: - * Used, when Read Forward does not work - */ -static int -tape_34xx_erp_read_opposite(struct tape_device *device, - struct tape_request *request) -{ - if (request->op == TO_RFO) { - /* - * We did read forward, but the data could not be read - * *correctly*. We transform the request to a read backward - * and try again. - */ - tape_std_read_backward(device, request); - return tape_34xx_erp_retry(request); - } - - /* - * We tried to read forward and backward, but hat no - * success -> failed. - */ - return tape_34xx_erp_failed(request, -EIO); -} - static int tape_34xx_erp_bug(struct tape_device *device, struct tape_request *request, struct irb *irb, int no) @@ -440,9 +414,6 @@ tape_34xx_unit_check(struct tape_device *device, struct tape_request *request, dev_warn (&device->cdev->dev, "A write error on the " "tape cannot be recovered\n"); return tape_34xx_erp_failed(request, -EIO); - case 0x26: - /* Data Check (read opposite) occurred. */ - return tape_34xx_erp_read_opposite(device, request); case 0x28: /* ID-Mark at tape start couldn't be written */ dev_warn (&device->cdev->dev, "Writing the ID-mark " diff --git a/drivers/s390/char/tape_3590.c b/drivers/s390/char/tape_3590.c index 2a2931d303cb..0d80f43b175d 100644 --- a/drivers/s390/char/tape_3590.c +++ b/drivers/s390/char/tape_3590.c @@ -8,8 +8,7 @@ * Martin Schwidefsky <schwidefsky@de.ibm.com> */ -#define KMSG_COMPONENT "tape_3590" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "tape_3590: " fmt #include <linux/export.h> #include <linux/module.h> @@ -551,31 +550,6 @@ tape_3590_mtseek(struct tape_device *device, int count) } /* - * Read Opposite Error Recovery Function: - * Used, when Read Forward does not work - */ -static void -tape_3590_read_opposite(struct tape_device *device, - struct tape_request *request) -{ - struct tape_3590_disc_data *data; - - /* - * We have allocated 4 ccws in tape_std_read, so we can now - * transform the request to a read backward, followed by a - * forward space block. - */ - request->op = TO_RBA; - tape_ccw_cc(request->cpaddr, MODE_SET_DB, 1, device->modeset_byte); - data = device->discdata; - tape_ccw_cc_idal(request->cpaddr + 1, data->read_back_op, - device->char_data.idal_buf); - tape_ccw_cc(request->cpaddr + 2, FORSPACEBLOCK, 0, NULL); - tape_ccw_end(request->cpaddr + 3, NOP, 0, NULL); - DBF_EVENT(6, "xrop ccwg\n"); -} - -/* * Read Attention Msg * This should be done after an interrupt with attention bit (0x80) * in device state. @@ -897,60 +871,6 @@ tape_3590_erp_special_interrupt(struct tape_device *device, } /* - * RDA: Read Alternate - */ -static int -tape_3590_erp_read_alternate(struct tape_device *device, - struct tape_request *request, struct irb *irb) -{ - struct tape_3590_disc_data *data; - - /* - * The issued Read Backward or Read Previous command is not - * supported by the device - * The recovery action should be to issue another command: - * Read Revious: if Read Backward is not supported - * Read Backward: if Read Previous is not supported - */ - data = device->discdata; - if (data->read_back_op == READ_PREVIOUS) { - DBF_EVENT(2, "(%08x): No support for READ_PREVIOUS command\n", - device->cdev_id); - data->read_back_op = READ_BACKWARD; - } else { - DBF_EVENT(2, "(%08x): No support for READ_BACKWARD command\n", - device->cdev_id); - data->read_back_op = READ_PREVIOUS; - } - tape_3590_read_opposite(device, request); - return tape_3590_erp_retry(device, request, irb); -} - -/* - * Error Recovery read opposite - */ -static int -tape_3590_erp_read_opposite(struct tape_device *device, - struct tape_request *request, struct irb *irb) -{ - switch (request->op) { - case TO_RFO: - /* - * We did read forward, but the data could not be read. - * We will read backward and then skip forward again. - */ - tape_3590_read_opposite(device, request); - return tape_3590_erp_retry(device, request, irb); - case TO_RBA: - /* We tried to read forward and backward, but hat no success */ - return tape_3590_erp_failed(device, request, irb, -EIO); - break; - default: - return tape_3590_erp_failed(device, request, irb, -EIO); - } -} - -/* * Print an MIM (Media Information Message) (message code f0) */ static void @@ -1348,10 +1268,6 @@ tape_3590_unit_check(struct tape_device *device, struct tape_request *request, tape_3590_print_era_msg(device, irb); return tape_3590_erp_read_buf_log(device, request, irb); - case 0x2011: - tape_3590_print_era_msg(device, irb); - return tape_3590_erp_read_alternate(device, request, irb); - case 0x2230: case 0x2231: tape_3590_print_era_msg(device, irb); @@ -1405,12 +1321,6 @@ tape_3590_unit_check(struct tape_device *device, struct tape_request *request, tape_3590_print_era_msg(device, irb); return tape_3590_erp_swap(device, request, irb); } - if (sense->rac == 0x26) { - /* Read Opposite */ - tape_3590_print_era_msg(device, irb); - return tape_3590_erp_read_opposite(device, request, - irb); - } return tape_3590_erp_basic(device, request, irb, -EIO); case 0x5020: case 0x5021: diff --git a/drivers/s390/char/tape_char.c b/drivers/s390/char/tape_char.c index 89778d922d9f..c5d3c303c15c 100644 --- a/drivers/s390/char/tape_char.c +++ b/drivers/s390/char/tape_char.c @@ -10,14 +10,12 @@ * Martin Schwidefsky <schwidefsky@de.ibm.com> */ -#define KMSG_COMPONENT "tape" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "tape: " fmt #include <linux/module.h> #include <linux/types.h> #include <linux/proc_fs.h> #include <linux/mtio.h> -#include <linux/compat.h> #include <linux/uaccess.h> @@ -37,9 +35,6 @@ static ssize_t tapechar_write(struct file *, const char __user *, size_t, loff_t static int tapechar_open(struct inode *,struct file *); static int tapechar_release(struct inode *,struct file *); static long tapechar_ioctl(struct file *, unsigned int, unsigned long); -#ifdef CONFIG_COMPAT -static long tapechar_compat_ioctl(struct file *, unsigned int, unsigned long); -#endif static const struct file_operations tape_fops = { @@ -47,9 +42,6 @@ static const struct file_operations tape_fops = .read = tapechar_read, .write = tapechar_write, .unlocked_ioctl = tapechar_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = tapechar_compat_ioctl, -#endif .open = tapechar_open, .release = tapechar_release, }; @@ -64,7 +56,7 @@ tapechar_setup_device(struct tape_device * device) { char device_name[20]; - sprintf(device_name, "ntibm%i", device->first_minor / 2); + scnprintf(device_name, sizeof(device_name), "ntibm%i", device->first_minor / 2); device->nt = register_tape_dev( &device->cdev->dev, MKDEV(tapechar_major, device->first_minor), @@ -93,33 +85,6 @@ tapechar_cleanup_device(struct tape_device *device) device->nt = NULL; } -static int -tapechar_check_idalbuffer(struct tape_device *device, size_t block_size) -{ - struct idal_buffer *new; - - if (device->char_data.idal_buf != NULL && - device->char_data.idal_buf->size == block_size) - return 0; - - if (block_size > MAX_BLOCKSIZE) { - DBF_EVENT(3, "Invalid blocksize (%zd > %d)\n", - block_size, MAX_BLOCKSIZE); - return -EINVAL; - } - - /* The current idal buffer is not correct. Allocate a new one. */ - new = idal_buffer_alloc(block_size, 0); - if (IS_ERR(new)) - return -ENOMEM; - - if (device->char_data.idal_buf != NULL) - idal_buffer_free(device->char_data.idal_buf); - - device->char_data.idal_buf = new; - - return 0; -} /* * Tape device read function @@ -127,9 +92,12 @@ tapechar_check_idalbuffer(struct tape_device *device, size_t block_size) static ssize_t tapechar_read(struct file *filp, char __user *data, size_t count, loff_t *ppos) { - struct tape_device *device; struct tape_request *request; + struct ccw1 *ccw, *last_ccw; + struct tape_device *device; + struct idal_buffer **ibs; size_t block_size; + size_t read = 0; int rc; DBF_EVENT(6, "TCHAR:read\n"); @@ -156,24 +124,37 @@ tapechar_read(struct file *filp, char __user *data, size_t count, loff_t *ppos) block_size = count; } - rc = tapechar_check_idalbuffer(device, block_size); + rc = tape_check_idalbuffer(device, block_size); if (rc) return rc; DBF_EVENT(6, "TCHAR:nbytes: %lx\n", block_size); /* Let the discipline build the ccw chain. */ - request = device->discipline->read_block(device, block_size); + request = device->discipline->read_block(device); if (IS_ERR(request)) return PTR_ERR(request); /* Execute it. */ rc = tape_do_io(device, request); if (rc == 0) { - rc = block_size - request->rescnt; DBF_EVENT(6, "TCHAR:rbytes: %x\n", rc); - /* Copy data from idal buffer to user space. */ - if (idal_buffer_to_user(device->char_data.idal_buf, - data, rc) != 0) - rc = -EFAULT; + /* Channel Program Address (cpa) points to last CCW + 8 */ + last_ccw = dma32_to_virt(request->irb.scsw.cmd.cpa); + ccw = request->cpaddr; + ibs = device->char_data.ibs; + while (++ccw < last_ccw) { + /* Copy data from idal buffer to user space. */ + if (idal_buffer_to_user(*ibs++, data, ccw->count) != 0) { + rc = -EFAULT; + break; + } + read += ccw->count; + data += ccw->count; + } + if (&last_ccw[-1] == &request->cpaddr[1] && + request->rescnt == last_ccw[-1].count) + rc = 0; + else + rc = read - request->rescnt; } tape_free_request(request); return rc; @@ -185,10 +166,12 @@ tapechar_read(struct file *filp, char __user *data, size_t count, loff_t *ppos) static ssize_t tapechar_write(struct file *filp, const char __user *data, size_t count, loff_t *ppos) { - struct tape_device *device; struct tape_request *request; + struct ccw1 *ccw, *last_ccw; + struct tape_device *device; + struct idal_buffer **ibs; + size_t written = 0; size_t block_size; - size_t written; int nblocks; int i, rc; @@ -208,35 +191,45 @@ tapechar_write(struct file *filp, const char __user *data, size_t count, loff_t nblocks = 1; } - rc = tapechar_check_idalbuffer(device, block_size); + rc = tape_check_idalbuffer(device, block_size); if (rc) return rc; - DBF_EVENT(6,"TCHAR:nbytes: %lx\n", block_size); + DBF_EVENT(6, "TCHAR:nbytes: %lx\n", block_size); DBF_EVENT(6, "TCHAR:nblocks: %x\n", nblocks); /* Let the discipline build the ccw chain. */ - request = device->discipline->write_block(device, block_size); + request = device->discipline->write_block(device); if (IS_ERR(request)) return PTR_ERR(request); - rc = 0; - written = 0; + for (i = 0; i < nblocks; i++) { - /* Copy data from user space to idal buffer. */ - if (idal_buffer_from_user(device->char_data.idal_buf, - data, block_size)) { - rc = -EFAULT; - break; + size_t wbytes = 0; /* Used to trace written data in dbf */ + + ibs = device->char_data.ibs; + while (ibs && *ibs) { + if (idal_buffer_from_user(*ibs, data, (*ibs)->size)) { + rc = -EFAULT; + goto out; + } + data += (*ibs)->size; + ibs++; } rc = tape_do_io(device, request); if (rc) - break; - DBF_EVENT(6, "TCHAR:wbytes: %lx\n", - block_size - request->rescnt); - written += block_size - request->rescnt; + goto out; + + /* Channel Program Address (cpa) points to last CCW + 8 */ + last_ccw = dma32_to_virt(request->irb.scsw.cmd.cpa); + ccw = request->cpaddr; + while (++ccw < last_ccw) + wbytes += ccw->count; + DBF_EVENT(6, "TCHAR:wbytes: %lx\n", wbytes - request->rescnt); + written += wbytes - request->rescnt; if (request->rescnt != 0) break; - data += block_size; } + +out: tape_free_request(request); if (rc == -ENOSPC) { /* @@ -324,10 +317,8 @@ tapechar_release(struct inode *inode, struct file *filp) } } - if (device->char_data.idal_buf != NULL) { - idal_buffer_free(device->char_data.idal_buf); - device->char_data.idal_buf = NULL; - } + if (device->char_data.ibs) + idal_buffer_array_free(&device->char_data.ibs); tape_release(device); filp->private_data = NULL; tape_put_device(device); @@ -442,25 +433,6 @@ tapechar_ioctl(struct file *filp, unsigned int no, unsigned long data) return rc; } -#ifdef CONFIG_COMPAT -static long -tapechar_compat_ioctl(struct file *filp, unsigned int no, unsigned long data) -{ - struct tape_device *device = filp->private_data; - long rc; - - if (no == MTIOCPOS32) - no = MTIOCPOS; - else if (no == MTIOCGET32) - no = MTIOCGET; - - mutex_lock(&device->mutex); - rc = __tapechar_ioctl(device, no, compat_ptr(data)); - mutex_unlock(&device->mutex); - return rc; -} -#endif /* CONFIG_COMPAT */ - /* * Initialize character device frontend. */ diff --git a/drivers/s390/char/tape_class.c b/drivers/s390/char/tape_class.c index fb18adfb95b5..6fa7b7824856 100644 --- a/drivers/s390/char/tape_class.c +++ b/drivers/s390/char/tape_class.c @@ -8,8 +8,7 @@ * Based on simple class device code by Greg K-H */ -#define KMSG_COMPONENT "tape" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "tape: " fmt #include <linux/export.h> #include <linux/slab.h> diff --git a/drivers/s390/char/tape_core.c b/drivers/s390/char/tape_core.c index 6ec812280221..0250076a7d9f 100644 --- a/drivers/s390/char/tape_core.c +++ b/drivers/s390/char/tape_core.c @@ -11,8 +11,7 @@ * Stefan Bader <shbader@de.ibm.com> */ -#define KMSG_COMPONENT "tape" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "tape: " fmt #include <linux/export.h> #include <linux/module.h> @@ -726,6 +725,36 @@ tape_free_request (struct tape_request * request) kfree(request); } +int +tape_check_idalbuffer(struct tape_device *device, size_t size) +{ + struct idal_buffer **new; + size_t old_size = 0; + + old_size = idal_buffer_array_datasize(device->char_data.ibs); + if (old_size == size) + return 0; + + if (size > MAX_BLOCKSIZE) { + DBF_EVENT(3, "Invalid blocksize (%zd > %d)\n", + size, MAX_BLOCKSIZE); + return -EINVAL; + } + + /* The current idal buffer is not correct. Allocate a new one. */ + new = idal_buffer_array_alloc(size, 0); + if (IS_ERR(new)) + return -ENOMEM; + + /* Free old idal buffer array */ + if (device->char_data.ibs) + idal_buffer_array_free(&device->char_data.ibs); + + device->char_data.ibs = new; + + return 0; +} + static int __tape_start_io(struct tape_device *device, struct tape_request *request) { @@ -1099,9 +1128,10 @@ __tape_do_irq (struct ccw_device *cdev, unsigned long intparm, struct irb *irb) } /* May be an unsolicited irq */ - if(request != NULL) + if (request != NULL) { request->rescnt = irb->scsw.cmd.count; - else if ((irb->scsw.cmd.dstat == 0x85 || irb->scsw.cmd.dstat == 0x80) && + memcpy(&request->irb, irb, sizeof(*irb)); + } else if ((irb->scsw.cmd.dstat == 0x85 || irb->scsw.cmd.dstat == 0x80) && !list_empty(&device->req_queue)) { /* Not Ready to Ready after long busy ? */ struct tape_request *req; diff --git a/drivers/s390/char/tape_proc.c b/drivers/s390/char/tape_proc.c index 2238d9df6c47..a1e5fab12af2 100644 --- a/drivers/s390/char/tape_proc.c +++ b/drivers/s390/char/tape_proc.c @@ -11,8 +11,7 @@ * PROCFS Functions */ -#define KMSG_COMPONENT "tape" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "tape: " fmt #include <linux/module.h> #include <linux/vmalloc.h> diff --git a/drivers/s390/char/tape_std.c b/drivers/s390/char/tape_std.c index 176ae8e2eb6b..43a5586685ff 100644 --- a/drivers/s390/char/tape_std.c +++ b/drivers/s390/char/tape_std.c @@ -11,8 +11,7 @@ * Stefan Bader <shbader@de.ibm.com> */ -#define KMSG_COMPONENT "tape" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "tape: " fmt #include <linux/export.h> #include <linux/stddef.h> @@ -212,7 +211,7 @@ tape_std_mtload(struct tape_device *device, int count) int tape_std_mtsetblk(struct tape_device *device, int count) { - struct idal_buffer *new; + int rc; DBF_LH(6, "tape_std_mtsetblk(%d)\n", count); if (count <= 0) { @@ -224,26 +223,12 @@ tape_std_mtsetblk(struct tape_device *device, int count) device->char_data.block_size = 0; return 0; } - if (device->char_data.idal_buf != NULL && - device->char_data.idal_buf->size == count) - /* We already have a idal buffer of that size. */ - return 0; - if (count > MAX_BLOCKSIZE) { - DBF_EVENT(3, "Invalid block size (%d > %d) given.\n", - count, MAX_BLOCKSIZE); - return -EINVAL; - } + rc = tape_check_idalbuffer(device, count); + if (rc) + return rc; - /* Allocate a new idal buffer. */ - new = idal_buffer_alloc(count, 0); - if (IS_ERR(new)) - return -ENOMEM; - if (device->char_data.idal_buf != NULL) - idal_buffer_free(device->char_data.idal_buf); - device->char_data.idal_buf = new; device->char_data.block_size = count; - DBF_LH(6, "new blocksize is %d\n", device->char_data.block_size); return 0; @@ -641,63 +626,54 @@ tape_std_mtcompression(struct tape_device *device, int mt_count) * Read Block */ struct tape_request * -tape_std_read_block(struct tape_device *device, size_t count) +tape_std_read_block(struct tape_device *device) { struct tape_request *request; + struct idal_buffer **ibs; + struct ccw1 *ccw; + size_t count; - /* - * We have to alloc 4 ccws in order to be able to transform request - * into a read backward request in error case. - */ - request = tape_alloc_request(4, 0); + ibs = device->char_data.ibs; + count = idal_buffer_array_size(ibs); + request = tape_alloc_request(count + 1 /* MODE_SET_DB */, 0); if (IS_ERR(request)) { DBF_EXCEPTION(6, "xrbl fail"); return request; } request->op = TO_RFO; - tape_ccw_cc(request->cpaddr, MODE_SET_DB, 1, device->modeset_byte); - tape_ccw_end_idal(request->cpaddr + 1, READ_FORWARD, - device->char_data.idal_buf); + ccw = tape_ccw_cc(request->cpaddr, MODE_SET_DB, 1, device->modeset_byte); + while (count-- > 1) + ccw = tape_ccw_dc_idal(ccw, READ_FORWARD, *ibs++); + tape_ccw_end_idal(ccw, READ_FORWARD, *ibs); + DBF_EVENT(6, "xrbl ccwg\n"); return request; } /* - * Read Block backward transformation function. - */ -void -tape_std_read_backward(struct tape_device *device, struct tape_request *request) -{ - /* - * We have allocated 4 ccws in tape_std_read, so we can now - * transform the request to a read backward, followed by a - * forward space block. - */ - request->op = TO_RBA; - tape_ccw_cc(request->cpaddr, MODE_SET_DB, 1, device->modeset_byte); - tape_ccw_cc_idal(request->cpaddr + 1, READ_BACKWARD, - device->char_data.idal_buf); - tape_ccw_cc(request->cpaddr + 2, FORSPACEBLOCK, 0, NULL); - tape_ccw_end(request->cpaddr + 3, NOP, 0, NULL); - DBF_EVENT(6, "xrop ccwg");} - -/* * Write Block */ struct tape_request * -tape_std_write_block(struct tape_device *device, size_t count) +tape_std_write_block(struct tape_device *device) { struct tape_request *request; + struct idal_buffer **ibs; + struct ccw1 *ccw; + size_t count; - request = tape_alloc_request(2, 0); + count = idal_buffer_array_size(device->char_data.ibs); + request = tape_alloc_request(count + 1 /* MODE_SET_DB */, 0); if (IS_ERR(request)) { DBF_EXCEPTION(6, "xwbl fail\n"); return request; } request->op = TO_WRI; - tape_ccw_cc(request->cpaddr, MODE_SET_DB, 1, device->modeset_byte); - tape_ccw_end_idal(request->cpaddr + 1, WRITE_CMD, - device->char_data.idal_buf); + ccw = tape_ccw_cc(request->cpaddr, MODE_SET_DB, 1, device->modeset_byte); + ibs = device->char_data.ibs; + while (count-- > 1) + ccw = tape_ccw_dc_idal(ccw, WRITE_CMD, *ibs++); + tape_ccw_end_idal(ccw, WRITE_CMD, *ibs); + DBF_EVENT(6, "xwbl ccwg\n"); return request; } @@ -741,6 +717,5 @@ EXPORT_SYMBOL(tape_std_mterase); EXPORT_SYMBOL(tape_std_mtunload); EXPORT_SYMBOL(tape_std_mtcompression); EXPORT_SYMBOL(tape_std_read_block); -EXPORT_SYMBOL(tape_std_read_backward); EXPORT_SYMBOL(tape_std_write_block); EXPORT_SYMBOL(tape_std_process_eov); diff --git a/drivers/s390/char/tape_std.h b/drivers/s390/char/tape_std.h index dcc63ff587f9..2cf9f725b3b3 100644 --- a/drivers/s390/char/tape_std.h +++ b/drivers/s390/char/tape_std.h @@ -14,10 +14,9 @@ #include <asm/tape390.h> /* - * Biggest block size to handle. Currently 64K because we only build - * channel programs without data chaining. + * Biggest block size of 256K to handle. */ -#define MAX_BLOCKSIZE 65535 +#define MAX_BLOCKSIZE 262144 /* * The CCW commands for the Tape type of command. @@ -97,10 +96,10 @@ #define SENSE_TAPE_POSITIONING 0x01 /* discipline functions */ -struct tape_request *tape_std_read_block(struct tape_device *, size_t); +struct tape_request *tape_std_read_block(struct tape_device *); void tape_std_read_backward(struct tape_device *device, struct tape_request *request); -struct tape_request *tape_std_write_block(struct tape_device *, size_t); +struct tape_request *tape_std_write_block(struct tape_device *); /* Some non-mtop commands. */ int tape_std_assign(struct tape_device *); diff --git a/drivers/s390/char/vmcp.c b/drivers/s390/char/vmcp.c index 69899bb86b3e..bde6c9e59166 100644 --- a/drivers/s390/char/vmcp.c +++ b/drivers/s390/char/vmcp.c @@ -14,7 +14,6 @@ #include <linux/fs.h> #include <linux/init.h> -#include <linux/compat.h> #include <linux/kernel.h> #include <linux/miscdevice.h> #include <linux/slab.h> @@ -204,10 +203,7 @@ static long vmcp_ioctl(struct file *file, unsigned int cmd, unsigned long arg) int __user *argp; session = file->private_data; - if (is_compat_task()) - argp = compat_ptr(arg); - else - argp = (int __user *)arg; + argp = (int __user *)arg; if (mutex_lock_interruptible(&session->mutex)) return -ERESTARTSYS; switch (cmd) { @@ -241,7 +237,6 @@ static const struct file_operations vmcp_fops = { .read = vmcp_read, .write = vmcp_write, .unlocked_ioctl = vmcp_ioctl, - .compat_ioctl = vmcp_ioctl, }; static struct miscdevice vmcp_dev = { diff --git a/drivers/s390/char/vmlogrdr.c b/drivers/s390/char/vmlogrdr.c index e284eea331d7..383e7e2bd69f 100644 --- a/drivers/s390/char/vmlogrdr.c +++ b/drivers/s390/char/vmlogrdr.c @@ -11,8 +11,7 @@ * */ -#define KMSG_COMPONENT "vmlogrdr" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "vmlogrdr: " fmt #include <linux/module.h> #include <linux/init.h> diff --git a/drivers/s390/char/vmur.c b/drivers/s390/char/vmur.c index 0fd918769a4b..e3e0e9f36527 100644 --- a/drivers/s390/char/vmur.c +++ b/drivers/s390/char/vmur.c @@ -9,8 +9,7 @@ * Frank Munzert <munzert@de.ibm.com> */ -#define KMSG_COMPONENT "vmur" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "vmur: " fmt #include <linux/cdev.h> #include <linux/slab.h> diff --git a/drivers/s390/char/zcore.c b/drivers/s390/char/zcore.c index 33cebb91b933..b26b5fca6ce8 100644 --- a/drivers/s390/char/zcore.c +++ b/drivers/s390/char/zcore.c @@ -9,8 +9,7 @@ * Author(s): Michael Holzheu */ -#define KMSG_COMPONENT "zdump" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zdump: " fmt #include <linux/init.h> #include <linux/slab.h> diff --git a/drivers/s390/cio/blacklist.c b/drivers/s390/cio/blacklist.c index 93695d535380..738d5e2d5304 100644 --- a/drivers/s390/cio/blacklist.c +++ b/drivers/s390/cio/blacklist.c @@ -8,8 +8,7 @@ * Arnd Bergmann (arndb@de.ibm.com) */ -#define KMSG_COMPONENT "cio" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "cio: " fmt #include <linux/init.h> #include <linux/vmalloc.h> diff --git a/drivers/s390/cio/ccwgroup.c b/drivers/s390/cio/ccwgroup.c index 2fc2ea4b2e3b..185c99c5d4cc 100644 --- a/drivers/s390/cio/ccwgroup.c +++ b/drivers/s390/cio/ccwgroup.c @@ -41,7 +41,7 @@ static void __ccwgroup_remove_symlinks(struct ccwgroup_device *gdev) char str[16]; for (i = 0; i < gdev->count; i++) { - sprintf(str, "cdev%d", i); + scnprintf(str, sizeof(str), "cdev%d", i); sysfs_remove_link(&gdev->dev.kobj, str); sysfs_remove_link(&gdev->cdev[i]->dev.kobj, "group_device"); } @@ -249,12 +249,12 @@ static int __ccwgroup_create_symlinks(struct ccwgroup_device *gdev) } } for (i = 0; i < gdev->count; i++) { - sprintf(str, "cdev%d", i); + scnprintf(str, sizeof(str), "cdev%d", i); rc = sysfs_create_link(&gdev->dev.kobj, &gdev->cdev[i]->dev.kobj, str); if (rc) { while (i--) { - sprintf(str, "cdev%d", i); + scnprintf(str, sizeof(str), "cdev%d", i); sysfs_remove_link(&gdev->dev.kobj, str); } for (i = 0; i < gdev->count; i++) diff --git a/drivers/s390/cio/ccwreq.c b/drivers/s390/cio/ccwreq.c index 73582a0a2622..763f477cc431 100644 --- a/drivers/s390/cio/ccwreq.c +++ b/drivers/s390/cio/ccwreq.c @@ -6,8 +6,7 @@ * Author(s): Peter Oberparleiter <peter.oberparleiter@de.ibm.com> */ -#define KMSG_COMPONENT "cio" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "cio: " fmt #include <linux/types.h> #include <linux/err.h> diff --git a/drivers/s390/cio/chp.c b/drivers/s390/cio/chp.c index caa300160b17..c10e2444507e 100644 --- a/drivers/s390/cio/chp.c +++ b/drivers/s390/cio/chp.c @@ -111,8 +111,9 @@ static int s390_vary_chpid(struct chp_id chpid, int on) char dbf_text[15]; int status; - sprintf(dbf_text, on?"varyon%x.%02x":"varyoff%x.%02x", chpid.cssid, - chpid.id); + scnprintf(dbf_text, sizeof(dbf_text), + on ? "varyon%x.%02x" : "varyoff%x.%02x", + chpid.cssid, chpid.id); CIO_TRACE_EVENT(2, dbf_text); status = chp_get_status(chpid); diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c index 239c92d4ec11..fbb58edd6274 100644 --- a/drivers/s390/cio/chsc.c +++ b/drivers/s390/cio/chsc.c @@ -8,8 +8,7 @@ * Arnd Bergmann (arndb@de.ibm.com) */ -#define KMSG_COMPONENT "cio" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "cio: " fmt #include <linux/export.h> #include <linux/module.h> @@ -253,7 +252,7 @@ void chsc_chp_offline(struct chp_id chpid) struct chp_link link; char dbf_txt[15]; - sprintf(dbf_txt, "chpr%x.%02x", chpid.cssid, chpid.id); + scnprintf(dbf_txt, sizeof(dbf_txt), "chpr%x.%02x", chpid.cssid, chpid.id); CIO_TRACE_EVENT(2, dbf_txt); if (chp_get_status(chpid) <= 0) @@ -284,11 +283,11 @@ static void s390_process_res_acc(struct chp_link *link) { char dbf_txt[15]; - sprintf(dbf_txt, "accpr%x.%02x", link->chpid.cssid, - link->chpid.id); + scnprintf(dbf_txt, sizeof(dbf_txt), "accpr%x.%02x", link->chpid.cssid, + link->chpid.id); CIO_TRACE_EVENT( 2, dbf_txt); if (link->fla != 0) { - sprintf(dbf_txt, "fla%x", link->fla); + scnprintf(dbf_txt, sizeof(dbf_txt), "fla%x", link->fla); CIO_TRACE_EVENT( 2, dbf_txt); } /* Wait until previous actions have settled. */ @@ -757,7 +756,7 @@ void chsc_chp_online(struct chp_id chpid) struct chp_link link; char dbf_txt[15]; - sprintf(dbf_txt, "cadd%x.%02x", chpid.cssid, chpid.id); + scnprintf(dbf_txt, sizeof(dbf_txt), "cadd%x.%02x", chpid.cssid, chpid.id); CIO_TRACE_EVENT(2, dbf_txt); if (chp_get_status(chpid) != 0) { diff --git a/drivers/s390/cio/chsc_sch.c b/drivers/s390/cio/chsc_sch.c index 1e58ee3cc87d..ce992b2107cb 100644 --- a/drivers/s390/cio/chsc_sch.c +++ b/drivers/s390/cio/chsc_sch.c @@ -9,7 +9,6 @@ */ #include <linux/slab.h> -#include <linux/compat.h> #include <linux/device.h> #include <linux/io.h> #include <linux/module.h> @@ -845,10 +844,7 @@ static long chsc_ioctl(struct file *filp, unsigned int cmd, void __user *argp; CHSC_MSG(2, "chsc_ioctl called, cmd=%x\n", cmd); - if (is_compat_task()) - argp = compat_ptr(arg); - else - argp = (void __user *)arg; + argp = (void __user *)arg; switch (cmd) { case CHSC_START: return chsc_ioctl_start(argp); @@ -923,7 +919,6 @@ static const struct file_operations chsc_fops = { .open = chsc_open, .release = chsc_release, .unlocked_ioctl = chsc_ioctl, - .compat_ioctl = chsc_ioctl, }; static struct miscdevice chsc_misc_device = { diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c index 21508e4606d5..70dc8cc76594 100644 --- a/drivers/s390/cio/cio.c +++ b/drivers/s390/cio/cio.c @@ -9,8 +9,7 @@ * Martin Schwidefsky (schwidefsky@de.ibm.com) */ -#define KMSG_COMPONENT "cio" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "cio: " fmt #include <linux/export.h> #include <linux/ftrace.h> @@ -113,7 +112,7 @@ cio_start_handle_notoper(struct subchannel *sch, __u8 lpm) if (cio_update_schib(sch)) return -ENODEV; - sprintf(dbf_text, "no%s", dev_name(&sch->dev)); + scnprintf(dbf_text, sizeof(dbf_text), "no%s", dev_name(&sch->dev)); CIO_TRACE_EVENT(0, dbf_text); CIO_HEX_EVENT(0, &sch->schib, sizeof (struct schib)); diff --git a/drivers/s390/cio/cio_inject.c b/drivers/s390/cio/cio_inject.c index a2e771ebae8e..0e18cb921ef6 100644 --- a/drivers/s390/cio/cio_inject.c +++ b/drivers/s390/cio/cio_inject.c @@ -6,8 +6,7 @@ * Author(s): Vineeth Vijayan <vneethv@linux.ibm.com> */ -#define KMSG_COMPONENT "cio" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "cio: " fmt #include <linux/slab.h> #include <linux/spinlock.h> diff --git a/drivers/s390/cio/cmf.c b/drivers/s390/cio/cmf.c index b7048f2b036e..7d035e4937ce 100644 --- a/drivers/s390/cio/cmf.c +++ b/drivers/s390/cio/cmf.c @@ -10,8 +10,7 @@ * original idea from Natarajan Krishnaswami <nkrishna@us.ibm.com> */ -#define KMSG_COMPONENT "cio" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "cio: " fmt #include <linux/memblock.h> #include <linux/device.h> diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c index be78a57f9bfd..4c85df7a548e 100644 --- a/drivers/s390/cio/css.c +++ b/drivers/s390/cio/css.c @@ -8,8 +8,7 @@ * Cornelia Huck (cornelia.huck@de.ibm.com) */ -#define KMSG_COMPONENT "cio" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "cio: " fmt #include <linux/export.h> #include <linux/init.h> diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c index 4b2dae6eb376..602f36102c7c 100644 --- a/drivers/s390/cio/device.c +++ b/drivers/s390/cio/device.c @@ -8,8 +8,7 @@ * Martin Schwidefsky (schwidefsky@de.ibm.com) */ -#define KMSG_COMPONENT "cio" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "cio: " fmt #include <linux/export.h> #include <linux/init.h> diff --git a/drivers/s390/cio/device_status.c b/drivers/s390/cio/device_status.c index 0ff8482a7b15..f4096373c8c0 100644 --- a/drivers/s390/cio/device_status.c +++ b/drivers/s390/cio/device_status.c @@ -42,7 +42,7 @@ ccw_device_msg_control_check(struct ccw_device *cdev, struct irb *irb) cdev->private->dev_id.devno, sch->schid.ssid, sch->schid.sch_no, scsw_dstat(&irb->scsw), scsw_cstat(&irb->scsw)); - sprintf(dbf_text, "chk%x", sch->schid.sch_no); + scnprintf(dbf_text, sizeof(dbf_text), "chk%x", sch->schid.sch_no); CIO_TRACE_EVENT(0, dbf_text); CIO_HEX_EVENT(0, irb, sizeof(struct irb)); } diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c index 65f1a127cc3f..a445494fd2be 100644 --- a/drivers/s390/crypto/ap_bus.c +++ b/drivers/s390/crypto/ap_bus.c @@ -11,8 +11,7 @@ * Adjunct processor bus. */ -#define KMSG_COMPONENT "ap" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ap: " fmt #include <linux/kernel_stat.h> #include <linux/moduleparam.h> @@ -86,8 +85,17 @@ DEFINE_SPINLOCK(ap_queues_lock); /* Default permissions (ioctl, card and domain masking) */ struct ap_perms ap_perms; EXPORT_SYMBOL(ap_perms); -DEFINE_MUTEX(ap_perms_mutex); -EXPORT_SYMBOL(ap_perms_mutex); +/* true if apmask and/or aqmask are NOT default */ +bool ap_apmask_aqmask_in_use; +/* counter for how many driver_overrides are currently active */ +int ap_driver_override_ctr; +/* + * Mutex for consistent read and write of the ap_perms struct, + * ap_apmask_aqmask_in_use, ap_driver_override_ctr + * and the ap bus sysfs attributes apmask and aqmask. + */ +DEFINE_MUTEX(ap_attr_mutex); +EXPORT_SYMBOL(ap_attr_mutex); /* # of bindings complete since init */ static atomic64_t ap_bindings_complete_count = ATOMIC64_INIT(0); @@ -853,20 +861,38 @@ static int __ap_revise_reserved(struct device *dev, void *dummy) int rc, card, queue, devres, drvres; if (is_queue_dev(dev)) { - card = AP_QID_CARD(to_ap_queue(dev)->qid); - queue = AP_QID_QUEUE(to_ap_queue(dev)->qid); - mutex_lock(&ap_perms_mutex); - devres = test_bit_inv(card, ap_perms.apm) && - test_bit_inv(queue, ap_perms.aqm); - mutex_unlock(&ap_perms_mutex); - drvres = to_ap_drv(dev->driver)->flags - & AP_DRIVER_FLAG_DEFAULT; - if (!!devres != !!drvres) { - pr_debug("reprobing queue=%02x.%04x\n", card, queue); - rc = device_reprobe(dev); - if (rc) - AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", - __func__, card, queue); + struct ap_driver *ap_drv = to_ap_drv(dev->driver); + struct ap_queue *aq = to_ap_queue(dev); + struct ap_device *ap_dev = &aq->ap_dev; + + card = AP_QID_CARD(aq->qid); + queue = AP_QID_QUEUE(aq->qid); + + if (ap_dev->driver_override) { + if (strcmp(ap_dev->driver_override, + ap_drv->driver.name)) { + pr_debug("reprobing queue=%02x.%04x\n", card, queue); + rc = device_reprobe(dev); + if (rc) { + AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", + __func__, card, queue); + } + } + } else { + mutex_lock(&ap_attr_mutex); + devres = test_bit_inv(card, ap_perms.apm) && + test_bit_inv(queue, ap_perms.aqm); + mutex_unlock(&ap_attr_mutex); + drvres = to_ap_drv(dev->driver)->flags + & AP_DRIVER_FLAG_DEFAULT; + if (!!devres != !!drvres) { + pr_debug("reprobing queue=%02x.%04x\n", card, queue); + rc = device_reprobe(dev); + if (rc) { + AP_DBF_WARN("%s reprobing queue=%02x.%04x failed\n", + __func__, card, queue); + } + } } } @@ -884,22 +910,37 @@ static void ap_bus_revise_bindings(void) * @card: the APID of the adapter card to check * @queue: the APQI of the queue to check * - * Note: the ap_perms_mutex must be locked by the caller of this function. + * Note: the ap_attr_mutex must be locked by the caller of this function. * * Return: an int specifying whether the AP adapter is reserved for the host (1) * or not (0). */ int ap_owned_by_def_drv(int card, int queue) { + struct ap_queue *aq; int rc = 0; if (card < 0 || card >= AP_DEVICES || queue < 0 || queue >= AP_DOMAINS) return -EINVAL; + aq = ap_get_qdev(AP_MKQID(card, queue)); + if (aq) { + const struct device_driver *drv = aq->ap_dev.device.driver; + const struct ap_driver *ap_drv = to_ap_drv(drv); + bool override = !!aq->ap_dev.driver_override; + + if (override && drv && ap_drv->flags & AP_DRIVER_FLAG_DEFAULT) + rc = 1; + put_device(&aq->ap_dev.device); + if (override) + goto out; + } + if (test_bit_inv(card, ap_perms.apm) && test_bit_inv(queue, ap_perms.aqm)) rc = 1; +out: return rc; } EXPORT_SYMBOL(ap_owned_by_def_drv); @@ -911,7 +952,7 @@ EXPORT_SYMBOL(ap_owned_by_def_drv); * @apm: a bitmap specifying a set of APIDs comprising the APQNs to check * @aqm: a bitmap specifying a set of APQIs comprising the APQNs to check * - * Note: the ap_perms_mutex must be locked by the caller of this function. + * Note: the ap_attr_mutex must be locked by the caller of this function. * * Return: an int specifying whether each APQN is reserved for the host (1) or * not (0) @@ -922,12 +963,10 @@ int ap_apqn_in_matrix_owned_by_def_drv(unsigned long *apm, int card, queue, rc = 0; for (card = 0; !rc && card < AP_DEVICES; card++) - if (test_bit_inv(card, apm) && - test_bit_inv(card, ap_perms.apm)) + if (test_bit_inv(card, apm)) for (queue = 0; !rc && queue < AP_DOMAINS; queue++) - if (test_bit_inv(queue, aqm) && - test_bit_inv(queue, ap_perms.aqm)) - rc = 1; + if (test_bit_inv(queue, aqm)) + rc = ap_owned_by_def_drv(card, queue); return rc; } @@ -951,13 +990,19 @@ static int ap_device_probe(struct device *dev) */ card = AP_QID_CARD(to_ap_queue(dev)->qid); queue = AP_QID_QUEUE(to_ap_queue(dev)->qid); - mutex_lock(&ap_perms_mutex); - devres = test_bit_inv(card, ap_perms.apm) && - test_bit_inv(queue, ap_perms.aqm); - mutex_unlock(&ap_perms_mutex); - drvres = ap_drv->flags & AP_DRIVER_FLAG_DEFAULT; - if (!!devres != !!drvres) - goto out; + if (ap_dev->driver_override) { + if (strcmp(ap_dev->driver_override, + ap_drv->driver.name)) + goto out; + } else { + mutex_lock(&ap_attr_mutex); + devres = test_bit_inv(card, ap_perms.apm) && + test_bit_inv(queue, ap_perms.aqm); + mutex_unlock(&ap_attr_mutex); + drvres = ap_drv->flags & AP_DRIVER_FLAG_DEFAULT; + if (!!devres != !!drvres) + goto out; + } } /* @@ -983,8 +1028,17 @@ static int ap_device_probe(struct device *dev) } out: - if (rc) + if (rc) { put_device(dev); + } else { + if (is_queue_dev(dev)) { + pr_debug("queue=%02x.%04x new driver=%s\n", + card, queue, ap_drv->driver.name); + } else { + pr_debug("card=%02x new driver=%s\n", + to_ap_card(dev)->id, ap_drv->driver.name); + } + } return rc; } @@ -1437,12 +1491,12 @@ static ssize_t apmask_show(const struct bus_type *bus, char *buf) { int rc; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; rc = sysfs_emit(buf, "0x%016lx%016lx%016lx%016lx\n", ap_perms.apm[0], ap_perms.apm[1], ap_perms.apm[2], ap_perms.apm[3]); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return rc; } @@ -1452,6 +1506,7 @@ static int __verify_card_reservations(struct device_driver *drv, void *data) int rc = 0; struct ap_driver *ap_drv = to_ap_drv(drv); unsigned long *newapm = (unsigned long *)data; + unsigned long aqm_any[BITS_TO_LONGS(AP_DOMAINS)]; /* * increase the driver's module refcounter to be sure it is not @@ -1461,7 +1516,8 @@ static int __verify_card_reservations(struct device_driver *drv, void *data) return 0; if (ap_drv->in_use) { - rc = ap_drv->in_use(newapm, ap_perms.aqm); + bitmap_fill(aqm_any, AP_DOMAINS); + rc = ap_drv->in_use(newapm, aqm_any); if (rc) rc = -EBUSY; } @@ -1490,18 +1546,31 @@ static int apmask_commit(unsigned long *newapm) memcpy(ap_perms.apm, newapm, APMASKSIZE); + /* + * Update ap_apmask_aqmask_in_use. Note that the + * ap_attr_mutex has to be obtained here. + */ + ap_apmask_aqmask_in_use = + bitmap_full(ap_perms.apm, AP_DEVICES) && + bitmap_full(ap_perms.aqm, AP_DOMAINS) ? + false : true; + return 0; } static ssize_t apmask_store(const struct bus_type *bus, const char *buf, size_t count) { - int rc, changes = 0; DECLARE_BITMAP(newapm, AP_DEVICES); + int rc = -EINVAL, changes = 0; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; + /* Do not allow apmask/aqmask if driver override is active */ + if (ap_driver_override_ctr) + goto done; + rc = ap_parse_bitmap_str(buf, ap_perms.apm, AP_DEVICES, newapm); if (rc) goto done; @@ -1511,7 +1580,7 @@ static ssize_t apmask_store(const struct bus_type *bus, const char *buf, rc = apmask_commit(newapm); done: - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); if (rc) return rc; @@ -1529,12 +1598,12 @@ static ssize_t aqmask_show(const struct bus_type *bus, char *buf) { int rc; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; rc = sysfs_emit(buf, "0x%016lx%016lx%016lx%016lx\n", ap_perms.aqm[0], ap_perms.aqm[1], ap_perms.aqm[2], ap_perms.aqm[3]); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return rc; } @@ -1544,6 +1613,7 @@ static int __verify_queue_reservations(struct device_driver *drv, void *data) int rc = 0; struct ap_driver *ap_drv = to_ap_drv(drv); unsigned long *newaqm = (unsigned long *)data; + unsigned long apm_any[BITS_TO_LONGS(AP_DEVICES)]; /* * increase the driver's module refcounter to be sure it is not @@ -1553,7 +1623,8 @@ static int __verify_queue_reservations(struct device_driver *drv, void *data) return 0; if (ap_drv->in_use) { - rc = ap_drv->in_use(ap_perms.apm, newaqm); + bitmap_fill(apm_any, AP_DEVICES); + rc = ap_drv->in_use(apm_any, newaqm); if (rc) rc = -EBUSY; } @@ -1582,18 +1653,31 @@ static int aqmask_commit(unsigned long *newaqm) memcpy(ap_perms.aqm, newaqm, AQMASKSIZE); + /* + * Update ap_apmask_aqmask_in_use. Note that the + * ap_attr_mutex has to be obtained here. + */ + ap_apmask_aqmask_in_use = + bitmap_full(ap_perms.apm, AP_DEVICES) && + bitmap_full(ap_perms.aqm, AP_DOMAINS) ? + false : true; + return 0; } static ssize_t aqmask_store(const struct bus_type *bus, const char *buf, size_t count) { - int rc, changes = 0; DECLARE_BITMAP(newaqm, AP_DOMAINS); + int rc = -EINVAL, changes = 0; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; + /* Do not allow apmask/aqmask if driver override is active */ + if (ap_driver_override_ctr) + goto done; + rc = ap_parse_bitmap_str(buf, ap_perms.aqm, AP_DOMAINS, newaqm); if (rc) goto done; @@ -1603,7 +1687,7 @@ static ssize_t aqmask_store(const struct bus_type *bus, const char *buf, rc = aqmask_commit(newaqm); done: - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); if (rc) return rc; @@ -1650,6 +1734,15 @@ static ssize_t bindings_show(const struct bus_type *bus, char *buf) static BUS_ATTR_RO(bindings); +static ssize_t bindings_complete_count_show(const struct bus_type *bus, + char *buf) +{ + return sysfs_emit(buf, "%llu\n", + atomic64_read(&ap_bindings_complete_count)); +} + +static BUS_ATTR_RO(bindings_complete_count); + static ssize_t features_show(const struct bus_type *bus, char *buf) { int n = 0; @@ -1690,6 +1783,7 @@ static struct attribute *ap_bus_attrs[] = { &bus_attr_aqmask.attr, &bus_attr_scans.attr, &bus_attr_bindings.attr, + &bus_attr_bindings_complete_count.attr, &bus_attr_features.attr, NULL, }; @@ -2464,14 +2558,14 @@ static void __init ap_perms_init(void) if (apm_str) { memset(&ap_perms.apm, 0, sizeof(ap_perms.apm)); ap_parse_mask_str(apm_str, ap_perms.apm, AP_DEVICES, - &ap_perms_mutex); + &ap_attr_mutex); } /* aqm kernel parameter string */ if (aqm_str) { memset(&ap_perms.aqm, 0, sizeof(ap_perms.aqm)); ap_parse_mask_str(aqm_str, ap_perms.aqm, AP_DOMAINS, - &ap_perms_mutex); + &ap_attr_mutex); } } @@ -2484,15 +2578,15 @@ static int __init ap_module_init(void) { int rc; - rc = ap_debug_init(); - if (rc) - return rc; - if (!ap_instructions_available()) { pr_warn("The hardware system does not support AP instructions\n"); return -ENODEV; } + rc = ap_debug_init(); + if (rc) + return rc; + /* init ap_queue hashtable */ hash_init(ap_queues); diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h index 4b7ffa840563..51e08f27bd75 100644 --- a/drivers/s390/crypto/ap_bus.h +++ b/drivers/s390/crypto/ap_bus.h @@ -166,6 +166,7 @@ void ap_driver_unregister(struct ap_driver *); struct ap_device { struct device device; int device_type; /* AP device type. */ + const char *driver_override; }; #define to_ap_dev(x) container_of((x), struct ap_device, device) @@ -280,7 +281,9 @@ struct ap_perms { }; extern struct ap_perms ap_perms; -extern struct mutex ap_perms_mutex; +extern bool ap_apmask_aqmask_in_use; +extern int ap_driver_override_ctr; +extern struct mutex ap_attr_mutex; /* * Get ap_queue device for this qid. diff --git a/drivers/s390/crypto/ap_card.c b/drivers/s390/crypto/ap_card.c index ce953cbbd564..8102c8134c49 100644 --- a/drivers/s390/crypto/ap_card.c +++ b/drivers/s390/crypto/ap_card.c @@ -6,8 +6,7 @@ * Adjunct processor bus, card related code. */ -#define KMSG_COMPONENT "ap" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ap: " fmt #include <linux/init.h> #include <linux/slab.h> diff --git a/drivers/s390/crypto/ap_queue.c b/drivers/s390/crypto/ap_queue.c index 8977866fab1b..4a32c1e19a1e 100644 --- a/drivers/s390/crypto/ap_queue.c +++ b/drivers/s390/crypto/ap_queue.c @@ -6,17 +6,22 @@ * Adjunct processor bus, queue related code. */ -#define KMSG_COMPONENT "ap" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ap: " fmt #include <linux/export.h> #include <linux/init.h> #include <linux/slab.h> #include <asm/facility.h> +#define CREATE_TRACE_POINTS +#include <asm/trace/ap.h> + #include "ap_bus.h" #include "ap_debug.h" +EXPORT_TRACEPOINT_SYMBOL(s390_ap_nqap); +EXPORT_TRACEPOINT_SYMBOL(s390_ap_dqap); + static void __ap_flush_queue(struct ap_queue *aq); /* @@ -98,9 +103,17 @@ static inline struct ap_queue_status __ap_send(ap_qid_t qid, unsigned long psmid, void *msg, size_t msglen, int special) { + struct ap_queue_status status; + if (special) qid |= 0x400000UL; - return ap_nqap(qid, psmid, msg, msglen); + + status = ap_nqap(qid, psmid, msg, msglen); + + trace_s390_ap_nqap(AP_QID_CARD(qid), AP_QID_QUEUE(qid), + status.value, psmid); + + return status; } /* State machine definitions and helpers */ @@ -140,6 +153,9 @@ static struct ap_queue_status ap_sm_recv(struct ap_queue *aq) parts++; } while (status.response_code == 0xFF && resgr0 != 0); + trace_s390_ap_dqap(AP_QID_CARD(aq->qid), AP_QID_QUEUE(aq->qid), + status.value, aq->reply->psmid); + switch (status.response_code) { case AP_RESPONSE_NORMAL: print_hex_dump_debug("aprpl: ", DUMP_PREFIX_ADDRESS, 16, 1, @@ -714,6 +730,58 @@ static ssize_t ap_functions_show(struct device *dev, static DEVICE_ATTR_RO(ap_functions); +static ssize_t driver_override_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct ap_queue *aq = to_ap_queue(dev); + struct ap_device *ap_dev = &aq->ap_dev; + int rc; + + device_lock(dev); + if (ap_dev->driver_override) + rc = sysfs_emit(buf, "%s\n", ap_dev->driver_override); + else + rc = sysfs_emit(buf, "\n"); + device_unlock(dev); + + return rc; +} + +static ssize_t driver_override_store(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count) +{ + struct ap_queue *aq = to_ap_queue(dev); + struct ap_device *ap_dev = &aq->ap_dev; + int rc = -EINVAL; + bool old_value; + + if (mutex_lock_interruptible(&ap_attr_mutex)) + return -ERESTARTSYS; + + /* Do not allow driver override if apmask/aqmask is in use */ + if (ap_apmask_aqmask_in_use) + goto out; + + old_value = ap_dev->driver_override ? true : false; + rc = driver_set_override(dev, &ap_dev->driver_override, buf, count); + if (rc) + goto out; + if (old_value && !ap_dev->driver_override) + --ap_driver_override_ctr; + else if (!old_value && ap_dev->driver_override) + ++ap_driver_override_ctr; + + rc = count; + +out: + mutex_unlock(&ap_attr_mutex); + return rc; +} + +static DEVICE_ATTR_RW(driver_override); + #ifdef CONFIG_AP_DEBUG static ssize_t states_show(struct device *dev, struct device_attribute *attr, char *buf) @@ -826,6 +894,7 @@ static struct attribute *ap_queue_dev_attrs[] = { &dev_attr_config.attr, &dev_attr_chkstop.attr, &dev_attr_ap_functions.attr, + &dev_attr_driver_override.attr, #ifdef CONFIG_AP_DEBUG &dev_attr_states.attr, &dev_attr_last_err_rc.attr, diff --git a/drivers/s390/crypto/pkey_api.c b/drivers/s390/crypto/pkey_api.c index 01549003a903..ad1cd699f53b 100644 --- a/drivers/s390/crypto/pkey_api.c +++ b/drivers/s390/crypto/pkey_api.c @@ -7,8 +7,7 @@ * Author(s): Harald Freudenberger */ -#define KMSG_COMPONENT "pkey" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "pkey: " fmt #include <linux/init.h> #include <linux/miscdevice.h> diff --git a/drivers/s390/crypto/pkey_base.c b/drivers/s390/crypto/pkey_base.c index b15741461a63..d60cd987c16d 100644 --- a/drivers/s390/crypto/pkey_base.c +++ b/drivers/s390/crypto/pkey_base.c @@ -5,8 +5,7 @@ * Copyright IBM Corp. 2024 */ -#define KMSG_COMPONENT "pkey" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "pkey: " fmt #include <linux/cpufeature.h> #include <linux/export.h> diff --git a/drivers/s390/crypto/pkey_cca.c b/drivers/s390/crypto/pkey_cca.c index 6c7897a93f27..d4550d8d8eea 100644 --- a/drivers/s390/crypto/pkey_cca.c +++ b/drivers/s390/crypto/pkey_cca.c @@ -5,8 +5,7 @@ * Copyright IBM Corp. 2024 */ -#define KMSG_COMPONENT "pkey" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "pkey: " fmt #include <linux/init.h> #include <linux/module.h> diff --git a/drivers/s390/crypto/pkey_ep11.c b/drivers/s390/crypto/pkey_ep11.c index 6b23adc560c8..654eed20d0d9 100644 --- a/drivers/s390/crypto/pkey_ep11.c +++ b/drivers/s390/crypto/pkey_ep11.c @@ -5,8 +5,7 @@ * Copyright IBM Corp. 2024 */ -#define KMSG_COMPONENT "pkey" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "pkey: " fmt #include <linux/init.h> #include <linux/module.h> diff --git a/drivers/s390/crypto/pkey_pckmo.c b/drivers/s390/crypto/pkey_pckmo.c index 7eca9f1340bd..793326c4c59a 100644 --- a/drivers/s390/crypto/pkey_pckmo.c +++ b/drivers/s390/crypto/pkey_pckmo.c @@ -5,8 +5,7 @@ * Copyright IBM Corp. 2024 */ -#define KMSG_COMPONENT "pkey" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "pkey: " fmt #include <linux/init.h> #include <linux/module.h> diff --git a/drivers/s390/crypto/pkey_sysfs.c b/drivers/s390/crypto/pkey_sysfs.c index 792c0fce88fa..b6b0a46cb8a8 100644 --- a/drivers/s390/crypto/pkey_sysfs.c +++ b/drivers/s390/crypto/pkey_sysfs.c @@ -5,8 +5,7 @@ * Copyright IBM Corp. 2024 */ -#define KMSG_COMPONENT "pkey" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "pkey: " fmt #include <linux/sysfs.h> diff --git a/drivers/s390/crypto/pkey_uv.c b/drivers/s390/crypto/pkey_uv.c index e5c6e01acaf3..6cd3c49384b5 100644 --- a/drivers/s390/crypto/pkey_uv.c +++ b/drivers/s390/crypto/pkey_uv.c @@ -5,8 +5,7 @@ * Copyright IBM Corp. 2024 */ -#define KMSG_COMPONENT "pkey" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "pkey: " fmt #include <linux/cpufeature.h> #include <linux/init.h> diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c index eb5ff49f6fe7..48da32ad0493 100644 --- a/drivers/s390/crypto/vfio_ap_ops.c +++ b/drivers/s390/crypto/vfio_ap_ops.c @@ -968,7 +968,7 @@ static int vfio_ap_mdev_verify_no_sharing(struct ap_matrix_mdev *assignee, * * Return: One of the following values: * o the error returned from the ap_apqn_in_matrix_owned_by_def_drv() function, - * most likely -EBUSY indicating the ap_perms_mutex lock is already held. + * most likely -EBUSY indicating the ap_attr_mutex lock is already held. * o EADDRNOTAVAIL if an APQN assigned to @matrix_mdev is reserved for the * zcrypt default driver. * o EADDRINUSE if an APQN assigned to @matrix_mdev is assigned to another mdev @@ -1079,7 +1079,7 @@ static ssize_t assign_adapter_store(struct device *dev, DECLARE_BITMAP(apm_filtered, AP_DEVICES); struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&ap_perms_mutex); + mutex_lock(&ap_attr_mutex); get_update_locks_for_mdev(matrix_mdev); ret = kstrtoul(buf, 0, &apid); @@ -1114,7 +1114,7 @@ static ssize_t assign_adapter_store(struct device *dev, ret = count; done: release_update_locks_for_mdev(matrix_mdev); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return ret; } @@ -1303,7 +1303,7 @@ static ssize_t assign_domain_store(struct device *dev, DECLARE_BITMAP(apm_filtered, AP_DEVICES); struct ap_matrix_mdev *matrix_mdev = dev_get_drvdata(dev); - mutex_lock(&ap_perms_mutex); + mutex_lock(&ap_attr_mutex); get_update_locks_for_mdev(matrix_mdev); ret = kstrtoul(buf, 0, &apqi); @@ -1338,7 +1338,7 @@ static ssize_t assign_domain_store(struct device *dev, ret = count; done: release_update_locks_for_mdev(matrix_mdev); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return ret; } @@ -1718,7 +1718,7 @@ static ssize_t ap_config_store(struct device *dev, struct device_attribute *attr return -ENOMEM; rest = newbuf; - mutex_lock(&ap_perms_mutex); + mutex_lock(&ap_attr_mutex); get_update_locks_for_mdev(matrix_mdev); /* Save old state */ @@ -1779,7 +1779,7 @@ static ssize_t ap_config_store(struct device *dev, struct device_attribute *attr } out: release_update_locks_for_mdev(matrix_mdev); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); kfree(newbuf); return rc; } diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c index 4e6bf1cb3475..7a3b99f065f2 100644 --- a/drivers/s390/crypto/zcrypt_api.c +++ b/drivers/s390/crypto/zcrypt_api.c @@ -12,8 +12,7 @@ * Multiple device nodes: Harald Freudenberger <freude@linux.ibm.com> */ -#define KMSG_COMPONENT "zcrypt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zcrypt: " fmt #include <linux/export.h> #include <linux/module.h> @@ -21,7 +20,6 @@ #include <linux/interrupt.h> #include <linux/miscdevice.h> #include <linux/fs.h> -#include <linux/compat.h> #include <linux/slab.h> #include <linux/atomic.h> #include <linux/uaccess.h> @@ -163,7 +161,7 @@ static ssize_t ioctlmask_show(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); int i, n; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; n = sysfs_emit(buf, "0x"); @@ -171,7 +169,7 @@ static ssize_t ioctlmask_show(struct device *dev, n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.ioctlm[i]); n += sysfs_emit_at(buf, n, "\n"); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return n; } @@ -184,7 +182,7 @@ static ssize_t ioctlmask_store(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); rc = ap_parse_mask_str(buf, zcdndev->perms.ioctlm, - AP_IOCTLS, &ap_perms_mutex); + AP_IOCTLS, &ap_attr_mutex); if (rc) return rc; @@ -200,7 +198,7 @@ static ssize_t apmask_show(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); int i, n; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; n = sysfs_emit(buf, "0x"); @@ -208,7 +206,7 @@ static ssize_t apmask_show(struct device *dev, n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.apm[i]); n += sysfs_emit_at(buf, n, "\n"); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return n; } @@ -221,7 +219,7 @@ static ssize_t apmask_store(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); rc = ap_parse_mask_str(buf, zcdndev->perms.apm, - AP_DEVICES, &ap_perms_mutex); + AP_DEVICES, &ap_attr_mutex); if (rc) return rc; @@ -237,7 +235,7 @@ static ssize_t aqmask_show(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); int i, n; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; n = sysfs_emit(buf, "0x"); @@ -245,7 +243,7 @@ static ssize_t aqmask_show(struct device *dev, n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.aqm[i]); n += sysfs_emit_at(buf, n, "\n"); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return n; } @@ -258,7 +256,7 @@ static ssize_t aqmask_store(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); rc = ap_parse_mask_str(buf, zcdndev->perms.aqm, - AP_DOMAINS, &ap_perms_mutex); + AP_DOMAINS, &ap_attr_mutex); if (rc) return rc; @@ -274,7 +272,7 @@ static ssize_t admask_show(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); int i, n; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; n = sysfs_emit(buf, "0x"); @@ -282,7 +280,7 @@ static ssize_t admask_show(struct device *dev, n += sysfs_emit_at(buf, n, "%016lx", zcdndev->perms.adm[i]); n += sysfs_emit_at(buf, n, "\n"); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return n; } @@ -295,7 +293,7 @@ static ssize_t admask_store(struct device *dev, struct zcdn_device *zcdndev = to_zcdn_dev(dev); rc = ap_parse_mask_str(buf, zcdndev->perms.adm, - AP_DOMAINS, &ap_perms_mutex); + AP_DOMAINS, &ap_attr_mutex); if (rc) return rc; @@ -371,7 +369,7 @@ static int zcdn_create(const char *name) int i, rc = 0; struct zcdn_device *zcdndev; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; /* check if device node with this name already exists */ @@ -426,7 +424,7 @@ static int zcdn_create(const char *name) __func__, MAJOR(devt), MINOR(devt)); unlockout: - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return rc; } @@ -435,7 +433,7 @@ static int zcdn_destroy(const char *name) int rc = 0; struct zcdn_device *zcdndev; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; /* try to find this zcdn device */ @@ -453,7 +451,7 @@ static int zcdn_destroy(const char *name) device_unregister(&zcdndev->device); unlockout: - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); return rc; } @@ -463,7 +461,7 @@ static void zcdn_destroy_all(void) dev_t devt; struct zcdn_device *zcdndev; - mutex_lock(&ap_perms_mutex); + mutex_lock(&ap_attr_mutex); for (i = 0; i < ZCRYPT_MAX_MINOR_NODES; i++) { devt = MKDEV(MAJOR(zcrypt_devt), MINOR(zcrypt_devt) + i); zcdndev = find_zcdndev_by_devt(devt); @@ -472,7 +470,7 @@ static void zcdn_destroy_all(void) device_unregister(&zcdndev->device); } } - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); } /* @@ -509,11 +507,11 @@ static int zcrypt_open(struct inode *inode, struct file *filp) if (filp->f_inode->i_cdev == &zcrypt_cdev) { struct zcdn_device *zcdndev; - if (mutex_lock_interruptible(&ap_perms_mutex)) + if (mutex_lock_interruptible(&ap_attr_mutex)) return -ERESTARTSYS; zcdndev = find_zcdndev_by_devt(filp->f_inode->i_rdev); /* find returns a reference, no get_device() needed */ - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); if (zcdndev) perms = &zcdndev->perms; } @@ -533,9 +531,9 @@ static int zcrypt_release(struct inode *inode, struct file *filp) if (filp->f_inode->i_cdev == &zcrypt_cdev) { struct zcdn_device *zcdndev; - mutex_lock(&ap_perms_mutex); + mutex_lock(&ap_attr_mutex); zcdndev = find_zcdndev_by_devt(filp->f_inode->i_rdev); - mutex_unlock(&ap_perms_mutex); + mutex_unlock(&ap_attr_mutex); if (zcdndev) { /* 2 puts here: one for find, one for open */ put_device(&zcdndev->device); @@ -740,7 +738,8 @@ out: tr->last_qid = qid; } trace_s390_zcrypt_rep(mex, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } @@ -845,7 +844,8 @@ out: tr->last_qid = qid; } trace_s390_zcrypt_rep(crt, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } @@ -980,7 +980,8 @@ out: tr->last_qid = qid; } trace_s390_zcrypt_rep(xcrb, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } @@ -1182,7 +1183,8 @@ out: tr->last_qid = qid; } trace_s390_zcrypt_rep(xcrb, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } @@ -1274,7 +1276,8 @@ static long zcrypt_rng(char *buffer) out: ap_release_apmsg(&ap_msg); trace_s390_zcrypt_rep(buffer, func_code, rc, - AP_QID_CARD(qid), AP_QID_QUEUE(qid)); + AP_QID_CARD(qid), AP_QID_QUEUE(qid), + ap_msg.psmid); return rc; } @@ -1729,197 +1732,6 @@ static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd, } } -#ifdef CONFIG_COMPAT -/* - * ioctl32 conversion routines - */ -struct compat_ica_rsa_modexpo { - compat_uptr_t inputdata; - unsigned int inputdatalength; - compat_uptr_t outputdata; - unsigned int outputdatalength; - compat_uptr_t b_key; - compat_uptr_t n_modulus; -}; - -static long trans_modexpo32(struct ap_perms *perms, struct file *filp, - unsigned int cmd, unsigned long arg) -{ - struct compat_ica_rsa_modexpo __user *umex32 = compat_ptr(arg); - struct compat_ica_rsa_modexpo mex32; - struct ica_rsa_modexpo mex64; - struct zcrypt_track tr; - long rc; - - memset(&tr, 0, sizeof(tr)); - if (copy_from_user(&mex32, umex32, sizeof(mex32))) - return -EFAULT; - mex64.inputdata = compat_ptr(mex32.inputdata); - mex64.inputdatalength = mex32.inputdatalength; - mex64.outputdata = compat_ptr(mex32.outputdata); - mex64.outputdatalength = mex32.outputdatalength; - mex64.b_key = compat_ptr(mex32.b_key); - mex64.n_modulus = compat_ptr(mex32.n_modulus); - do { - rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); - } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); - - /* on ENODEV failure: retry once again after a requested rescan */ - if (rc == -ENODEV && zcrypt_process_rescan()) - do { - rc = zcrypt_rsa_modexpo(perms, &tr, &mex64); - } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); - if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) - rc = -EIO; - if (rc) - return rc; - return put_user(mex64.outputdatalength, - &umex32->outputdatalength); -} - -struct compat_ica_rsa_modexpo_crt { - compat_uptr_t inputdata; - unsigned int inputdatalength; - compat_uptr_t outputdata; - unsigned int outputdatalength; - compat_uptr_t bp_key; - compat_uptr_t bq_key; - compat_uptr_t np_prime; - compat_uptr_t nq_prime; - compat_uptr_t u_mult_inv; -}; - -static long trans_modexpo_crt32(struct ap_perms *perms, struct file *filp, - unsigned int cmd, unsigned long arg) -{ - struct compat_ica_rsa_modexpo_crt __user *ucrt32 = compat_ptr(arg); - struct compat_ica_rsa_modexpo_crt crt32; - struct ica_rsa_modexpo_crt crt64; - struct zcrypt_track tr; - long rc; - - memset(&tr, 0, sizeof(tr)); - if (copy_from_user(&crt32, ucrt32, sizeof(crt32))) - return -EFAULT; - crt64.inputdata = compat_ptr(crt32.inputdata); - crt64.inputdatalength = crt32.inputdatalength; - crt64.outputdata = compat_ptr(crt32.outputdata); - crt64.outputdatalength = crt32.outputdatalength; - crt64.bp_key = compat_ptr(crt32.bp_key); - crt64.bq_key = compat_ptr(crt32.bq_key); - crt64.np_prime = compat_ptr(crt32.np_prime); - crt64.nq_prime = compat_ptr(crt32.nq_prime); - crt64.u_mult_inv = compat_ptr(crt32.u_mult_inv); - do { - rc = zcrypt_rsa_crt(perms, &tr, &crt64); - } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); - - /* on ENODEV failure: retry once again after a requested rescan */ - if (rc == -ENODEV && zcrypt_process_rescan()) - do { - rc = zcrypt_rsa_crt(perms, &tr, &crt64); - } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); - if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) - rc = -EIO; - if (rc) - return rc; - return put_user(crt64.outputdatalength, - &ucrt32->outputdatalength); -} - -struct compat_ica_xcrb { - unsigned short agent_ID; - unsigned int user_defined; - unsigned short request_ID; - unsigned int request_control_blk_length; - unsigned char padding1[16 - sizeof(compat_uptr_t)]; - compat_uptr_t request_control_blk_addr; - unsigned int request_data_length; - char padding2[16 - sizeof(compat_uptr_t)]; - compat_uptr_t request_data_address; - unsigned int reply_control_blk_length; - char padding3[16 - sizeof(compat_uptr_t)]; - compat_uptr_t reply_control_blk_addr; - unsigned int reply_data_length; - char padding4[16 - sizeof(compat_uptr_t)]; - compat_uptr_t reply_data_addr; - unsigned short priority_window; - unsigned int status; -} __packed; - -static long trans_xcrb32(struct ap_perms *perms, struct file *filp, - unsigned int cmd, unsigned long arg) -{ - struct compat_ica_xcrb __user *uxcrb32 = compat_ptr(arg); - u32 xflags = ZCRYPT_XFLAG_USERSPACE; - struct compat_ica_xcrb xcrb32; - struct zcrypt_track tr; - struct ica_xcRB xcrb64; - long rc; - - memset(&tr, 0, sizeof(tr)); - if (copy_from_user(&xcrb32, uxcrb32, sizeof(xcrb32))) - return -EFAULT; - xcrb64.agent_ID = xcrb32.agent_ID; - xcrb64.user_defined = xcrb32.user_defined; - xcrb64.request_ID = xcrb32.request_ID; - xcrb64.request_control_blk_length = - xcrb32.request_control_blk_length; - xcrb64.request_control_blk_addr = - compat_ptr(xcrb32.request_control_blk_addr); - xcrb64.request_data_length = - xcrb32.request_data_length; - xcrb64.request_data_address = - compat_ptr(xcrb32.request_data_address); - xcrb64.reply_control_blk_length = - xcrb32.reply_control_blk_length; - xcrb64.reply_control_blk_addr = - compat_ptr(xcrb32.reply_control_blk_addr); - xcrb64.reply_data_length = xcrb32.reply_data_length; - xcrb64.reply_data_addr = - compat_ptr(xcrb32.reply_data_addr); - xcrb64.priority_window = xcrb32.priority_window; - xcrb64.status = xcrb32.status; - do { - rc = _zcrypt_send_cprb(xflags, perms, &tr, &xcrb64); - } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); - - /* on ENODEV failure: retry once again after a requested rescan */ - if (rc == -ENODEV && zcrypt_process_rescan()) - do { - rc = _zcrypt_send_cprb(xflags, perms, &tr, &xcrb64); - } while (rc == -EAGAIN && ++tr.again_counter < TRACK_AGAIN_MAX); - if (rc == -EAGAIN && tr.again_counter >= TRACK_AGAIN_MAX) - rc = -EIO; - xcrb32.reply_control_blk_length = xcrb64.reply_control_blk_length; - xcrb32.reply_data_length = xcrb64.reply_data_length; - xcrb32.status = xcrb64.status; - if (copy_to_user(uxcrb32, &xcrb32, sizeof(xcrb32))) - return -EFAULT; - return rc; -} - -static long zcrypt_compat_ioctl(struct file *filp, unsigned int cmd, - unsigned long arg) -{ - int rc; - struct ap_perms *perms = - (struct ap_perms *)filp->private_data; - - rc = zcrypt_check_ioctl(perms, cmd); - if (rc) - return rc; - - if (cmd == ICARSAMODEXPO) - return trans_modexpo32(perms, filp, cmd, arg); - if (cmd == ICARSACRT) - return trans_modexpo_crt32(perms, filp, cmd, arg); - if (cmd == ZSECSENDCPRB) - return trans_xcrb32(perms, filp, cmd, arg); - return zcrypt_unlocked_ioctl(filp, cmd, arg); -} -#endif - /* * Misc device file operations. */ @@ -1928,9 +1740,6 @@ static const struct file_operations zcrypt_fops = { .read = zcrypt_read, .write = zcrypt_write, .unlocked_ioctl = zcrypt_unlocked_ioctl, -#ifdef CONFIG_COMPAT - .compat_ioctl = zcrypt_compat_ioctl, -#endif .open = zcrypt_open, .release = zcrypt_release, }; diff --git a/drivers/s390/crypto/zcrypt_card.c b/drivers/s390/crypto/zcrypt_card.c index aa2c8ff2740e..6dea702a5cac 100644 --- a/drivers/s390/crypto/zcrypt_card.c +++ b/drivers/s390/crypto/zcrypt_card.c @@ -19,7 +19,6 @@ #include <linux/fs.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> -#include <linux/compat.h> #include <linux/slab.h> #include <linux/atomic.h> #include <linux/uaccess.h> diff --git a/drivers/s390/crypto/zcrypt_ccamisc.c b/drivers/s390/crypto/zcrypt_ccamisc.c index a96e25614303..573bad1d6d86 100644 --- a/drivers/s390/crypto/zcrypt_ccamisc.c +++ b/drivers/s390/crypto/zcrypt_ccamisc.c @@ -7,8 +7,7 @@ * Collection of CCA misc functions used by zcrypt and pkey */ -#define KMSG_COMPONENT "zcrypt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zcrypt: " fmt #include <linux/export.h> #include <linux/init.h> diff --git a/drivers/s390/crypto/zcrypt_ep11misc.c b/drivers/s390/crypto/zcrypt_ep11misc.c index e92e2fd8ce5d..3dda9589f2b9 100644 --- a/drivers/s390/crypto/zcrypt_ep11misc.c +++ b/drivers/s390/crypto/zcrypt_ep11misc.c @@ -6,8 +6,7 @@ * Collection of EP11 misc functions used by zcrypt and pkey */ -#define KMSG_COMPONENT "zcrypt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zcrypt: " fmt #include <linux/export.h> #include <linux/init.h> diff --git a/drivers/s390/crypto/zcrypt_msgtype50.c b/drivers/s390/crypto/zcrypt_msgtype50.c index fc0a2a053dc2..d6fc2d8e7fad 100644 --- a/drivers/s390/crypto/zcrypt_msgtype50.c +++ b/drivers/s390/crypto/zcrypt_msgtype50.c @@ -10,8 +10,7 @@ * MSGTYPE restruct: Holger Dengler <hd@linux.vnet.ibm.com> */ -#define KMSG_COMPONENT "zcrypt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zcrypt: " fmt #include <linux/module.h> #include <linux/slab.h> diff --git a/drivers/s390/crypto/zcrypt_msgtype6.c b/drivers/s390/crypto/zcrypt_msgtype6.c index 9cefbb30960f..a0dcab5dc4f2 100644 --- a/drivers/s390/crypto/zcrypt_msgtype6.c +++ b/drivers/s390/crypto/zcrypt_msgtype6.c @@ -10,8 +10,7 @@ * MSGTYPE restruct: Holger Dengler <hd@linux.vnet.ibm.com> */ -#define KMSG_COMPONENT "zcrypt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zcrypt: " fmt #include <linux/module.h> #include <linux/init.h> diff --git a/drivers/s390/crypto/zcrypt_queue.c b/drivers/s390/crypto/zcrypt_queue.c index 76a8678bdad6..a173d32eb6e8 100644 --- a/drivers/s390/crypto/zcrypt_queue.c +++ b/drivers/s390/crypto/zcrypt_queue.c @@ -19,7 +19,6 @@ #include <linux/fs.h> #include <linux/proc_fs.h> #include <linux/seq_file.h> -#include <linux/compat.h> #include <linux/slab.h> #include <linux/atomic.h> #include <linux/uaccess.h> diff --git a/drivers/s390/net/ctcm_fsms.c b/drivers/s390/net/ctcm_fsms.c index 1a48258b63b2..bf917f426453 100644 --- a/drivers/s390/net/ctcm_fsms.c +++ b/drivers/s390/net/ctcm_fsms.c @@ -12,8 +12,7 @@ #undef DEBUGDATA #undef DEBUGCCW -#define KMSG_COMPONENT "ctcm" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ctcm: " fmt #include <linux/module.h> #include <linux/init.h> diff --git a/drivers/s390/net/ctcm_main.c b/drivers/s390/net/ctcm_main.c index b93c2eb45916..3d7ccf2366a0 100644 --- a/drivers/s390/net/ctcm_main.c +++ b/drivers/s390/net/ctcm_main.c @@ -20,8 +20,7 @@ #undef DEBUGDATA #undef DEBUGCCW -#define KMSG_COMPONENT "ctcm" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ctcm: " fmt #include <linux/module.h> #include <linux/init.h> diff --git a/drivers/s390/net/ctcm_mpc.c b/drivers/s390/net/ctcm_mpc.c index 407b7c516658..0f329fb514ee 100644 --- a/drivers/s390/net/ctcm_mpc.c +++ b/drivers/s390/net/ctcm_mpc.c @@ -18,8 +18,7 @@ #undef DEBUGDATA #undef DEBUGCCW -#define KMSG_COMPONENT "ctcm" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ctcm: " fmt #include <linux/export.h> #include <linux/module.h> diff --git a/drivers/s390/net/ctcm_sysfs.c b/drivers/s390/net/ctcm_sysfs.c index 0c5d8a3eaa2e..529a1c40ae63 100644 --- a/drivers/s390/net/ctcm_sysfs.c +++ b/drivers/s390/net/ctcm_sysfs.c @@ -9,8 +9,7 @@ #undef DEBUGDATA #undef DEBUGCCW -#define KMSG_COMPONENT "ctcm" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ctcm: " fmt #include <linux/device.h> #include <linux/sysfs.h> diff --git a/drivers/s390/net/ism_drv.c b/drivers/s390/net/ism_drv.c index f84aa2e676e9..8b8e4f06be0f 100644 --- a/drivers/s390/net/ism_drv.c +++ b/drivers/s390/net/ism_drv.c @@ -4,8 +4,7 @@ * * Copyright IBM Corp. 2018 */ -#define KMSG_COMPONENT "ism" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "ism: " fmt #include <linux/export.h> #include <linux/module.h> diff --git a/drivers/s390/net/qeth_core_main.c b/drivers/s390/net/qeth_core_main.c index 10b53bba373c..1c80e8ca67b5 100644 --- a/drivers/s390/net/qeth_core_main.c +++ b/drivers/s390/net/qeth_core_main.c @@ -7,10 +7,8 @@ * Frank Blaschka <frank.blaschka@de.ibm.com> */ -#define KMSG_COMPONENT "qeth" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "qeth: " fmt -#include <linux/compat.h> #include <linux/export.h> #include <linux/module.h> #include <linux/moduleparam.h> @@ -4805,8 +4803,7 @@ static int qeth_query_oat_command(struct qeth_card *card, char __user *udata) rc = qeth_send_ipa_cmd(card, iob, qeth_setadpparms_query_oat_cb, &priv); if (!rc) { - tmp = is_compat_task() ? compat_ptr(oat_data.ptr) : - u64_to_user_ptr(oat_data.ptr); + tmp = u64_to_user_ptr(oat_data.ptr); oat_data.response_len = priv.response_len; if (copy_to_user(tmp, priv.buffer, priv.response_len) || diff --git a/drivers/s390/net/qeth_core_sys.c b/drivers/s390/net/qeth_core_sys.c index c0e4883be6d0..a3b16d4d16fb 100644 --- a/drivers/s390/net/qeth_core_sys.c +++ b/drivers/s390/net/qeth_core_sys.c @@ -7,8 +7,7 @@ * Frank Blaschka <frank.blaschka@de.ibm.com> */ -#define KMSG_COMPONENT "qeth" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "qeth: " fmt #include <linux/list.h> #include <linux/rwsem.h> diff --git a/drivers/s390/net/qeth_ethtool.c b/drivers/s390/net/qeth_ethtool.c index f184c58ecf24..d214a889cf4e 100644 --- a/drivers/s390/net/qeth_ethtool.c +++ b/drivers/s390/net/qeth_ethtool.c @@ -3,8 +3,7 @@ * Copyright IBM Corp. 2018 */ -#define KMSG_COMPONENT "qeth" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "qeth: " fmt #include <linux/ethtool.h> #include "qeth_core.h" diff --git a/drivers/s390/net/qeth_l2_main.c b/drivers/s390/net/qeth_l2_main.c index 2a3888283a94..7498a83b1f06 100644 --- a/drivers/s390/net/qeth_l2_main.c +++ b/drivers/s390/net/qeth_l2_main.c @@ -7,8 +7,7 @@ * Frank Blaschka <frank.blaschka@de.ibm.com> */ -#define KMSG_COMPONENT "qeth" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "qeth: " fmt #include <linux/export.h> #include <linux/module.h> diff --git a/drivers/s390/net/qeth_l3_main.c b/drivers/s390/net/qeth_l3_main.c index 3525be819362..027bc346232f 100644 --- a/drivers/s390/net/qeth_l3_main.c +++ b/drivers/s390/net/qeth_l3_main.c @@ -7,8 +7,7 @@ * Frank Blaschka <frank.blaschka@de.ibm.com> */ -#define KMSG_COMPONENT "qeth" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "qeth: " fmt #include <linux/export.h> #include <linux/module.h> diff --git a/drivers/s390/net/smsgiucv_app.c b/drivers/s390/net/smsgiucv_app.c index 768108c90b32..1bd0370460cd 100644 --- a/drivers/s390/net/smsgiucv_app.c +++ b/drivers/s390/net/smsgiucv_app.c @@ -10,8 +10,7 @@ * Author(s): Hendrik Brueckner <brueckner@linux.vnet.ibm.com> * */ -#define KMSG_COMPONENT "smsgiucv_app" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "smsgiucv_app: " fmt #include <linux/ctype.h> #include <linux/err.h> @@ -162,7 +161,7 @@ static int __init smsgiucv_app_init(void) if (!smsgiucv_drv) return -ENODEV; - smsg_app_dev = iucv_alloc_device(NULL, smsgiucv_drv, NULL, KMSG_COMPONENT); + smsg_app_dev = iucv_alloc_device(NULL, smsgiucv_drv, NULL, "smsgiucv_app"); if (!smsg_app_dev) return -ENOMEM; diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c index dc2265ebb11b..01f927ae61b5 100644 --- a/drivers/s390/scsi/zfcp_aux.c +++ b/drivers/s390/scsi/zfcp_aux.c @@ -28,8 +28,7 @@ * Benjamin Block */ -#define KMSG_COMPONENT "zfcp" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zfcp: " fmt #include <linux/seq_file.h> #include <linux/slab.h> diff --git a/drivers/s390/scsi/zfcp_ccw.c b/drivers/s390/scsi/zfcp_ccw.c index bdf2cc1ea713..67cb947048c4 100644 --- a/drivers/s390/scsi/zfcp_ccw.c +++ b/drivers/s390/scsi/zfcp_ccw.c @@ -7,8 +7,7 @@ * Copyright IBM Corp. 2002, 2010 */ -#define KMSG_COMPONENT "zfcp" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zfcp: " fmt #include <linux/module.h> #include "zfcp_ext.h" diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c index d904625afd40..6b5561c54e2f 100644 --- a/drivers/s390/scsi/zfcp_dbf.c +++ b/drivers/s390/scsi/zfcp_dbf.c @@ -7,8 +7,7 @@ * Copyright IBM Corp. 2002, 2023 */ -#define KMSG_COMPONENT "zfcp" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zfcp: " fmt #include <linux/module.h> #include <linux/ctype.h> diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c index ffd994416995..ec6c0e102119 100644 --- a/drivers/s390/scsi/zfcp_erp.c +++ b/drivers/s390/scsi/zfcp_erp.c @@ -7,8 +7,7 @@ * Copyright IBM Corp. 2002, 2020 */ -#define KMSG_COMPONENT "zfcp" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zfcp: " fmt #include <linux/kthread.h> #include <linux/bug.h> diff --git a/drivers/s390/scsi/zfcp_fc.c b/drivers/s390/scsi/zfcp_fc.c index 1d50f463afe7..78ca394e1195 100644 --- a/drivers/s390/scsi/zfcp_fc.c +++ b/drivers/s390/scsi/zfcp_fc.c @@ -7,8 +7,7 @@ * Copyright IBM Corp. 2008, 2017 */ -#define KMSG_COMPONENT "zfcp" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zfcp: " fmt #include <linux/types.h> #include <linux/slab.h> diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c index c5bba1be88f4..9418086368c3 100644 --- a/drivers/s390/scsi/zfcp_fsf.c +++ b/drivers/s390/scsi/zfcp_fsf.c @@ -7,8 +7,7 @@ * Copyright IBM Corp. 2002, 2023 */ -#define KMSG_COMPONENT "zfcp" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zfcp: " fmt #include <linux/blktrace_api.h> #include <linux/jiffies.h> diff --git a/drivers/s390/scsi/zfcp_qdio.c b/drivers/s390/scsi/zfcp_qdio.c index f2410bc44ad3..e15a1eabe42d 100644 --- a/drivers/s390/scsi/zfcp_qdio.c +++ b/drivers/s390/scsi/zfcp_qdio.c @@ -7,8 +7,7 @@ * Copyright IBM Corp. 2002, 2020 */ -#define KMSG_COMPONENT "zfcp" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zfcp: " fmt #include <linux/lockdep.h> #include <linux/slab.h> diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c index b31f860af47b..141476ea21bb 100644 --- a/drivers/s390/scsi/zfcp_scsi.c +++ b/drivers/s390/scsi/zfcp_scsi.c @@ -7,8 +7,7 @@ * Copyright IBM Corp. 2002, 2020 */ -#define KMSG_COMPONENT "zfcp" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zfcp: " fmt #include <linux/module.h> #include <linux/types.h> diff --git a/drivers/s390/scsi/zfcp_sysfs.c b/drivers/s390/scsi/zfcp_sysfs.c index 90a84ae98b97..10a3840b2b6b 100644 --- a/drivers/s390/scsi/zfcp_sysfs.c +++ b/drivers/s390/scsi/zfcp_sysfs.c @@ -7,8 +7,7 @@ * Copyright IBM Corp. 2008, 2020 */ -#define KMSG_COMPONENT "zfcp" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "zfcp: " fmt #include <linux/slab.h> #include "zfcp_diag.h" diff --git a/drivers/scsi/fnic/fnic_trace.c b/drivers/scsi/fnic/fnic_trace.c index cdc6b12b1ec2..4ed57ea1f854 100644 --- a/drivers/scsi/fnic/fnic_trace.c +++ b/drivers/scsi/fnic/fnic_trace.c @@ -138,9 +138,8 @@ int fnic_get_trace_data(fnic_dbgfs_t *fnic_dbgfs_prt) */ len += scnprintf(fnic_dbgfs_prt->buffer + len, (trace_max_pages * PAGE_SIZE * 3) - len, - "%16llu.%09lu %-50s %8x %8x %16llx %16llx " - "%16llx %16llx %16llx\n", (u64)val.tv_sec, - val.tv_nsec, str, tbp->host_no, tbp->tag, + "%ptSp %-50s %8x %8x %16llx %16llx %16llx %16llx %16llx\n", + &val, str, tbp->host_no, tbp->tag, tbp->data[0], tbp->data[1], tbp->data[2], tbp->data[3], tbp->data[4]); rd_idx++; @@ -180,9 +179,8 @@ int fnic_get_trace_data(fnic_dbgfs_t *fnic_dbgfs_prt) */ len += scnprintf(fnic_dbgfs_prt->buffer + len, (trace_max_pages * PAGE_SIZE * 3) - len, - "%16llu.%09lu %-50s %8x %8x %16llx %16llx " - "%16llx %16llx %16llx\n", (u64)val.tv_sec, - val.tv_nsec, str, tbp->host_no, tbp->tag, + "%ptSp %-50s %8x %8x %16llx %16llx %16llx %16llx %16llx\n", + &val, str, tbp->host_no, tbp->tag, tbp->data[0], tbp->data[1], tbp->data[2], tbp->data[3], tbp->data[4]); rd_idx++; @@ -215,32 +213,29 @@ int fnic_get_stats_data(struct stats_debug_info *debug, { int len = 0; int buf_size = debug->buf_size; - struct timespec64 val1, val2; + struct timespec64 val, val1, val2; int i = 0; - ktime_get_real_ts64(&val1); + ktime_get_real_ts64(&val); len = scnprintf(debug->debug_buffer + len, buf_size - len, "------------------------------------------\n" "\t\tTime\n" "------------------------------------------\n"); + val1 = timespec64_sub(val, stats->stats_timestamps.last_reset_time); + val2 = timespec64_sub(val, stats->stats_timestamps.last_read_time); len += scnprintf(debug->debug_buffer + len, buf_size - len, - "Current time : [%lld:%ld]\n" - "Last stats reset time: [%lld:%09ld]\n" - "Last stats read time: [%lld:%ld]\n" - "delta since last reset: [%lld:%ld]\n" - "delta since last read: [%lld:%ld]\n", - (s64)val1.tv_sec, val1.tv_nsec, - (s64)stats->stats_timestamps.last_reset_time.tv_sec, - stats->stats_timestamps.last_reset_time.tv_nsec, - (s64)stats->stats_timestamps.last_read_time.tv_sec, - stats->stats_timestamps.last_read_time.tv_nsec, - (s64)timespec64_sub(val1, stats->stats_timestamps.last_reset_time).tv_sec, - timespec64_sub(val1, stats->stats_timestamps.last_reset_time).tv_nsec, - (s64)timespec64_sub(val1, stats->stats_timestamps.last_read_time).tv_sec, - timespec64_sub(val1, stats->stats_timestamps.last_read_time).tv_nsec); - - stats->stats_timestamps.last_read_time = val1; + "Current time : [%ptSp]\n" + "Last stats reset time: [%ptSp]\n" + "Last stats read time: [%ptSp]\n" + "delta since last reset: [%ptSp]\n" + "delta since last read: [%ptSp]\n", + &val, + &stats->stats_timestamps.last_reset_time, + &stats->stats_timestamps.last_read_time, + &val1, &val2); + + stats->stats_timestamps.last_read_time = val; len += scnprintf(debug->debug_buffer + len, buf_size - len, "------------------------------------------\n" @@ -416,8 +411,8 @@ int fnic_get_stats_data(struct stats_debug_info *debug, jiffies_to_timespec64(stats->misc_stats.last_ack_time, &val2); len += scnprintf(debug->debug_buffer + len, buf_size - len, - "Last ISR time: %llu (%8llu.%09lu)\n" - "Last ACK time: %llu (%8llu.%09lu)\n" + "Last ISR time: %llu (%ptSp)\n" + "Last ACK time: %llu (%ptSp)\n" "Max ISR jiffies: %llu\n" "Max ISR time (ms) (0 denotes < 1 ms): %llu\n" "Corr. work done: %llu\n" @@ -437,10 +432,8 @@ int fnic_get_stats_data(struct stats_debug_info *debug, "Number of rport not ready: %lld\n" "Number of receive frame errors: %lld\n" "Port speed (in Mbps): %lld\n", - (u64)stats->misc_stats.last_isr_time, - (s64)val1.tv_sec, val1.tv_nsec, - (u64)stats->misc_stats.last_ack_time, - (s64)val2.tv_sec, val2.tv_nsec, + (u64)stats->misc_stats.last_isr_time, &val1, + (u64)stats->misc_stats.last_ack_time, &val2, (u64)atomic64_read(&stats->misc_stats.max_isr_jiffies), (u64)atomic64_read(&stats->misc_stats.max_isr_time_ms), (u64)atomic64_read(&stats->misc_stats.corr_work_done), @@ -857,8 +850,8 @@ void copy_and_format_trace_data(struct fc_trace_hdr *tdata, len = *orig_len; len += scnprintf(fnic_dbgfs_prt->buffer + len, max_size - len, - "%ptTs.%09lu ns%8x %c%8x\t", - &tdata->time_stamp.tv_sec, tdata->time_stamp.tv_nsec, + "%ptSs ns%8x %c%8x\t", + &tdata->time_stamp, tdata->host_no, tdata->frame_type, tdata->frame_len); fc_trace = (char *)FC_TRACE_ADDRESS(tdata); diff --git a/drivers/scsi/mesh.c b/drivers/scsi/mesh.c index 1c15cac41d80..768b85eecc8f 100644 --- a/drivers/scsi/mesh.c +++ b/drivers/scsi/mesh.c @@ -1762,6 +1762,7 @@ static int mesh_suspend(struct macio_dev *mdev, pm_message_t mesg) case PM_EVENT_SUSPEND: case PM_EVENT_HIBERNATE: case PM_EVENT_FREEZE: + case PM_EVENT_POWEROFF: break; default: return 0; diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h index 36382eca941c..574af8243016 100644 --- a/drivers/scsi/sd.h +++ b/drivers/scsi/sd.h @@ -240,7 +240,7 @@ blk_status_t sd_zbc_setup_zone_mgmt_cmnd(struct scsi_cmnd *cmd, unsigned int sd_zbc_complete(struct scsi_cmnd *cmd, unsigned int good_bytes, struct scsi_sense_hdr *sshdr); int sd_zbc_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data); + unsigned int nr_zones, struct blk_report_zones_args *args); #else /* CONFIG_BLK_DEV_ZONED */ diff --git a/drivers/scsi/sd_zbc.c b/drivers/scsi/sd_zbc.c index a8db66428f80..56e455fb5add 100644 --- a/drivers/scsi/sd_zbc.c +++ b/drivers/scsi/sd_zbc.c @@ -35,8 +35,7 @@ static bool sd_zbc_is_gap_zone(const u8 buf[64]) * @buf: SCSI zone descriptor. * @idx: Index of the zone relative to the first zone reported by the current * sd_zbc_report_zones() call. - * @cb: Callback function pointer. - * @data: Second argument passed to @cb. + * @args: report zones arguments (callback, etc) * * Return: Value returned by @cb. * @@ -44,12 +43,11 @@ static bool sd_zbc_is_gap_zone(const u8 buf[64]) * call @cb(blk_zone, @data). */ static int sd_zbc_parse_report(struct scsi_disk *sdkp, const u8 buf[64], - unsigned int idx, report_zones_cb cb, void *data) + unsigned int idx, struct blk_report_zones_args *args) { struct scsi_device *sdp = sdkp->device; struct blk_zone zone = { 0 }; sector_t start_lba, gran; - int ret; if (WARN_ON_ONCE(sd_zbc_is_gap_zone(buf))) return -EINVAL; @@ -87,11 +85,7 @@ static int sd_zbc_parse_report(struct scsi_disk *sdkp, const u8 buf[64], else zone.wp = logical_to_sectors(sdp, get_unaligned_be64(&buf[24])); - ret = cb(&zone, idx, data); - if (ret) - return ret; - - return 0; + return disk_report_zone(sdkp->disk, &zone, idx, args); } /** @@ -217,14 +211,14 @@ static inline sector_t sd_zbc_zone_sectors(struct scsi_disk *sdkp) * @disk: Disk to report zones for. * @sector: Start sector. * @nr_zones: Maximum number of zones to report. - * @cb: Callback function called to report zone information. - * @data: Second argument passed to @cb. + * @args: Callback arguments. * * Called by the block layer to iterate over zone information. See also the * disk->fops->report_zones() calls in block/blk-zoned.c. */ int sd_zbc_report_zones(struct gendisk *disk, sector_t sector, - unsigned int nr_zones, report_zones_cb cb, void *data) + unsigned int nr_zones, + struct blk_report_zones_args *args) { struct scsi_disk *sdkp = scsi_disk(disk); sector_t lba = sectors_to_logical(sdkp->device, sector); @@ -283,7 +277,7 @@ int sd_zbc_report_zones(struct gendisk *disk, sector_t sector, } ret = sd_zbc_parse_report(sdkp, buf + offset, zone_idx, - cb, data); + args); if (ret) goto out; diff --git a/drivers/scsi/snic/snic_debugfs.c b/drivers/scsi/snic/snic_debugfs.c index 9dd975b36b5b..edf3e5ef28a6 100644 --- a/drivers/scsi/snic/snic_debugfs.c +++ b/drivers/scsi/snic/snic_debugfs.c @@ -282,8 +282,8 @@ snic_stats_show(struct seq_file *sfp, void *data) jiffies_to_timespec64(stats->misc.last_ack_time, &last_ack_tms); seq_printf(sfp, - "Last ISR Time : %llu (%8llu.%09lu)\n" - "Last Ack Time : %llu (%8llu.%09lu)\n" + "Last ISR Time : %llu (%ptSp)\n" + "Last Ack Time : %llu (%ptSp)\n" "Ack ISRs : %llu\n" "IO Cmpl ISRs : %llu\n" "Err Notify ISRs : %llu\n" @@ -298,10 +298,8 @@ snic_stats_show(struct seq_file *sfp, void *data) "Queue Ramp Down : %lld\n" "Queue Last Queue Depth : %lld\n" "Target Not Ready : %lld\n", - (u64) stats->misc.last_isr_time, - last_isr_tms.tv_sec, last_isr_tms.tv_nsec, - (u64)stats->misc.last_ack_time, - last_ack_tms.tv_sec, last_ack_tms.tv_nsec, + (u64) stats->misc.last_isr_time, &last_isr_tms, + (u64) stats->misc.last_ack_time, &last_ack_tms, (u64) atomic64_read(&stats->misc.ack_isr_cnt), (u64) atomic64_read(&stats->misc.cmpl_isr_cnt), (u64) atomic64_read(&stats->misc.errnotify_isr_cnt), diff --git a/drivers/scsi/snic/snic_trc.c b/drivers/scsi/snic/snic_trc.c index c2e5ab7e976c..6bad1ea9a6a7 100644 --- a/drivers/scsi/snic/snic_trc.c +++ b/drivers/scsi/snic/snic_trc.c @@ -56,9 +56,8 @@ snic_fmt_trc_data(struct snic_trc_data *td, char *buf, int buf_sz) jiffies_to_timespec64(td->ts, &tmspec); len += snprintf(buf, buf_sz, - "%llu.%09lu %-25s %3d %4x %16llx %16llx %16llx %16llx %16llx\n", - tmspec.tv_sec, - tmspec.tv_nsec, + "%ptSp %-25s %3d %4x %16llx %16llx %16llx %16llx %16llx\n", + &tmspec, td->fn, td->hno, td->tag, diff --git a/drivers/scsi/stex.c b/drivers/scsi/stex.c index d8ad02c29320..e6357bc301cb 100644 --- a/drivers/scsi/stex.c +++ b/drivers/scsi/stex.c @@ -1965,6 +1965,7 @@ static int stex_choice_sleep_mic(struct st_hba *hba, pm_message_t state) case PM_EVENT_SUSPEND: return ST_S3; case PM_EVENT_HIBERNATE: + case PM_EVENT_POWEROFF: hba->msi_lock = 0; return ST_S4; default: diff --git a/drivers/slimbus/qcom-ngd-ctrl.c b/drivers/slimbus/qcom-ngd-ctrl.c index fdb94dc4a730..ba3d80d12605 100644 --- a/drivers/slimbus/qcom-ngd-ctrl.c +++ b/drivers/slimbus/qcom-ngd-ctrl.c @@ -1241,6 +1241,7 @@ static void qcom_slim_ngd_notify_slaves(struct qcom_slim_ngd_ctrl *ctrl) if (slim_get_logical_addr(sbdev)) dev_err(ctrl->dev, "Failed to get logical address\n"); + put_device(&sbdev->dev); } } diff --git a/drivers/spi/Kconfig b/drivers/spi/Kconfig index 4d8f00c850c1..55675750182e 100644 --- a/drivers/spi/Kconfig +++ b/drivers/spi/Kconfig @@ -1181,10 +1181,10 @@ config SPI_TEGRA210_QUAD config SPI_TEGRA114 tristate "NVIDIA Tegra114 SPI Controller" - depends on (ARCH_TEGRA && TEGRA20_APB_DMA) || COMPILE_TEST + depends on ARCH_TEGRA || COMPILE_TEST depends on RESET_CONTROLLER help - SPI driver for NVIDIA Tegra114 SPI Controller interface. This controller + SPI controller driver for NVIDIA Tegra114 and later SoCs. This controller is different than the older SoCs SPI controller and also register interface get changed with this controller. diff --git a/drivers/spi/spi-amlogic-spifc-a1.c b/drivers/spi/spi-amlogic-spifc-a1.c index 18c9aa2cbc29..eb503790017b 100644 --- a/drivers/spi/spi-amlogic-spifc-a1.c +++ b/drivers/spi/spi-amlogic-spifc-a1.c @@ -353,7 +353,9 @@ static int amlogic_spifc_a1_probe(struct platform_device *pdev) pm_runtime_set_autosuspend_delay(spifc->dev, 500); pm_runtime_use_autosuspend(spifc->dev); - devm_pm_runtime_enable(spifc->dev); + ret = devm_pm_runtime_enable(spifc->dev); + if (ret) + return ret; ctrl->num_chipselect = 1; ctrl->dev.of_node = pdev->dev.of_node; diff --git a/drivers/spi/spi-bcm63xx.c b/drivers/spi/spi-bcm63xx.c index b56210734caa..2e3c62f12bef 100644 --- a/drivers/spi/spi-bcm63xx.c +++ b/drivers/spi/spi-bcm63xx.c @@ -247,6 +247,20 @@ static int bcm63xx_txrx_bufs(struct spi_device *spi, struct spi_transfer *first, if (t->rx_buf) { do_rx = true; + + /* + * In certain hardware implementations, there appears to be a + * hidden accumulator that tracks the number of bytes written into + * the hardware FIFO, and this accumulator overrides the length in + * the SPI_MSG_CTL register. + * + * Therefore, for read-only transfers, we need to write some dummy + * value into the FIFO to keep the accumulator tracking the correct + * length. + */ + if (!t->tx_buf) + memset_io(bs->tx_io + len, 0xFF, t->len); + /* prepend is half-duplex write only */ if (t == first) prepend_len = 0; diff --git a/drivers/spi/spi-cadence-quadspi.c b/drivers/spi/spi-cadence-quadspi.c index 81017402bc56..af6d050da1c8 100644 --- a/drivers/spi/spi-cadence-quadspi.c +++ b/drivers/spi/spi-cadence-quadspi.c @@ -1981,6 +1981,13 @@ static int cqspi_probe(struct platform_device *pdev) cqspi->current_cs = -1; cqspi->sclk = 0; + if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { + pm_runtime_enable(dev); + pm_runtime_set_autosuspend_delay(dev, CQSPI_AUTOSUSPEND_TIMEOUT); + pm_runtime_use_autosuspend(dev); + pm_runtime_get_noresume(dev); + } + ret = cqspi_setup_flash(cqspi); if (ret) { dev_err(dev, "failed to setup flash parameters %d\n", ret); @@ -1995,14 +2002,7 @@ static int cqspi_probe(struct platform_device *pdev) if (cqspi->use_direct_mode) { ret = cqspi_request_mmap_dma(cqspi); if (ret == -EPROBE_DEFER) - goto probe_dma_failed; - } - - if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { - pm_runtime_enable(dev); - pm_runtime_set_autosuspend_delay(dev, CQSPI_AUTOSUSPEND_TIMEOUT); - pm_runtime_use_autosuspend(dev); - pm_runtime_get_noresume(dev); + goto probe_setup_failed; } ret = spi_register_controller(host); @@ -2012,7 +2012,6 @@ static int cqspi_probe(struct platform_device *pdev) } if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) { - pm_runtime_put_autosuspend(dev); pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); } @@ -2021,7 +2020,6 @@ static int cqspi_probe(struct platform_device *pdev) probe_setup_failed: if (!(ddata && (ddata->quirks & CQSPI_DISABLE_RUNTIME_PM))) pm_runtime_disable(dev); -probe_dma_failed: cqspi_controller_enable(cqspi, 0); probe_reset_failed: if (cqspi->is_jh7110) diff --git a/drivers/spi/spi-fsl-lpspi.c b/drivers/spi/spi-fsl-lpspi.c index 8da66e101386..065456aba2ae 100644 --- a/drivers/spi/spi-fsl-lpspi.c +++ b/drivers/spi/spi-fsl-lpspi.c @@ -486,7 +486,13 @@ static int fsl_lpspi_setup_transfer(struct spi_controller *controller, fsl_lpspi->tx = fsl_lpspi_buf_tx_u32; } - fsl_lpspi->watermark = min_t(typeof(fsl_lpspi->watermark), + /* + * t->len is 'unsigned' and txfifosize and watermrk is 'u8', force + * type cast is inevitable. When len > 255, len will be truncated in min_t(), + * it caused wrong watermark set. 'unsigned int' is as the designated type + * for min_t() to avoid truncation. + */ + fsl_lpspi->watermark = min_t(unsigned int, fsl_lpspi->txfifosize, t->len); diff --git a/drivers/spi/spi-nxp-fspi.c b/drivers/spi/spi-nxp-fspi.c index b6c79e50d842..50a7e4916a60 100644 --- a/drivers/spi/spi-nxp-fspi.c +++ b/drivers/spi/spi-nxp-fspi.c @@ -1287,7 +1287,7 @@ static int nxp_fspi_probe(struct platform_device *pdev) { struct spi_controller *ctlr; struct device *dev = &pdev->dev; - struct device_node *np = dev->of_node; + struct fwnode_handle *fwnode = dev_fwnode(dev); struct resource *res; struct nxp_fspi *f; int ret, irq; @@ -1309,7 +1309,7 @@ static int nxp_fspi_probe(struct platform_device *pdev) platform_set_drvdata(pdev, f); /* find the resources - configuration register address space */ - if (is_acpi_node(dev_fwnode(f->dev))) + if (is_acpi_node(fwnode)) f->iobase = devm_platform_ioremap_resource(pdev, 0); else f->iobase = devm_platform_ioremap_resource_byname(pdev, "fspi_base"); @@ -1317,7 +1317,7 @@ static int nxp_fspi_probe(struct platform_device *pdev) return PTR_ERR(f->iobase); /* find the resources - controller memory mapped space */ - if (is_acpi_node(dev_fwnode(f->dev))) + if (is_acpi_node(fwnode)) res = platform_get_resource(pdev, IORESOURCE_MEM, 1); else res = platform_get_resource_byname(pdev, @@ -1330,7 +1330,7 @@ static int nxp_fspi_probe(struct platform_device *pdev) f->memmap_phy_size = resource_size(res); /* find the clocks */ - if (dev_of_node(&pdev->dev)) { + if (is_of_node(fwnode)) { f->clk_en = devm_clk_get(dev, "fspi_en"); if (IS_ERR(f->clk_en)) return PTR_ERR(f->clk_en); @@ -1383,7 +1383,7 @@ static int nxp_fspi_probe(struct platform_device *pdev) else ctlr->mem_caps = &nxp_fspi_mem_caps; - ctlr->dev.of_node = np; + device_set_node(&ctlr->dev, fwnode); ret = devm_add_action_or_reset(dev, nxp_fspi_cleanup, f); if (ret) diff --git a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c index 6fc39ab95e46..6050637a0def 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-gc2235.c @@ -491,7 +491,7 @@ static int gc2235_s_power(struct v4l2_subdev *sd, int on) return ret; } -static int startup(struct v4l2_subdev *sd) +static int gc2235_startup(struct v4l2_subdev *sd) { struct gc2235_device *dev = to_gc2235_sensor(sd); struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -556,7 +556,7 @@ static int gc2235_set_fmt(struct v4l2_subdev *sd, return 0; } - ret = startup(sd); + ret = gc2235_startup(sd); if (ret) { dev_err(&client->dev, "gc2235 startup err\n"); goto err; diff --git a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c index c7de7800799a..a4519babf37d 100644 --- a/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c +++ b/drivers/staging/media/atomisp/i2c/atomisp-ov2722.c @@ -600,7 +600,7 @@ static int ov2722_s_power(struct v4l2_subdev *sd, int on) } /* TODO: remove it. */ -static int startup(struct v4l2_subdev *sd) +static int ov2722_startup(struct v4l2_subdev *sd) { struct ov2722_device *dev = to_ov2722_sensor(sd); struct i2c_client *client = v4l2_get_subdevdata(sd); @@ -662,7 +662,7 @@ static int ov2722_set_fmt(struct v4l2_subdev *sd, dev->pixels_per_line = dev->res->pixels_per_line; dev->lines_per_frame = dev->res->lines_per_frame; - ret = startup(sd); + ret = ov2722_startup(sd); if (ret) { int i = 0; @@ -677,7 +677,7 @@ static int ov2722_set_fmt(struct v4l2_subdev *sd, dev_err(&client->dev, "power up failed, continue\n"); continue; } - ret = startup(sd); + ret = ov2722_startup(sd); if (ret) { dev_err(&client->dev, " startup FAILED!\n"); } else { diff --git a/drivers/staging/media/av7110/av7110.c b/drivers/staging/media/av7110/av7110.c index bc9a2a40afcb..602342d1174f 100644 --- a/drivers/staging/media/av7110/av7110.c +++ b/drivers/staging/media/av7110/av7110.c @@ -321,7 +321,7 @@ static inline void print_time(char *s) struct timespec64 ts; ktime_get_real_ts64(&ts); - pr_info("%s(): %lld.%09ld\n", s, (s64)ts.tv_sec, ts.tv_nsec); + pr_info("%s(): %ptSp\n", s, &ts); #endif } diff --git a/drivers/target/target_core_configfs.c b/drivers/target/target_core_configfs.c index b19acd662726..9e51c535ba8c 100644 --- a/drivers/target/target_core_configfs.c +++ b/drivers/target/target_core_configfs.c @@ -3670,8 +3670,6 @@ static int __init target_core_init_configfs(void) { struct configfs_subsystem *subsys = &target_core_fabrics; struct t10_alua_lu_gp *lu_gp; - struct cred *kern_cred; - const struct cred *old_cred; int ret; pr_debug("TARGET_CORE[0]: Loading Generic Kernel Storage" @@ -3748,16 +3746,8 @@ static int __init target_core_init_configfs(void) if (ret < 0) goto out; - /* We use the kernel credentials to access the target directory */ - kern_cred = prepare_kernel_cred(&init_task); - if (!kern_cred) { - ret = -ENOMEM; - goto out; - } - old_cred = override_creds(kern_cred); - target_init_dbroot(); - revert_creds(old_cred); - put_cred(kern_cred); + scoped_with_kernel_creds() + target_init_dbroot(); return 0; diff --git a/drivers/tee/qcomtee/call.c b/drivers/tee/qcomtee/call.c index ac134452cc9c..65f9140d4e1f 100644 --- a/drivers/tee/qcomtee/call.c +++ b/drivers/tee/qcomtee/call.c @@ -645,7 +645,7 @@ static void qcomtee_get_version(struct tee_device *teedev, static void qcomtee_get_qtee_feature_list(struct tee_context *ctx, u32 id, u32 *version) { - struct qcomtee_object_invoke_ctx *oic __free(kfree); + struct qcomtee_object_invoke_ctx *oic __free(kfree) = NULL; struct qcomtee_object *client_env, *service; struct qcomtee_arg u[3] = { 0 }; int result; diff --git a/drivers/tee/qcomtee/core.c b/drivers/tee/qcomtee/core.c index b6715ada7700..ecd04403591c 100644 --- a/drivers/tee/qcomtee/core.c +++ b/drivers/tee/qcomtee/core.c @@ -82,7 +82,7 @@ static void qcomtee_do_release_qtee_object(struct work_struct *work) { struct qcomtee_object *object; struct qcomtee *qcomtee; - int ret, result; + int ret, result = 0; /* RELEASE does not require any argument. */ struct qcomtee_arg args[] = { { .type = QCOMTEE_ARG_TYPE_INV } }; diff --git a/drivers/thermal/Kconfig b/drivers/thermal/Kconfig index a09c188b9ad1..b10080d61860 100644 --- a/drivers/thermal/Kconfig +++ b/drivers/thermal/Kconfig @@ -296,6 +296,16 @@ config IMX8MM_THERMAL cpufreq is used as the cooling device to throttle CPUs when the passive trip is crossed. +config IMX91_THERMAL + tristate "Temperature sensor driver for NXP i.MX91 SoC" + depends on ARCH_MXC || COMPILE_TEST + depends on OF + help + Include one sensor and six comparators. Each of them compares the + temperature value (from the sensor) against the programmable + threshold values. The direction of the comparison is configurable + (greater / lesser than). + config K3_THERMAL tristate "Texas Instruments K3 thermal support" depends on ARCH_K3 || COMPILE_TEST diff --git a/drivers/thermal/Makefile b/drivers/thermal/Makefile index d7718978db24..bb21e7ea7fc6 100644 --- a/drivers/thermal/Makefile +++ b/drivers/thermal/Makefile @@ -51,6 +51,7 @@ obj-$(CONFIG_ARMADA_THERMAL) += armada_thermal.o obj-$(CONFIG_IMX_THERMAL) += imx_thermal.o obj-$(CONFIG_IMX_SC_THERMAL) += imx_sc_thermal.o obj-$(CONFIG_IMX8MM_THERMAL) += imx8mm_thermal.o +obj-$(CONFIG_IMX91_THERMAL) += imx91_thermal.o obj-$(CONFIG_MAX77620_THERMAL) += max77620_thermal.o obj-$(CONFIG_QORIQ_THERMAL) += qoriq_thermal.o obj-$(CONFIG_DA9062_THERMAL) += da9062-thermal.o diff --git a/drivers/thermal/imx91_thermal.c b/drivers/thermal/imx91_thermal.c new file mode 100644 index 000000000000..9b20be03d6de --- /dev/null +++ b/drivers/thermal/imx91_thermal.c @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright 2025 NXP. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/nvmem-consumer.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_device.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/thermal.h> +#include <linux/units.h> + +#define REG_SET 0x4 +#define REG_CLR 0x8 +#define REG_TOG 0xc + +#define IMX91_TMU_CTRL0 0x0 +#define IMX91_TMU_CTRL0_THR1_IE BIT(9) +#define IMX91_TMU_CTRL0_THR1_MASK GENMASK(3, 2) +#define IMX91_TMU_CTRL0_CLR_FLT1 BIT(21) + +#define IMX91_TMU_THR_MODE_LE 0 +#define IMX91_TMU_THR_MODE_GE 1 + +#define IMX91_TMU_STAT0 0x10 +#define IMX91_TMU_STAT0_THR1_IF BIT(9) +#define IMX91_TMU_STAT0_THR1_STAT BIT(13) +#define IMX91_TMU_STAT0_DRDY0_IF_MASK BIT(16) + +#define IMX91_TMU_DATA0 0x20 + +#define IMX91_TMU_CTRL1 0x200 +#define IMX91_TMU_CTRL1_EN BIT(31) +#define IMX91_TMU_CTRL1_START BIT(30) +#define IMX91_TMU_CTRL1_STOP BIT(29) +#define IMX91_TMU_CTRL1_RES_MASK GENMASK(19, 18) +#define IMX91_TMU_CTRL1_MEAS_MODE_MASK GENMASK(25, 24) +#define IMX91_TMU_CTRL1_MEAS_MODE_SINGLE 0 +#define IMX91_TMU_CTRL1_MEAS_MODE_CONTINUES 1 +#define IMX91_TMU_CTRL1_MEAS_MODE_PERIODIC 2 + +#define IMX91_TMU_THR_CTRL01 0x30 +#define IMX91_TMU_THR_CTRL01_THR1_MASK GENMASK(31, 16) + +#define IMX91_TMU_REF_DIV 0x280 +#define IMX91_TMU_DIV_EN BIT(31) +#define IMX91_TMU_DIV_MASK GENMASK(23, 16) +#define IMX91_TMU_DIV_MAX 255 + +#define IMX91_TMU_PUD_ST_CTRL 0x2b0 +#define IMX91_TMU_PUDL_MASK GENMASK(23, 16) + +#define IMX91_TMU_TRIM1 0x2e0 +#define IMX91_TMU_TRIM2 0x2f0 + +#define IMX91_TMU_TEMP_LOW_LIMIT -40000 +#define IMX91_TMU_TEMP_HIGH_LIMIT 125000 + +#define IMX91_TMU_DEFAULT_TRIM1_CONFIG 0xb561bc2d +#define IMX91_TMU_DEFAULT_TRIM2_CONFIG 0x65d4 + +#define IMX91_TMU_PERIOD_CTRL 0x270 +#define IMX91_TMU_PERIOD_CTRL_MEAS_MASK GENMASK(23, 0) + +#define IMX91_TMP_FRAC 64 + +struct imx91_tmu { + void __iomem *base; + struct clk *clk; + struct device *dev; + struct thermal_zone_device *tzd; +}; + +static void imx91_tmu_start(struct imx91_tmu *tmu, bool start) +{ + u32 val = start ? IMX91_TMU_CTRL1_START : IMX91_TMU_CTRL1_STOP; + + writel_relaxed(val, tmu->base + IMX91_TMU_CTRL1 + REG_SET); +} + +static void imx91_tmu_enable(struct imx91_tmu *tmu, bool enable) +{ + u32 reg = IMX91_TMU_CTRL1; + + reg += enable ? REG_SET : REG_CLR; + + writel_relaxed(IMX91_TMU_CTRL1_EN, tmu->base + reg); +} + +static int imx91_tmu_to_mcelsius(int x) +{ + return x * MILLIDEGREE_PER_DEGREE / IMX91_TMP_FRAC; +} + +static int imx91_tmu_from_mcelsius(int x) +{ + return x * IMX91_TMP_FRAC / MILLIDEGREE_PER_DEGREE; +} + +static int imx91_tmu_get_temp(struct thermal_zone_device *tz, int *temp) +{ + struct imx91_tmu *tmu = thermal_zone_device_priv(tz); + s16 data; + + /* DATA0 is 16bit signed number */ + data = readw_relaxed(tmu->base + IMX91_TMU_DATA0); + *temp = imx91_tmu_to_mcelsius(data); + + return 0; +} + +static int imx91_tmu_set_trips(struct thermal_zone_device *tz, int low, int high) +{ + struct imx91_tmu *tmu = thermal_zone_device_priv(tz); + int val; + + if (high >= IMX91_TMU_TEMP_HIGH_LIMIT) + return -EINVAL; + + writel_relaxed(IMX91_TMU_CTRL0_THR1_IE, tmu->base + IMX91_TMU_CTRL0 + REG_CLR); + + /* Comparator1 for temperature threshold */ + writel_relaxed(IMX91_TMU_THR_CTRL01_THR1_MASK, tmu->base + IMX91_TMU_THR_CTRL01 + REG_CLR); + val = FIELD_PREP(IMX91_TMU_THR_CTRL01_THR1_MASK, imx91_tmu_from_mcelsius(high)); + + writel_relaxed(val, tmu->base + IMX91_TMU_THR_CTRL01 + REG_SET); + + writel_relaxed(IMX91_TMU_STAT0_THR1_IF, tmu->base + IMX91_TMU_STAT0 + REG_CLR); + + writel_relaxed(IMX91_TMU_CTRL0_THR1_IE, tmu->base + IMX91_TMU_CTRL0 + REG_SET); + + return 0; +} + +static int imx91_init_from_nvmem_cells(struct imx91_tmu *tmu) +{ + struct device *dev = tmu->dev; + u32 trim1, trim2; + int ret; + + ret = nvmem_cell_read_u32(dev, "trim1", &trim1); + if (ret) + return ret; + + ret = nvmem_cell_read_u32(dev, "trim2", &trim2); + if (ret) + return ret; + + if (trim1 == 0 || trim2 == 0) + return -EINVAL; + + writel_relaxed(trim1, tmu->base + IMX91_TMU_TRIM1); + writel_relaxed(trim2, tmu->base + IMX91_TMU_TRIM2); + + return 0; +} + +static void imx91_tmu_action_remove(void *data) +{ + struct imx91_tmu *tmu = data; + + /* disable tmu */ + imx91_tmu_enable(tmu, false); +} + +static irqreturn_t imx91_tmu_alarm_irq(int irq, void *data) +{ + struct imx91_tmu *tmu = data; + u32 val; + + val = readl_relaxed(tmu->base + IMX91_TMU_STAT0); + + /* Check if comparison interrupt occurred */ + if (val & IMX91_TMU_STAT0_THR1_IF) { + /* Clear irq flag and disable interrupt until reconfigured */ + writel(IMX91_TMU_STAT0_THR1_IF, tmu->base + IMX91_TMU_STAT0 + REG_CLR); + writel_relaxed(IMX91_TMU_CTRL0_THR1_IE, tmu->base + IMX91_TMU_CTRL0 + REG_CLR); + + return IRQ_WAKE_THREAD; + } + + return IRQ_NONE; +} + +static irqreturn_t imx91_tmu_alarm_irq_thread(int irq, void *data) +{ + struct imx91_tmu *tmu = data; + + thermal_zone_device_update(tmu->tzd, THERMAL_EVENT_UNSPECIFIED); + + return IRQ_HANDLED; +} + +static int imx91_tmu_change_mode(struct thermal_zone_device *tz, enum thermal_device_mode mode) +{ + struct imx91_tmu *tmu = thermal_zone_device_priv(tz); + int ret; + + if (mode == THERMAL_DEVICE_ENABLED) { + ret = pm_runtime_get(tmu->dev); + if (ret < 0) + return ret; + + writel_relaxed(IMX91_TMU_CTRL0_THR1_IE | IMX91_TMU_CTRL0_THR1_MASK, + tmu->base + IMX91_TMU_CTRL0 + REG_CLR); + + writel_relaxed(FIELD_PREP(IMX91_TMU_CTRL0_THR1_MASK, IMX91_TMU_THR_MODE_GE), + tmu->base + IMX91_TMU_CTRL0 + REG_SET); + imx91_tmu_start(tmu, true); + } else { + writel_relaxed(IMX91_TMU_CTRL0_THR1_IE, tmu->base + IMX91_TMU_CTRL0 + REG_CLR); + imx91_tmu_start(tmu, false); + pm_runtime_put(tmu->dev); + } + + return 0; +} + +static struct thermal_zone_device_ops tmu_tz_ops = { + .get_temp = imx91_tmu_get_temp, + .change_mode = imx91_tmu_change_mode, + .set_trips = imx91_tmu_set_trips, +}; + +static int imx91_tmu_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct imx91_tmu *tmu; + unsigned long rate; + int irq, ret; + u32 div; + + tmu = devm_kzalloc(dev, sizeof(struct imx91_tmu), GFP_KERNEL); + if (!tmu) + return -ENOMEM; + + tmu->dev = dev; + + tmu->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(tmu->base)) + return dev_err_probe(dev, PTR_ERR(tmu->base), "failed to get io resource"); + + tmu->clk = devm_clk_get_enabled(dev, NULL); + if (IS_ERR(tmu->clk)) + return dev_err_probe(dev, PTR_ERR(tmu->clk), "failed to get tmu clock\n"); + + platform_set_drvdata(pdev, tmu); + + /* disable the monitor during initialization */ + imx91_tmu_enable(tmu, false); + imx91_tmu_start(tmu, false); + + ret = imx91_init_from_nvmem_cells(tmu); + if (ret) { + dev_warn(dev, "can't get trim value, use default settings\n"); + + writel_relaxed(IMX91_TMU_DEFAULT_TRIM1_CONFIG, tmu->base + IMX91_TMU_TRIM1); + writel_relaxed(IMX91_TMU_DEFAULT_TRIM2_CONFIG, tmu->base + IMX91_TMU_TRIM2); + } + + /* The typical conv clk is 4MHz, the output freq is 'rate / (div + 1)' */ + rate = clk_get_rate(tmu->clk); + div = (rate / (4 * HZ_PER_MHZ)) - 1; + if (div > IMX91_TMU_DIV_MAX) + return dev_err_probe(dev, -EINVAL, "clock divider exceed hardware limitation"); + + /* Set divider value and enable divider */ + writel_relaxed(IMX91_TMU_DIV_EN | FIELD_PREP(IMX91_TMU_DIV_MASK, div), + tmu->base + IMX91_TMU_REF_DIV); + + /* Set max power up delay: 'Tpud(ms) = 0xFF * 1000 / 4000000' */ + writel_relaxed(FIELD_PREP(IMX91_TMU_PUDL_MASK, 100U), tmu->base + IMX91_TMU_PUD_ST_CTRL); + + /* + * Set resolution mode + * 00b - Conversion time = 0.59325 ms + * 01b - Conversion time = 1.10525 ms + * 10b - Conversion time = 2.12925 ms + * 11b - Conversion time = 4.17725 ms + */ + writel_relaxed(FIELD_PREP(IMX91_TMU_CTRL1_RES_MASK, 0x3), + tmu->base + IMX91_TMU_CTRL1 + REG_CLR); + writel_relaxed(FIELD_PREP(IMX91_TMU_CTRL1_RES_MASK, 0x1), + tmu->base + IMX91_TMU_CTRL1 + REG_SET); + + writel_relaxed(IMX91_TMU_CTRL1_MEAS_MODE_MASK, tmu->base + IMX91_TMU_CTRL1 + REG_CLR); + writel_relaxed(FIELD_PREP(IMX91_TMU_CTRL1_MEAS_MODE_MASK, + IMX91_TMU_CTRL1_MEAS_MODE_PERIODIC), + tmu->base + IMX91_TMU_CTRL1 + REG_SET); + + /* + * Set Periodic Measurement Frequency to 25Hz: + * tMEAS_FREQ = tCONV_CLK * PERIOD_CTRL[MEAS_FREQ] + */ + writel_relaxed(FIELD_PREP(IMX91_TMU_PERIOD_CTRL_MEAS_MASK, 4 * HZ_PER_MHZ / 25), + tmu->base + IMX91_TMU_PERIOD_CTRL); + + imx91_tmu_enable(tmu, true); + ret = devm_add_action(dev, imx91_tmu_action_remove, tmu); + if (ret) + return dev_err_probe(dev, ret, "Failure to add action imx91_tmu_action_remove()\n"); + + pm_runtime_set_active(dev); + pm_runtime_get_noresume(dev); + ret = devm_pm_runtime_enable(dev); + if (ret) + return ret; + + tmu->tzd = devm_thermal_of_zone_register(dev, 0, tmu, &tmu_tz_ops); + if (IS_ERR(tmu->tzd)) + return dev_err_probe(dev, PTR_ERR(tmu->tzd), + "failed to register thermal zone sensor\n"); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + ret = devm_request_threaded_irq(dev, irq, imx91_tmu_alarm_irq, + imx91_tmu_alarm_irq_thread, + IRQF_ONESHOT, "imx91_thermal", tmu); + + if (ret < 0) + return dev_err_probe(dev, ret, "failed to request alarm irq\n"); + + pm_runtime_put(dev); + + return 0; +} + +static int imx91_tmu_runtime_suspend(struct device *dev) +{ + struct imx91_tmu *tmu = dev_get_drvdata(dev); + + /* disable tmu */ + imx91_tmu_enable(tmu, false); + + clk_disable_unprepare(tmu->clk); + + return 0; +} + +static int imx91_tmu_runtime_resume(struct device *dev) +{ + struct imx91_tmu *tmu = dev_get_drvdata(dev); + int ret; + + ret = clk_prepare_enable(tmu->clk); + if (ret) + return ret; + + imx91_tmu_enable(tmu, true); + + return 0; +} + +static DEFINE_RUNTIME_DEV_PM_OPS(imx91_tmu_pm_ops, imx91_tmu_runtime_suspend, + imx91_tmu_runtime_resume, NULL); + +static const struct of_device_id imx91_tmu_table[] = { + { .compatible = "fsl,imx91-tmu", }, + { }, +}; +MODULE_DEVICE_TABLE(of, imx91_tmu_table); + +static struct platform_driver imx91_tmu = { + .driver = { + .name = "imx91_thermal", + .pm = pm_ptr(&imx91_tmu_pm_ops), + .of_match_table = imx91_tmu_table, + }, + .probe = imx91_tmu_probe, +}; +module_platform_driver(imx91_tmu); + +MODULE_AUTHOR("Peng Fan <peng.fan@nxp.com>"); +MODULE_DESCRIPTION("i.MX91 Thermal Monitor Unit driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/thermal/intel/Kconfig b/drivers/thermal/intel/Kconfig index e0268fac7093..347c59bc87d6 100644 --- a/drivers/thermal/intel/Kconfig +++ b/drivers/thermal/intel/Kconfig @@ -44,7 +44,8 @@ config INTEL_SOC_DTS_IOSF_CORE config INTEL_SOC_DTS_THERMAL tristate "Intel SoCs DTS thermal driver" - depends on X86 && PCI && ACPI + depends on X86_64 && PCI && ACPI && NET + select INT340X_THERMAL select INTEL_SOC_DTS_IOSF_CORE help Enable this to register Intel SoCs (e.g. Bay Trail) platform digital diff --git a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c index 908cc1bf57f1..41d3bc3ed8a2 100644 --- a/drivers/thermal/intel/int340x_thermal/int3400_thermal.c +++ b/drivers/thermal/intel/int340x_thermal/int3400_thermal.c @@ -16,6 +16,8 @@ #define INT3400_ODVP_CHANGED 0x88 #define INT3400_KEEP_ALIVE 0xA0 #define INT3400_FAKE_TEMP (20 * 1000) /* faked temp sensor with 20C */ +/* UUID prefix length for comparison - sufficient for all UUIDs */ +#define INT3400_UUID_PREFIX_LEN 7 enum int3400_thermal_uuid { INT3400_THERMAL_ACTIVE = 0, @@ -112,7 +114,7 @@ static ssize_t available_uuids_show(struct device *dev, int length = 0; if (!priv->uuid_bitmap) - return sprintf(buf, "UNKNOWN\n"); + return sysfs_emit(buf, "UNKNOWN\n"); for (i = 0; i < INT3400_THERMAL_MAXIMUM_UUID; i++) { if (priv->uuid_bitmap & (1 << i)) @@ -129,7 +131,7 @@ static ssize_t current_uuid_show(struct device *dev, int i, length = 0; if (priv->current_uuid_index >= 0) - return sprintf(buf, "%s\n", + return sysfs_emit(buf, "%s\n", int3400_thermal_uuids[priv->current_uuid_index]); for (i = 0; i <= INT3400_THERMAL_CRITICAL; i++) { @@ -140,7 +142,7 @@ static ssize_t current_uuid_show(struct device *dev, if (length) return length; - return sprintf(buf, "INVALID\n"); + return sysfs_emit(buf, "INVALID\n"); } static int int3400_thermal_run_osc(acpi_handle handle, char *uuid_str, int *enable) @@ -199,7 +201,7 @@ static ssize_t current_uuid_store(struct device *dev, for (i = 0; i < INT3400_THERMAL_MAXIMUM_UUID; ++i) { if (!strncmp(buf, int3400_thermal_uuids[i], - sizeof(int3400_thermal_uuids[i]) - 1)) { + INT3400_UUID_PREFIX_LEN)) { /* * If we have a list of supported UUIDs, make sure * this one is supported. @@ -340,7 +342,7 @@ static ssize_t odvp_show(struct device *dev, struct device_attribute *attr, odvp_attr = container_of(attr, struct odvp_attr, attr); - return sprintf(buf, "%d\n", odvp_attr->priv->odvp[odvp_attr->odvp]); + return sysfs_emit(buf, "%d\n", odvp_attr->priv->odvp[odvp_attr->odvp]); } static void cleanup_odvp(struct int3400_thermal_priv *priv) @@ -691,6 +693,7 @@ static const struct acpi_device_id int3400_thermal_match[] = { {"INTC10A0", 0}, {"INTC10D4", 0}, {"INTC10FC", 0}, + {"INTC10F3", 0}, {} }; diff --git a/drivers/thermal/intel/int340x_thermal/int3403_thermal.c b/drivers/thermal/intel/int340x_thermal/int3403_thermal.c index ba63796761eb..264c9bc8e645 100644 --- a/drivers/thermal/intel/int340x_thermal/int3403_thermal.c +++ b/drivers/thermal/intel/int340x_thermal/int3403_thermal.c @@ -277,6 +277,7 @@ static const struct acpi_device_id int3403_device_ids[] = { {"INTC10A1", 0}, {"INTC10D5", 0}, {"INTC10FD", 0}, + {"INTC10F4", 0}, {"", 0}, }; MODULE_DEVICE_TABLE(acpi, int3403_device_ids); diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h index 30760475102f..b79937a386ec 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device.h @@ -27,6 +27,8 @@ #define PCI_DEVICE_ID_INTEL_JSL_THERMAL 0x4E03 #define PCI_DEVICE_ID_INTEL_LNLM_THERMAL 0x641D #define PCI_DEVICE_ID_INTEL_MTLP_THERMAL 0x7D03 +#define PCI_DEVICE_ID_INTEL_NVL_H_THERMAL 0xD703 +#define PCI_DEVICE_ID_INTEL_NVL_S_THERMAL 0xAD03 #define PCI_DEVICE_ID_INTEL_RPL_THERMAL 0xA71D #define PCI_DEVICE_ID_INTEL_SKL_THERMAL 0x1903 #define PCI_DEVICE_ID_INTEL_TGL_THERMAL 0x9A03 diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c index e2471768d355..0d4dcc66e097 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_device_pci.c @@ -504,6 +504,16 @@ static const struct pci_device_id proc_thermal_pci_ids[] = { PROC_THERMAL_FEATURE_RAPL | PROC_THERMAL_FEATURE_DLVR | PROC_THERMAL_FEATURE_DVFS | PROC_THERMAL_FEATURE_WT_HINT | PROC_THERMAL_FEATURE_POWER_FLOOR | PROC_THERMAL_FEATURE_PTC) }, + { PCI_DEVICE_DATA(INTEL, NVL_H_THERMAL, PROC_THERMAL_FEATURE_RAPL | + PROC_THERMAL_FEATURE_DLVR | PROC_THERMAL_FEATURE_DVFS | + PROC_THERMAL_FEATURE_MSI_SUPPORT | PROC_THERMAL_FEATURE_WT_HINT | + PROC_THERMAL_FEATURE_POWER_FLOOR | PROC_THERMAL_FEATURE_PTC | + PROC_THERMAL_FEATURE_SOC_POWER_SLIDER) }, + { PCI_DEVICE_DATA(INTEL, NVL_S_THERMAL, PROC_THERMAL_FEATURE_RAPL | + PROC_THERMAL_FEATURE_DLVR | PROC_THERMAL_FEATURE_DVFS | + PROC_THERMAL_FEATURE_MSI_SUPPORT | PROC_THERMAL_FEATURE_WT_HINT | + PROC_THERMAL_FEATURE_POWER_FLOOR | PROC_THERMAL_FEATURE_PTC | + PROC_THERMAL_FEATURE_SOC_POWER_SLIDER) }, { }, }; diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c index bde2cc386afd..bf51a17c5be6 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rapl.c @@ -19,7 +19,7 @@ static const struct rapl_mmio_regs rapl_mmio_default = { .limits[RAPL_DOMAIN_DRAM] = BIT(POWER_LIMIT2), }; -static int rapl_mmio_read_raw(int cpu, struct reg_action *ra) +static int rapl_mmio_read_raw(int cpu, struct reg_action *ra, bool atomic) { if (!ra->reg.mmio) return -EINVAL; diff --git a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c index 1f3d22b659db..589a3a71f0c4 100644 --- a/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c +++ b/drivers/thermal/intel/int340x_thermal/processor_thermal_rfim.c @@ -87,6 +87,17 @@ static const struct mapping_table lnl_dlvr_mapping[] = { {NULL, 0, NULL}, }; +static const struct mmio_reg nvl_dlvr_mmio_regs[] = { + { 0, 0x19208, 5, 0x1F, 0}, /* dlvr_spread_spectrum_pct */ + { 0, 0x19208, 1, 0x1, 5}, /* dlvr_control_mode */ + { 0, 0x19208, 1, 0x1, 6}, /* dlvr_control_lock */ + { 0, 0x19208, 1, 0x1, 7}, /* dlvr_rfim_enable */ + { 0, 0x19208, 12, 0xFFF, 8}, /* dlvr_freq_select */ + { 1, 0x19210, 2, 0x3, 30}, /* dlvr_hardware_rev */ + { 1, 0x19210, 16, 0xFFFF, 0}, /* dlvr_freq_mhz */ + { 1, 0x19210, 1, 0x1, 16}, /* dlvr_pll_busy */ +}; + static int match_mapping_table(const struct mapping_table *table, const char *attr_name, bool match_int_value, const u32 value, const char *value_str, char **result_str, u32 *result_int) @@ -446,6 +457,10 @@ int proc_thermal_rfim_add(struct pci_dev *pdev, struct proc_thermal_device *proc dlvr_mmio_regs_table = lnl_dlvr_mmio_regs; dlvr_mapping = lnl_dlvr_mapping; break; + case PCI_DEVICE_ID_INTEL_NVL_H_THERMAL: + case PCI_DEVICE_ID_INTEL_NVL_S_THERMAL: + dlvr_mmio_regs_table = nvl_dlvr_mmio_regs; + break; default: dlvr_mmio_regs_table = dlvr_mmio_regs; break; diff --git a/drivers/thermal/renesas/rcar_gen3_thermal.c b/drivers/thermal/renesas/rcar_gen3_thermal.c index 3223de238d01..94804816e9e1 100644 --- a/drivers/thermal/renesas/rcar_gen3_thermal.c +++ b/drivers/thermal/renesas/rcar_gen3_thermal.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * R-Car Gen3 THS thermal sensor driver + * R-Car Gen3, Gen4 and RZ/G2 THS thermal sensor driver * Based on rcar_thermal.c and work from Hien Dang and Khiem Nguyen. * * Copyright (C) 2016 Renesas Electronics Corporation. @@ -601,7 +601,7 @@ error_unregister: return ret; } -static int __maybe_unused rcar_gen3_thermal_resume(struct device *dev) +static int rcar_gen3_thermal_resume(struct device *dev) { struct rcar_gen3_thermal_priv *priv = dev_get_drvdata(dev); unsigned int i; @@ -615,13 +615,13 @@ static int __maybe_unused rcar_gen3_thermal_resume(struct device *dev) return 0; } -static SIMPLE_DEV_PM_OPS(rcar_gen3_thermal_pm_ops, NULL, - rcar_gen3_thermal_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(rcar_gen3_thermal_pm_ops, NULL, + rcar_gen3_thermal_resume); static struct platform_driver rcar_gen3_thermal_driver = { .driver = { .name = "rcar_gen3_thermal", - .pm = &rcar_gen3_thermal_pm_ops, + .pm = pm_sleep_ptr(&rcar_gen3_thermal_pm_ops), .of_match_table = rcar_gen3_thermal_dt_ids, }, .probe = rcar_gen3_thermal_probe, diff --git a/drivers/thermal/renesas/rcar_thermal.c b/drivers/thermal/renesas/rcar_thermal.c index fdd7afdc4ff6..6e5dcac5d47a 100644 --- a/drivers/thermal/renesas/rcar_thermal.c +++ b/drivers/thermal/renesas/rcar_thermal.c @@ -534,7 +534,6 @@ error_unregister: return ret; } -#ifdef CONFIG_PM_SLEEP static int rcar_thermal_suspend(struct device *dev) { struct rcar_thermal_common *common = dev_get_drvdata(dev); @@ -567,15 +566,14 @@ static int rcar_thermal_resume(struct device *dev) return 0; } -#endif -static SIMPLE_DEV_PM_OPS(rcar_thermal_pm_ops, rcar_thermal_suspend, - rcar_thermal_resume); +static DEFINE_SIMPLE_DEV_PM_OPS(rcar_thermal_pm_ops, rcar_thermal_suspend, + rcar_thermal_resume); static struct platform_driver rcar_thermal_driver = { .driver = { .name = "rcar_thermal", - .pm = &rcar_thermal_pm_ops, + .pm = pm_sleep_ptr(&rcar_thermal_pm_ops), .of_match_table = rcar_thermal_dt_ids, }, .probe = rcar_thermal_probe, diff --git a/drivers/thunderbolt/nhi.c b/drivers/thunderbolt/nhi.c index 5f63f9b9cf40..addb4a20d5ea 100644 --- a/drivers/thunderbolt/nhi.c +++ b/drivers/thunderbolt/nhi.c @@ -1538,6 +1538,8 @@ static struct pci_device_id nhi_ids[] = { .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_PTL_P_NHI1), .driver_data = (kernel_ulong_t)&icl_nhi_ops }, + { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_WCL_NHI0), + .driver_data = (kernel_ulong_t)&icl_nhi_ops }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HOST_80G_NHI) }, { PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HOST_40G_NHI) }, diff --git a/drivers/thunderbolt/nhi.h b/drivers/thunderbolt/nhi.h index 16744f25a9a0..24ac4246d0ca 100644 --- a/drivers/thunderbolt/nhi.h +++ b/drivers/thunderbolt/nhi.h @@ -75,6 +75,7 @@ extern const struct tb_nhi_ops icl_nhi_ops; #define PCI_DEVICE_ID_INTEL_TITAN_RIDGE_DD_BRIDGE 0x15ef #define PCI_DEVICE_ID_INTEL_ADL_NHI0 0x463e #define PCI_DEVICE_ID_INTEL_ADL_NHI1 0x466d +#define PCI_DEVICE_ID_INTEL_WCL_NHI0 0x4d33 #define PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HOST_80G_NHI 0x5781 #define PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HOST_40G_NHI 0x5784 #define PCI_DEVICE_ID_INTEL_BARLOW_RIDGE_HUB_80G_BRIDGE 0x5786 diff --git a/drivers/tty/amiserial.c b/drivers/tty/amiserial.c index 5af46442a792..81eaca751541 100644 --- a/drivers/tty/amiserial.c +++ b/drivers/tty/amiserial.c @@ -438,7 +438,7 @@ static irqreturn_t ser_tx_int(int irq, void *dev_id) * --------------------------------------------------------------- */ -static int startup(struct tty_struct *tty, struct serial_state *info) +static int rs_startup(struct tty_struct *tty, struct serial_state *info) { struct tty_port *port = &info->tport; unsigned long flags; @@ -513,7 +513,7 @@ errout: * This routine will shutdown a serial port; interrupts are disabled, and * DTR is dropped if the hangup on close termio flag is on. */ -static void shutdown(struct tty_struct *tty, struct serial_state *info) +static void rs_shutdown(struct tty_struct *tty, struct serial_state *info) { unsigned long flags; @@ -975,7 +975,7 @@ check_and_exit: change_speed(tty, state, NULL); } } else - retval = startup(tty, state); + retval = rs_startup(tty, state); tty_unlock(tty); return retval; } @@ -1251,9 +1251,9 @@ static void rs_close(struct tty_struct *tty, struct file * filp) */ rs_wait_until_sent(tty, state->timeout); } - shutdown(tty, state); + rs_shutdown(tty, state); rs_flush_buffer(tty); - + tty_ldisc_flush(tty); port->tty = NULL; @@ -1325,7 +1325,7 @@ static void rs_hangup(struct tty_struct *tty) struct serial_state *info = tty->driver_data; rs_flush_buffer(tty); - shutdown(tty, info); + rs_shutdown(tty, info); info->tport.count = 0; tty_port_set_active(&info->tport, false); info->tport.tty = NULL; @@ -1349,7 +1349,7 @@ static int rs_open(struct tty_struct *tty, struct file * filp) port->tty = tty; tty->driver_data = info; - retval = startup(tty, info); + retval = rs_startup(tty, info); if (retval) { return retval; } diff --git a/drivers/tty/pty.c b/drivers/tty/pty.c index 8bb1a01fef2a..41c1d909525c 100644 --- a/drivers/tty/pty.c +++ b/drivers/tty/pty.c @@ -589,6 +589,23 @@ static inline void legacy_pty_init(void) { } #ifdef CONFIG_UNIX98_PTYS static struct cdev ptmx_cdev; +static struct file *ptm_open_peer_file(struct file *master, + struct tty_struct *tty, int flags) +{ + struct path path; + struct file *file; + + /* Compute the slave's path */ + path.mnt = devpts_mntget(master, tty->driver_data); + if (IS_ERR(path.mnt)) + return ERR_CAST(path.mnt); + path.dentry = tty->link->driver_data; + + file = dentry_open(&path, flags, current_cred()); + mntput(path.mnt); + return file; +} + /** * ptm_open_peer - open the peer of a pty * @master: the open struct file of the ptmx device node @@ -601,42 +618,10 @@ static struct cdev ptmx_cdev; */ int ptm_open_peer(struct file *master, struct tty_struct *tty, int flags) { - int fd; - struct file *filp; - int retval = -EINVAL; - struct path path; - if (tty->driver != ptm_driver) return -EIO; - fd = get_unused_fd_flags(flags); - if (fd < 0) { - retval = fd; - goto err; - } - - /* Compute the slave's path */ - path.mnt = devpts_mntget(master, tty->driver_data); - if (IS_ERR(path.mnt)) { - retval = PTR_ERR(path.mnt); - goto err_put; - } - path.dentry = tty->link->driver_data; - - filp = dentry_open(&path, flags, current_cred()); - mntput(path.mnt); - if (IS_ERR(filp)) { - retval = PTR_ERR(filp); - goto err_put; - } - - fd_install(fd, filp); - return fd; - -err_put: - put_unused_fd(fd); -err: - return retval; + return FD_ADD(flags, ptm_open_peer_file(master, tty, flags)); } static int pty_unix98_ioctl(struct tty_struct *tty, diff --git a/drivers/tty/serial/8250/8250.h b/drivers/tty/serial/8250/8250.h index 58e64c4e1e3a..e99f5193d8f1 100644 --- a/drivers/tty/serial/8250/8250.h +++ b/drivers/tty/serial/8250/8250.h @@ -322,13 +322,13 @@ static inline void serial8250_pnp_exit(void) { } #endif #ifdef CONFIG_SERIAL_8250_RSA -void univ8250_rsa_support(struct uart_ops *ops); +void univ8250_rsa_support(struct uart_ops *ops, const struct uart_ops *core_ops); void rsa_enable(struct uart_8250_port *up); void rsa_disable(struct uart_8250_port *up); void rsa_autoconfig(struct uart_8250_port *up); void rsa_reset(struct uart_8250_port *up); #else -static inline void univ8250_rsa_support(struct uart_ops *ops) { } +static inline void univ8250_rsa_support(struct uart_ops *ops, const struct uart_ops *core_ops) { } static inline void rsa_enable(struct uart_8250_port *up) {} static inline void rsa_disable(struct uart_8250_port *up) {} static inline void rsa_autoconfig(struct uart_8250_port *up) {} diff --git a/drivers/tty/serial/8250/8250_platform.c b/drivers/tty/serial/8250/8250_platform.c index b27981340e76..fe7ec440ffa5 100644 --- a/drivers/tty/serial/8250/8250_platform.c +++ b/drivers/tty/serial/8250/8250_platform.c @@ -75,7 +75,7 @@ static void __init __serial8250_isa_init_ports(void) /* chain base port ops to support Remote Supervisor Adapter */ univ8250_port_ops = *univ8250_port_base_ops; - univ8250_rsa_support(&univ8250_port_ops); + univ8250_rsa_support(&univ8250_port_ops, univ8250_port_base_ops); if (share_irqs) irqflag = IRQF_SHARED; diff --git a/drivers/tty/serial/8250/8250_rsa.c b/drivers/tty/serial/8250/8250_rsa.c index 40a3dbd9e452..1f182f165525 100644 --- a/drivers/tty/serial/8250/8250_rsa.c +++ b/drivers/tty/serial/8250/8250_rsa.c @@ -14,6 +14,8 @@ static unsigned long probe_rsa[PORT_RSA_MAX]; static unsigned int probe_rsa_count; +static const struct uart_ops *core_port_base_ops; + static int rsa8250_request_resource(struct uart_8250_port *up) { struct uart_port *port = &up->port; @@ -67,7 +69,7 @@ static void univ8250_config_port(struct uart_port *port, int flags) } } - univ8250_port_base_ops->config_port(port, flags); + core_port_base_ops->config_port(port, flags); if (port->type != PORT_RSA && up->probe & UART_PROBE_RSA) rsa8250_release_resource(up); @@ -78,11 +80,11 @@ static int univ8250_request_port(struct uart_port *port) struct uart_8250_port *up = up_to_u8250p(port); int ret; - ret = univ8250_port_base_ops->request_port(port); + ret = core_port_base_ops->request_port(port); if (ret == 0 && port->type == PORT_RSA) { ret = rsa8250_request_resource(up); if (ret < 0) - univ8250_port_base_ops->release_port(port); + core_port_base_ops->release_port(port); } return ret; @@ -94,15 +96,25 @@ static void univ8250_release_port(struct uart_port *port) if (port->type == PORT_RSA) rsa8250_release_resource(up); - univ8250_port_base_ops->release_port(port); + core_port_base_ops->release_port(port); } -void univ8250_rsa_support(struct uart_ops *ops) +/* + * It is not allowed to directly reference any symbols from 8250.ko here as + * that would result in a dependency loop between the 8250.ko and + * 8250_base.ko modules. This function is called from 8250.ko and is used to + * break the symbolic dependency cycle. Anything that is needed from 8250.ko + * has to be passed as pointers to this function which then can adjust those + * variables on 8250.ko side or store them locally as needed. + */ +void univ8250_rsa_support(struct uart_ops *ops, const struct uart_ops *core_ops) { + core_port_base_ops = core_ops; ops->config_port = univ8250_config_port; ops->request_port = univ8250_request_port; ops->release_port = univ8250_release_port; } +EXPORT_SYMBOL_FOR_MODULES(univ8250_rsa_support, "8250"); module_param_hw_array(probe_rsa, ulong, ioport, &probe_rsa_count, 0444); MODULE_PARM_DESC(probe_rsa, "Probe I/O ports for RSA"); @@ -146,7 +158,6 @@ void rsa_enable(struct uart_8250_port *up) if (up->port.uartclk == SERIAL_RSA_BAUD_BASE * 16) serial_out(up, UART_RSA_FRR, 0); } -EXPORT_SYMBOL_FOR_MODULES(rsa_enable, "8250_base"); /* * Attempts to turn off the RSA FIFO and resets the RSA board back to 115kbps compat mode. It is @@ -178,7 +189,6 @@ void rsa_disable(struct uart_8250_port *up) if (result) up->port.uartclk = SERIAL_RSA_BAUD_BASE_LO * 16; } -EXPORT_SYMBOL_FOR_MODULES(rsa_disable, "8250_base"); void rsa_autoconfig(struct uart_8250_port *up) { @@ -191,7 +201,6 @@ void rsa_autoconfig(struct uart_8250_port *up) if (__rsa_enable(up)) up->port.type = PORT_RSA; } -EXPORT_SYMBOL_FOR_MODULES(rsa_autoconfig, "8250_base"); void rsa_reset(struct uart_8250_port *up) { @@ -200,7 +209,6 @@ void rsa_reset(struct uart_8250_port *up) serial_out(up, UART_RSA_FRR, 0); } -EXPORT_SYMBOL_FOR_MODULES(rsa_reset, "8250_base"); #ifdef CONFIG_SERIAL_8250_DEPRECATED_OPTIONS #ifndef MODULE diff --git a/drivers/tty/serial/8250/Makefile b/drivers/tty/serial/8250/Makefile index 513a0941c284..9ec4d5fe64de 100644 --- a/drivers/tty/serial/8250/Makefile +++ b/drivers/tty/serial/8250/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_SERIAL_8250) += 8250.o 8250-y := 8250_core.o 8250-y += 8250_platform.o 8250-$(CONFIG_SERIAL_8250_PNP) += 8250_pnp.o -8250-$(CONFIG_SERIAL_8250_RSA) += 8250_rsa.o obj-$(CONFIG_SERIAL_8250) += 8250_base.o 8250_base-y := 8250_port.o @@ -15,6 +14,7 @@ obj-$(CONFIG_SERIAL_8250) += 8250_base.o 8250_base-$(CONFIG_SERIAL_8250_DWLIB) += 8250_dwlib.o 8250_base-$(CONFIG_SERIAL_8250_FINTEK) += 8250_fintek.o 8250_base-$(CONFIG_SERIAL_8250_PCILIB) += 8250_pcilib.o +8250_base-$(CONFIG_SERIAL_8250_RSA) += 8250_rsa.o obj-$(CONFIG_SERIAL_8250_CONSOLE) += 8250_early.o diff --git a/drivers/tty/serial/amba-pl011.c b/drivers/tty/serial/amba-pl011.c index 22939841b1de..7f17d288c807 100644 --- a/drivers/tty/serial/amba-pl011.c +++ b/drivers/tty/serial/amba-pl011.c @@ -628,7 +628,7 @@ static int pl011_dma_tx_refill(struct uart_amba_port *uap) dmatx->len = count; dmatx->dma = dma_map_single(dma_dev->dev, dmatx->buf, count, DMA_TO_DEVICE); - if (dmatx->dma == DMA_MAPPING_ERROR) { + if (dma_mapping_error(dma_dev->dev, dmatx->dma)) { uap->dmatx.queued = false; dev_dbg(uap->port.dev, "unable to map TX DMA\n"); return -EBUSY; diff --git a/drivers/tty/serial/icom.c b/drivers/tty/serial/icom.c index 7fb995a8490e..d00903cfa841 100644 --- a/drivers/tty/serial/icom.c +++ b/drivers/tty/serial/icom.c @@ -760,7 +760,7 @@ static void load_code(struct icom_port *icom_port) dma_free_coherent(&dev->dev, 4096, new_page, temp_pci); } -static int startup(struct icom_port *icom_port) +static int icom_startup(struct icom_port *icom_port) { unsigned long temp; unsigned char cable_id, raw_cable_id; @@ -832,7 +832,7 @@ unlock: return 0; } -static void shutdown(struct icom_port *icom_port) +static void icom_shutdown(struct icom_port *icom_port) { unsigned long temp; unsigned char cmdReg; @@ -1311,7 +1311,7 @@ static int icom_open(struct uart_port *port) int retval; kref_get(&icom_port->adapter->kref); - retval = startup(icom_port); + retval = icom_startup(icom_port); if (retval) { kref_put(&icom_port->adapter->kref, icom_kref_release); @@ -1333,7 +1333,7 @@ static void icom_close(struct uart_port *port) cmdReg = readb(&icom_port->dram->CmdReg); writeb(cmdReg & ~CMD_RCV_ENABLE, &icom_port->dram->CmdReg); - shutdown(icom_port); + icom_shutdown(icom_port); kref_put(&icom_port->adapter->kref, icom_kref_release); } diff --git a/drivers/tty/serial/kgdboc.c b/drivers/tty/serial/kgdboc.c index 85f6c5a76e0f..5a955c80a853 100644 --- a/drivers/tty/serial/kgdboc.c +++ b/drivers/tty/serial/kgdboc.c @@ -577,7 +577,6 @@ static int __init kgdboc_earlycon_init(char *opt) console_list_lock(); for_each_console(con) { if (con->write && con->read && - (con->flags & (CON_BOOT | CON_ENABLED)) && (!opt || !opt[0] || strcmp(con->name, opt) == 0)) break; } diff --git a/drivers/tty/synclink_gt.c b/drivers/tty/synclink_gt.c index 3865b10d2d43..9d591fb291fd 100644 --- a/drivers/tty/synclink_gt.c +++ b/drivers/tty/synclink_gt.c @@ -407,9 +407,9 @@ static void wr_reg32(struct slgt_info *info, unsigned int addr, __u32 value); static void msc_set_vcr(struct slgt_info *info); -static int startup(struct slgt_info *info); +static int startup_hw(struct slgt_info *info); static int block_til_ready(struct tty_struct *tty, struct file * filp,struct slgt_info *info); -static void shutdown(struct slgt_info *info); +static void shutdown_hw(struct slgt_info *info); static void program_hw(struct slgt_info *info); static void change_params(struct slgt_info *info); @@ -622,7 +622,7 @@ static int open(struct tty_struct *tty, struct file *filp) if (info->port.count == 1) { /* 1st open on this device, init hardware */ - retval = startup(info); + retval = startup_hw(info); if (retval < 0) { mutex_unlock(&info->port.mutex); goto cleanup; @@ -666,7 +666,7 @@ static void close(struct tty_struct *tty, struct file *filp) flush_buffer(tty); tty_ldisc_flush(tty); - shutdown(info); + shutdown_hw(info); mutex_unlock(&info->port.mutex); tty_port_close_end(&info->port, tty); @@ -687,7 +687,7 @@ static void hangup(struct tty_struct *tty) flush_buffer(tty); mutex_lock(&info->port.mutex); - shutdown(info); + shutdown_hw(info); spin_lock_irqsave(&info->port.lock, flags); info->port.count = 0; @@ -1445,7 +1445,7 @@ static int hdlcdev_open(struct net_device *dev) spin_unlock_irqrestore(&info->netlock, flags); /* claim resources and init adapter */ - if ((rc = startup(info)) != 0) { + if ((rc = startup_hw(info)) != 0) { spin_lock_irqsave(&info->netlock, flags); info->netcount=0; spin_unlock_irqrestore(&info->netlock, flags); @@ -1455,7 +1455,7 @@ static int hdlcdev_open(struct net_device *dev) /* generic HDLC layer open processing */ rc = hdlc_open(dev); if (rc) { - shutdown(info); + shutdown_hw(info); spin_lock_irqsave(&info->netlock, flags); info->netcount = 0; spin_unlock_irqrestore(&info->netlock, flags); @@ -1499,7 +1499,7 @@ static int hdlcdev_close(struct net_device *dev) netif_stop_queue(dev); /* shutdown adapter and release resources */ - shutdown(info); + shutdown_hw(info); hdlc_close(dev); @@ -2328,7 +2328,7 @@ static irqreturn_t slgt_interrupt(int dummy, void *dev_id) return IRQ_HANDLED; } -static int startup(struct slgt_info *info) +static int startup_hw(struct slgt_info *info) { DBGINFO(("%s startup\n", info->device_name)); @@ -2361,7 +2361,7 @@ static int startup(struct slgt_info *info) /* * called by close() and hangup() to shutdown hardware */ -static void shutdown(struct slgt_info *info) +static void shutdown_hw(struct slgt_info *info) { unsigned long flags; diff --git a/drivers/usb/cdns3/cdns3-pci-wrap.c b/drivers/usb/cdns3/cdns3-pci-wrap.c index 3b3b3dc75f35..57f57c24c663 100644 --- a/drivers/usb/cdns3/cdns3-pci-wrap.c +++ b/drivers/usb/cdns3/cdns3-pci-wrap.c @@ -98,10 +98,8 @@ static int cdns3_pci_probe(struct pci_dev *pdev, wrap = pci_get_drvdata(func); } else { wrap = kzalloc(sizeof(*wrap), GFP_KERNEL); - if (!wrap) { - pci_disable_device(pdev); + if (!wrap) return -ENOMEM; - } } res = wrap->dev_res; @@ -160,7 +158,6 @@ static int cdns3_pci_probe(struct pci_dev *pdev, /* register platform device */ wrap->plat_dev = platform_device_register_full(&plat_info); if (IS_ERR(wrap->plat_dev)) { - pci_disable_device(pdev); err = PTR_ERR(wrap->plat_dev); kfree(wrap); return err; diff --git a/drivers/usb/dwc3/core.c b/drivers/usb/dwc3/core.c index ae140c356295..c2ce2f5e60a1 100644 --- a/drivers/usb/dwc3/core.c +++ b/drivers/usb/dwc3/core.c @@ -25,6 +25,7 @@ #include <linux/of.h> #include <linux/of_graph.h> #include <linux/acpi.h> +#include <linux/pci.h> #include <linux/pinctrl/consumer.h> #include <linux/pinctrl/devinfo.h> #include <linux/reset.h> @@ -2241,7 +2242,7 @@ int dwc3_core_probe(const struct dwc3_probe_data *data) dev_set_drvdata(dev, dwc); dwc3_cache_hwparams(dwc); - if (!dwc->sysdev_is_parent && + if (!dev_is_pci(dwc->sysdev) && DWC3_GHWPARAMS0_AWIDTH(dwc->hwparams.hwparams0) == 64) { ret = dma_set_mask_and_coherent(dwc->sysdev, DMA_BIT_MASK(64)); if (ret) diff --git a/drivers/usb/dwc3/dwc3-pci.c b/drivers/usb/dwc3/dwc3-pci.c index 39c72cb52ce7..8f5faf632a8b 100644 --- a/drivers/usb/dwc3/dwc3-pci.c +++ b/drivers/usb/dwc3/dwc3-pci.c @@ -21,40 +21,41 @@ #include <linux/acpi.h> #include <linux/delay.h> +#define PCI_DEVICE_ID_INTEL_CMLLP 0x02ee +#define PCI_DEVICE_ID_INTEL_CMLH 0x06ee +#define PCI_DEVICE_ID_INTEL_BXT 0x0aaa #define PCI_DEVICE_ID_INTEL_BYT 0x0f37 #define PCI_DEVICE_ID_INTEL_MRFLD 0x119e -#define PCI_DEVICE_ID_INTEL_BSW 0x22b7 -#define PCI_DEVICE_ID_INTEL_SPTLP 0x9d30 -#define PCI_DEVICE_ID_INTEL_SPTH 0xa130 -#define PCI_DEVICE_ID_INTEL_BXT 0x0aaa #define PCI_DEVICE_ID_INTEL_BXT_M 0x1aaa -#define PCI_DEVICE_ID_INTEL_APL 0x5aaa -#define PCI_DEVICE_ID_INTEL_KBP 0xa2b0 -#define PCI_DEVICE_ID_INTEL_CMLLP 0x02ee -#define PCI_DEVICE_ID_INTEL_CMLH 0x06ee +#define PCI_DEVICE_ID_INTEL_BSW 0x22b7 #define PCI_DEVICE_ID_INTEL_GLK 0x31aa -#define PCI_DEVICE_ID_INTEL_CNPLP 0x9dee -#define PCI_DEVICE_ID_INTEL_CNPH 0xa36e -#define PCI_DEVICE_ID_INTEL_CNPV 0xa3b0 #define PCI_DEVICE_ID_INTEL_ICLLP 0x34ee -#define PCI_DEVICE_ID_INTEL_EHL 0x4b7e -#define PCI_DEVICE_ID_INTEL_TGPLP 0xa0ee #define PCI_DEVICE_ID_INTEL_TGPH 0x43ee -#define PCI_DEVICE_ID_INTEL_JSP 0x4dee -#define PCI_DEVICE_ID_INTEL_WCL 0x4d7e #define PCI_DEVICE_ID_INTEL_ADL 0x460e -#define PCI_DEVICE_ID_INTEL_ADL_PCH 0x51ee #define PCI_DEVICE_ID_INTEL_ADLN 0x465e +#define PCI_DEVICE_ID_INTEL_EHL 0x4b7e +#define PCI_DEVICE_ID_INTEL_WCL 0x4d7e +#define PCI_DEVICE_ID_INTEL_JSP 0x4dee +#define PCI_DEVICE_ID_INTEL_ADL_PCH 0x51ee #define PCI_DEVICE_ID_INTEL_ADLN_PCH 0x54ee -#define PCI_DEVICE_ID_INTEL_ADLS 0x7ae1 -#define PCI_DEVICE_ID_INTEL_RPL 0xa70e +#define PCI_DEVICE_ID_INTEL_APL 0x5aaa +#define PCI_DEVICE_ID_INTEL_NVLS_PCH 0x6e6f +#define PCI_DEVICE_ID_INTEL_ARLH_PCH 0x777e #define PCI_DEVICE_ID_INTEL_RPLS 0x7a61 +#define PCI_DEVICE_ID_INTEL_MTL 0x7e7e +#define PCI_DEVICE_ID_INTEL_ADLS 0x7ae1 #define PCI_DEVICE_ID_INTEL_MTLM 0x7eb1 #define PCI_DEVICE_ID_INTEL_MTLP 0x7ec1 #define PCI_DEVICE_ID_INTEL_MTLS 0x7f6f -#define PCI_DEVICE_ID_INTEL_MTL 0x7e7e -#define PCI_DEVICE_ID_INTEL_ARLH_PCH 0x777e #define PCI_DEVICE_ID_INTEL_TGL 0x9a15 +#define PCI_DEVICE_ID_INTEL_SPTLP 0x9d30 +#define PCI_DEVICE_ID_INTEL_CNPLP 0x9dee +#define PCI_DEVICE_ID_INTEL_TGPLP 0xa0ee +#define PCI_DEVICE_ID_INTEL_SPTH 0xa130 +#define PCI_DEVICE_ID_INTEL_KBP 0xa2b0 +#define PCI_DEVICE_ID_INTEL_CNPH 0xa36e +#define PCI_DEVICE_ID_INTEL_CNPV 0xa3b0 +#define PCI_DEVICE_ID_INTEL_RPL 0xa70e #define PCI_DEVICE_ID_INTEL_PTLH 0xe332 #define PCI_DEVICE_ID_INTEL_PTLH_PCH 0xe37e #define PCI_DEVICE_ID_INTEL_PTLU 0xe432 @@ -412,40 +413,41 @@ static void dwc3_pci_remove(struct pci_dev *pci) } static const struct pci_device_id dwc3_pci_id_table[] = { - { PCI_DEVICE_DATA(INTEL, BSW, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, BYT, &dwc3_pci_intel_byt_swnode) }, - { PCI_DEVICE_DATA(INTEL, MRFLD, &dwc3_pci_intel_mrfld_swnode) }, { PCI_DEVICE_DATA(INTEL, CMLLP, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, CMLH, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, SPTLP, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, SPTH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, BXT, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, BYT, &dwc3_pci_intel_byt_swnode) }, + { PCI_DEVICE_DATA(INTEL, MRFLD, &dwc3_pci_intel_mrfld_swnode) }, { PCI_DEVICE_DATA(INTEL, BXT_M, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, APL, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, KBP, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, BSW, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, GLK, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, CNPLP, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, CNPH, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, CNPV, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, ICLLP, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, EHL, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, TGPLP, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, TGPH, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, JSP, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, WCL, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, ADL, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, ADL_PCH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, ADLN, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, EHL, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, WCL, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, JSP, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, ADL_PCH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, ADLN_PCH, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, ADLS, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, RPL, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, APL, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, NVLS_PCH, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, ARLH_PCH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, RPLS, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, MTL, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, ADLS, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, MTLM, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, MTLP, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, MTL, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, MTLS, &dwc3_pci_intel_swnode) }, - { PCI_DEVICE_DATA(INTEL, ARLH_PCH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, TGL, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, SPTLP, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, CNPLP, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, TGPLP, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, SPTH, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, KBP, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, CNPH, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, CNPV, &dwc3_pci_intel_swnode) }, + { PCI_DEVICE_DATA(INTEL, RPL, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, PTLH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, PTLH_PCH, &dwc3_pci_intel_swnode) }, { PCI_DEVICE_DATA(INTEL, PTLU, &dwc3_pci_intel_swnode) }, diff --git a/drivers/usb/dwc3/ep0.c b/drivers/usb/dwc3/ep0.c index b4229aa13f37..e0bad5708664 100644 --- a/drivers/usb/dwc3/ep0.c +++ b/drivers/usb/dwc3/ep0.c @@ -94,6 +94,7 @@ static int __dwc3_gadget_ep0_queue(struct dwc3_ep *dep, req->request.actual = 0; req->request.status = -EINPROGRESS; req->epnum = dep->number; + req->status = DWC3_REQUEST_STATUS_QUEUED; list_add_tail(&req->list, &dep->pending_list); diff --git a/drivers/usb/dwc3/gadget.c b/drivers/usb/dwc3/gadget.c index 6f18b4840a25..5e4997f974dd 100644 --- a/drivers/usb/dwc3/gadget.c +++ b/drivers/usb/dwc3/gadget.c @@ -228,6 +228,13 @@ void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, { struct dwc3 *dwc = dep->dwc; + /* + * The request might have been processed and completed while the + * spinlock was released. Skip processing if already completed. + */ + if (req->status == DWC3_REQUEST_STATUS_COMPLETED) + return; + dwc3_gadget_del_and_unmap_request(dep, req, status); req->status = DWC3_REQUEST_STATUS_COMPLETED; diff --git a/drivers/usb/gadget/function/f_eem.c b/drivers/usb/gadget/function/f_eem.c index 6de81ea17274..edbbadad6138 100644 --- a/drivers/usb/gadget/function/f_eem.c +++ b/drivers/usb/gadget/function/f_eem.c @@ -477,8 +477,13 @@ static int eem_unwrap(struct gether *port, req->complete = eem_cmd_complete; req->zero = 1; req->context = ctx; - if (usb_ep_queue(port->in_ep, req, GFP_ATOMIC)) + if (usb_ep_queue(port->in_ep, req, GFP_ATOMIC)) { DBG(cdev, "echo response queue fail\n"); + kfree(ctx); + kfree(req->buf); + usb_ep_free_request(ep, req); + dev_kfree_skb_any(skb2); + } break; case 1: /* echo response */ diff --git a/drivers/usb/gadget/udc/core.c b/drivers/usb/gadget/udc/core.c index 694653761c44..8dbe79bdc0f9 100644 --- a/drivers/usb/gadget/udc/core.c +++ b/drivers/usb/gadget/udc/core.c @@ -1126,8 +1126,13 @@ static void usb_gadget_state_work(struct work_struct *work) void usb_gadget_set_state(struct usb_gadget *gadget, enum usb_device_state state) { + unsigned long flags; + + spin_lock_irqsave(&gadget->state_lock, flags); gadget->state = state; - schedule_work(&gadget->work); + if (!gadget->teardown) + schedule_work(&gadget->work); + spin_unlock_irqrestore(&gadget->state_lock, flags); trace_usb_gadget_set_state(gadget, 0); } EXPORT_SYMBOL_GPL(usb_gadget_set_state); @@ -1361,6 +1366,8 @@ static void usb_udc_nop_release(struct device *dev) void usb_initialize_gadget(struct device *parent, struct usb_gadget *gadget, void (*release)(struct device *dev)) { + spin_lock_init(&gadget->state_lock); + gadget->teardown = false; INIT_WORK(&gadget->work, usb_gadget_state_work); gadget->dev.parent = parent; @@ -1535,6 +1542,7 @@ EXPORT_SYMBOL_GPL(usb_add_gadget_udc); void usb_del_gadget(struct usb_gadget *gadget) { struct usb_udc *udc = gadget->udc; + unsigned long flags; if (!udc) return; @@ -1548,6 +1556,13 @@ void usb_del_gadget(struct usb_gadget *gadget) kobject_uevent(&udc->dev.kobj, KOBJ_REMOVE); sysfs_remove_link(&udc->dev.kobj, "gadget"); device_del(&gadget->dev); + /* + * Set the teardown flag before flushing the work to prevent new work + * from being scheduled while we are cleaning up. + */ + spin_lock_irqsave(&gadget->state_lock, flags); + gadget->teardown = true; + spin_unlock_irqrestore(&gadget->state_lock, flags); flush_work(&gadget->work); ida_free(&gadget_id_numbers, gadget->id_number); cancel_work_sync(&udc->vbus_work); diff --git a/drivers/usb/gadget/udc/renesas_usbf.c b/drivers/usb/gadget/udc/renesas_usbf.c index 14f4b2cf05a4..4c201574a0af 100644 --- a/drivers/usb/gadget/udc/renesas_usbf.c +++ b/drivers/usb/gadget/udc/renesas_usbf.c @@ -3262,7 +3262,9 @@ static int usbf_probe(struct platform_device *pdev) if (IS_ERR(udc->regs)) return PTR_ERR(udc->regs); - devm_pm_runtime_enable(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) return ret; diff --git a/drivers/usb/host/sl811-hcd.c b/drivers/usb/host/sl811-hcd.c index ea3cab99c5d4..5d6dba681e50 100644 --- a/drivers/usb/host/sl811-hcd.c +++ b/drivers/usb/host/sl811-hcd.c @@ -1748,6 +1748,7 @@ sl811h_suspend(struct platform_device *dev, pm_message_t state) break; case PM_EVENT_SUSPEND: case PM_EVENT_HIBERNATE: + case PM_EVENT_POWEROFF: case PM_EVENT_PRETHAW: /* explicitly discard hw state */ port_power(sl811, 0); break; diff --git a/drivers/usb/host/xhci-dbgcap.h b/drivers/usb/host/xhci-dbgcap.h index 47ac72c2286d..5426c971d2d3 100644 --- a/drivers/usb/host/xhci-dbgcap.h +++ b/drivers/usb/host/xhci-dbgcap.h @@ -114,6 +114,7 @@ struct dbc_port { unsigned int tx_boundary; bool registered; + bool tx_running; }; struct dbc_driver { diff --git a/drivers/usb/host/xhci-dbgtty.c b/drivers/usb/host/xhci-dbgtty.c index d894081d8d15..57cdda4e09c8 100644 --- a/drivers/usb/host/xhci-dbgtty.c +++ b/drivers/usb/host/xhci-dbgtty.c @@ -47,7 +47,7 @@ dbc_kfifo_to_req(struct dbc_port *port, char *packet) return len; } -static int dbc_start_tx(struct dbc_port *port) +static int dbc_do_start_tx(struct dbc_port *port) __releases(&port->port_lock) __acquires(&port->port_lock) { @@ -57,6 +57,8 @@ static int dbc_start_tx(struct dbc_port *port) bool do_tty_wake = false; struct list_head *pool = &port->write_pool; + port->tx_running = true; + while (!list_empty(pool)) { req = list_entry(pool->next, struct dbc_request, list_pool); len = dbc_kfifo_to_req(port, req->buf); @@ -77,12 +79,25 @@ static int dbc_start_tx(struct dbc_port *port) } } + port->tx_running = false; + if (do_tty_wake && port->port.tty) tty_wakeup(port->port.tty); return status; } +/* must be called with port->port_lock held */ +static int dbc_start_tx(struct dbc_port *port) +{ + lockdep_assert_held(&port->port_lock); + + if (port->tx_running) + return -EBUSY; + + return dbc_do_start_tx(port); +} + static void dbc_start_rx(struct dbc_port *port) __releases(&port->port_lock) __acquires(&port->port_lock) @@ -535,6 +550,12 @@ static void xhci_dbc_tty_unregister_device(struct xhci_dbc *dbc) if (!port->registered) return; + /* + * Hang up the TTY. This wakes up any blocked + * writers and causes subsequent writes to fail. + */ + tty_vhangup(port->port.tty); + tty_unregister_device(dbc_tty_driver, port->minor); xhci_dbc_tty_exit_port(port); port->registered = false; diff --git a/drivers/usb/host/xhci-ring.c b/drivers/usb/host/xhci-ring.c index 8e209aa33ea7..5bdcf9ab2b99 100644 --- a/drivers/usb/host/xhci-ring.c +++ b/drivers/usb/host/xhci-ring.c @@ -1985,6 +1985,7 @@ static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci) static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) { + struct xhci_virt_device *vdev = NULL; struct usb_hcd *hcd; u32 port_id; u32 portsc, cmd_reg; @@ -2016,6 +2017,9 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) goto cleanup; } + if (port->slot_id) + vdev = xhci->devs[port->slot_id]; + /* We might get interrupts after shared_hcd is removed */ if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n"); @@ -2038,10 +2042,11 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) usb_hcd_resume_root_hub(hcd); } - if (hcd->speed >= HCD_USB3 && - (portsc & PORT_PLS_MASK) == XDEV_INACTIVE) { - if (port->slot_id && xhci->devs[port->slot_id]) - xhci->devs[port->slot_id]->flags |= VDEV_PORT_ERROR; + if (vdev && (portsc & PORT_PLS_MASK) == XDEV_INACTIVE) { + if (!(portsc & PORT_RESET)) + vdev->flags |= VDEV_PORT_ERROR; + } else if (vdev && portsc & PORT_RC) { + vdev->flags &= ~VDEV_PORT_ERROR; } if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) { @@ -2099,7 +2104,7 @@ static void handle_port_status(struct xhci_hcd *xhci, union xhci_trb *event) * so the roothub behavior is consistent with external * USB 3.0 hub behavior. */ - if (port->slot_id && xhci->devs[port->slot_id]) + if (vdev) xhci_ring_device(xhci, port->slot_id); if (bus_state->port_remote_wakeup & (1 << hcd_portnum)) { xhci_test_and_clear_bit(xhci, port, PORT_PLC); diff --git a/drivers/usb/host/xhci-sideband.c b/drivers/usb/host/xhci-sideband.c index e771a476fef2..a85f62a73313 100644 --- a/drivers/usb/host/xhci-sideband.c +++ b/drivers/usb/host/xhci-sideband.c @@ -73,9 +73,12 @@ err: return NULL; } +/* Caller must hold sb->mutex */ static void __xhci_sideband_remove_endpoint(struct xhci_sideband *sb, struct xhci_virt_ep *ep) { + lockdep_assert_held(&sb->mutex); + /* * Issue a stop endpoint command when an endpoint is removed. * The stop ep cmd handler will handle the ring cleanup. @@ -86,6 +89,25 @@ __xhci_sideband_remove_endpoint(struct xhci_sideband *sb, struct xhci_virt_ep *e sb->eps[ep->ep_index] = NULL; } +/* Caller must hold sb->mutex */ +static void +__xhci_sideband_remove_interrupter(struct xhci_sideband *sb) +{ + struct usb_device *udev; + + lockdep_assert_held(&sb->mutex); + + if (!sb->ir) + return; + + xhci_remove_secondary_interrupter(xhci_to_hcd(sb->xhci), sb->ir); + sb->ir = NULL; + udev = sb->vdev->udev; + + if (udev->state != USB_STATE_NOTATTACHED) + usb_offload_put(udev); +} + /* sideband api functions */ /** @@ -131,14 +153,16 @@ xhci_sideband_add_endpoint(struct xhci_sideband *sb, struct xhci_virt_ep *ep; unsigned int ep_index; - mutex_lock(&sb->mutex); + guard(mutex)(&sb->mutex); + + if (!sb->vdev) + return -ENODEV; + ep_index = xhci_get_endpoint_index(&host_ep->desc); ep = &sb->vdev->eps[ep_index]; - if (ep->ep_state & EP_HAS_STREAMS) { - mutex_unlock(&sb->mutex); + if (ep->ep_state & EP_HAS_STREAMS) return -EINVAL; - } /* * Note, we don't know the DMA mask of the audio DSP device, if its @@ -148,14 +172,11 @@ xhci_sideband_add_endpoint(struct xhci_sideband *sb, * and let this function add the endpoint and allocate the ring buffer * with the smallest common DMA mask */ - if (sb->eps[ep_index] || ep->sideband) { - mutex_unlock(&sb->mutex); + if (sb->eps[ep_index] || ep->sideband) return -EBUSY; - } ep->sideband = sb; sb->eps[ep_index] = ep; - mutex_unlock(&sb->mutex); return 0; } @@ -180,18 +201,16 @@ xhci_sideband_remove_endpoint(struct xhci_sideband *sb, struct xhci_virt_ep *ep; unsigned int ep_index; - mutex_lock(&sb->mutex); + guard(mutex)(&sb->mutex); + ep_index = xhci_get_endpoint_index(&host_ep->desc); ep = sb->eps[ep_index]; - if (!ep || !ep->sideband || ep->sideband != sb) { - mutex_unlock(&sb->mutex); + if (!ep || !ep->sideband || ep->sideband != sb) return -ENODEV; - } __xhci_sideband_remove_endpoint(sb, ep); xhci_initialize_ring_info(ep->ring); - mutex_unlock(&sb->mutex); return 0; } @@ -316,28 +335,25 @@ xhci_sideband_create_interrupter(struct xhci_sideband *sb, int num_seg, if (!sb || !sb->xhci) return -ENODEV; - mutex_lock(&sb->mutex); - if (sb->ir) { - ret = -EBUSY; - goto out; - } + guard(mutex)(&sb->mutex); + + if (!sb->vdev) + return -ENODEV; + + if (sb->ir) + return -EBUSY; sb->ir = xhci_create_secondary_interrupter(xhci_to_hcd(sb->xhci), num_seg, imod_interval, intr_num); - if (!sb->ir) { - ret = -ENOMEM; - goto out; - } + if (!sb->ir) + return -ENOMEM; udev = sb->vdev->udev; ret = usb_offload_get(udev); sb->ir->ip_autoclear = ip_autoclear; -out: - mutex_unlock(&sb->mutex); - return ret; } EXPORT_SYMBOL_GPL(xhci_sideband_create_interrupter); @@ -352,21 +368,12 @@ EXPORT_SYMBOL_GPL(xhci_sideband_create_interrupter); void xhci_sideband_remove_interrupter(struct xhci_sideband *sb) { - struct usb_device *udev; - - if (!sb || !sb->ir) + if (!sb) return; - mutex_lock(&sb->mutex); - xhci_remove_secondary_interrupter(xhci_to_hcd(sb->xhci), sb->ir); - - sb->ir = NULL; - udev = sb->vdev->udev; + guard(mutex)(&sb->mutex); - if (udev->state != USB_STATE_NOTATTACHED) - usb_offload_put(udev); - - mutex_unlock(&sb->mutex); + __xhci_sideband_remove_interrupter(sb); } EXPORT_SYMBOL_GPL(xhci_sideband_remove_interrupter); @@ -465,6 +472,7 @@ EXPORT_SYMBOL_GPL(xhci_sideband_register); void xhci_sideband_unregister(struct xhci_sideband *sb) { + struct xhci_virt_device *vdev; struct xhci_hcd *xhci; int i; @@ -473,17 +481,23 @@ xhci_sideband_unregister(struct xhci_sideband *sb) xhci = sb->xhci; - mutex_lock(&sb->mutex); - for (i = 0; i < EP_CTX_PER_DEV; i++) - if (sb->eps[i]) - __xhci_sideband_remove_endpoint(sb, sb->eps[i]); - mutex_unlock(&sb->mutex); + scoped_guard(mutex, &sb->mutex) { + vdev = sb->vdev; + if (!vdev) + return; + + for (i = 0; i < EP_CTX_PER_DEV; i++) + if (sb->eps[i]) + __xhci_sideband_remove_endpoint(sb, sb->eps[i]); - xhci_sideband_remove_interrupter(sb); + __xhci_sideband_remove_interrupter(sb); + + sb->vdev = NULL; + } spin_lock_irq(&xhci->lock); sb->xhci = NULL; - sb->vdev->sideband = NULL; + vdev->sideband = NULL; spin_unlock_irq(&xhci->lock); kfree(sb); diff --git a/drivers/usb/host/xhci.c b/drivers/usb/host/xhci.c index 0cb45b95e4f5..a148a1280126 100644 --- a/drivers/usb/host/xhci.c +++ b/drivers/usb/host/xhci.c @@ -4007,6 +4007,7 @@ static int xhci_discover_or_reset_device(struct usb_hcd *hcd, xhci_get_slot_state(xhci, virt_dev->out_ctx)); xhci_dbg(xhci, "Not freeing device rings.\n"); /* Don't treat this as an error. May change my mind later. */ + virt_dev->flags = 0; ret = 0; goto command_cleanup; case COMP_SUCCESS: diff --git a/drivers/usb/renesas_usbhs/common.c b/drivers/usb/renesas_usbhs/common.c index 8f536f2c500f..dc2fec9168b7 100644 --- a/drivers/usb/renesas_usbhs/common.c +++ b/drivers/usb/renesas_usbhs/common.c @@ -813,18 +813,18 @@ static void usbhs_remove(struct platform_device *pdev) flush_delayed_work(&priv->notify_hotplug_work); - /* power off */ - if (!usbhs_get_dparam(priv, runtime_pwctrl)) - usbhsc_power_ctrl(priv, 0); - - pm_runtime_disable(&pdev->dev); - usbhs_platform_call(priv, hardware_exit, pdev); - usbhsc_clk_put(priv); reset_control_assert(priv->rsts); usbhs_mod_remove(priv); usbhs_fifo_remove(priv); usbhs_pipe_remove(priv); + + /* power off */ + if (!usbhs_get_dparam(priv, runtime_pwctrl)) + usbhsc_power_ctrl(priv, 0); + + usbhsc_clk_put(priv); + pm_runtime_disable(&pdev->dev); } static int usbhsc_suspend(struct device *dev) diff --git a/drivers/usb/serial/ftdi_sio.c b/drivers/usb/serial/ftdi_sio.c index 49666c33b41f..b37fa31f5694 100644 --- a/drivers/usb/serial/ftdi_sio.c +++ b/drivers/usb/serial/ftdi_sio.c @@ -1074,6 +1074,7 @@ static const struct usb_device_id id_table_combined[] = { /* U-Blox devices */ { USB_DEVICE(UBLOX_VID, UBLOX_C099F9P_ZED_PID) }, { USB_DEVICE(UBLOX_VID, UBLOX_C099F9P_ODIN_PID) }, + { USB_DEVICE_INTERFACE_NUMBER(UBLOX_VID, UBLOX_EVK_M101_PID, 2) }, /* FreeCalypso USB adapters */ { USB_DEVICE(FTDI_VID, FTDI_FALCONIA_JTAG_BUF_PID), .driver_info = (kernel_ulong_t)&ftdi_jtag_quirk }, diff --git a/drivers/usb/serial/ftdi_sio_ids.h b/drivers/usb/serial/ftdi_sio_ids.h index 4cc1fae8acb9..2539b9e2f712 100644 --- a/drivers/usb/serial/ftdi_sio_ids.h +++ b/drivers/usb/serial/ftdi_sio_ids.h @@ -1614,6 +1614,7 @@ #define UBLOX_VID 0x1546 #define UBLOX_C099F9P_ZED_PID 0x0502 #define UBLOX_C099F9P_ODIN_PID 0x0503 +#define UBLOX_EVK_M101_PID 0x0506 /* * GMC devices diff --git a/drivers/usb/serial/option.c b/drivers/usb/serial/option.c index 5de856f65f0d..e9400727ad36 100644 --- a/drivers/usb/serial/option.c +++ b/drivers/usb/serial/option.c @@ -2424,12 +2424,18 @@ static const struct usb_device_id option_ids[] = { { USB_DEVICE_INTERFACE_CLASS(0x305a, 0x1406, 0xff) }, /* GosunCn GM500 ECM/NCM */ { USB_DEVICE(0x33f8, 0x0104), /* Rolling RW101-GL (laptop RMNET) */ .driver_info = RSVD(4) | RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0115, 0xff), /* Rolling RW135-GL (laptop MBIM) */ + .driver_info = RSVD(5) }, { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x01a2, 0xff) }, /* Rolling RW101-GL (laptop MBIM) */ { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x01a3, 0xff) }, /* Rolling RW101-GL (laptop MBIM) */ { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x01a4, 0xff), /* Rolling RW101-GL (laptop MBIM) */ .driver_info = RSVD(4) }, - { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0115, 0xff), /* Rolling RW135-GL (laptop MBIM) */ - .driver_info = RSVD(5) }, + { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x01a8, 0xff), /* Rolling RW101R-GL (laptop MBIM) */ + .driver_info = RSVD(4) }, + { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x01a9, 0xff), /* Rolling RW101R-GL (laptop MBIM) */ + .driver_info = RSVD(4) }, + { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0301, 0xff) }, /* Rolling RW101R-GL (laptop MBIM) */ + { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0302, 0xff) }, /* Rolling RW101R-GL (laptop MBIM) */ { USB_DEVICE_INTERFACE_CLASS(0x33f8, 0x0802, 0xff), /* Rolling RW350-GL (laptop MBIM) */ .driver_info = RSVD(5) }, { USB_DEVICE_AND_INTERFACE_INFO(0x3731, 0x0100, 0xff, 0xff, 0x30) }, /* NetPrisma LCUK54-WWD for Global */ diff --git a/drivers/usb/storage/sddr55.c b/drivers/usb/storage/sddr55.c index b323f0a36260..9d813727e65f 100644 --- a/drivers/usb/storage/sddr55.c +++ b/drivers/usb/storage/sddr55.c @@ -469,6 +469,12 @@ static int sddr55_write_data(struct us_data *us, new_pba = (status[3] + (status[4] << 8) + (status[5] << 16)) >> info->blockshift; + /* check if device-reported new_pba is out of range */ + if (new_pba >= (info->capacity >> (info->blockshift + info->pageshift))) { + result = USB_STOR_TRANSPORT_FAILED; + goto leave; + } + /* check status for error */ if (status[0] == 0xff && status[1] == 0x4) { info->pba_to_lba[new_pba] = BAD_BLOCK; diff --git a/drivers/usb/storage/transport.c b/drivers/usb/storage/transport.c index 1aa1bd26c81f..9a4bf86e7b6a 100644 --- a/drivers/usb/storage/transport.c +++ b/drivers/usb/storage/transport.c @@ -1200,7 +1200,23 @@ int usb_stor_Bulk_transport(struct scsi_cmnd *srb, struct us_data *us) US_BULK_CS_WRAP_LEN && bcs->Signature == cpu_to_le32(US_BULK_CS_SIGN)) { + unsigned char buf[US_BULK_CS_WRAP_LEN]; + usb_stor_dbg(us, "Device skipped data phase\n"); + + /* + * Devices skipping data phase might leave CSW data in srb's + * transfer buffer. Zero it to prevent USB protocol leakage. + */ + sg = NULL; + offset = 0; + memset(buf, 0, sizeof(buf)); + if (usb_stor_access_xfer_buf(buf, + US_BULK_CS_WRAP_LEN, srb, &sg, + &offset, TO_XFER_BUF) != + US_BULK_CS_WRAP_LEN) + usb_stor_dbg(us, "Failed to clear CSW data\n"); + scsi_set_resid(srb, transfer_length); goto skipped_data_phase; } diff --git a/drivers/usb/storage/uas.c b/drivers/usb/storage/uas.c index 4ed0dc19afe0..45b01df364f7 100644 --- a/drivers/usb/storage/uas.c +++ b/drivers/usb/storage/uas.c @@ -698,6 +698,10 @@ static int uas_queuecommand_lck(struct scsi_cmnd *cmnd) * of queueing, no matter how fatal the error */ if (err == -ENODEV) { + if (cmdinfo->state & (COMMAND_INFLIGHT | DATA_IN_URB_INFLIGHT | + DATA_OUT_URB_INFLIGHT)) + goto out; + set_host_byte(cmnd, DID_NO_CONNECT); scsi_done(cmnd); goto zombie; @@ -711,6 +715,7 @@ static int uas_queuecommand_lck(struct scsi_cmnd *cmnd) uas_add_work(cmnd); } +out: devinfo->cmnd[idx] = cmnd; zombie: spin_unlock_irqrestore(&devinfo->lock, flags); diff --git a/drivers/usb/storage/unusual_devs.h b/drivers/usb/storage/unusual_devs.h index dfa5276a5a43..47f50d7a385c 100644 --- a/drivers/usb/storage/unusual_devs.h +++ b/drivers/usb/storage/unusual_devs.h @@ -938,7 +938,7 @@ UNUSUAL_DEV( 0x05e3, 0x0723, 0x9451, 0x9451, UNUSUAL_DEV( 0x0603, 0x8611, 0x0000, 0xffff, "Novatek", "NTK96550-based camera", - USB_SC_SCSI, USB_PR_BULK, NULL, + USB_SC_DEVICE, USB_PR_DEVICE, NULL, US_FL_BULK_IGNORE_TAG ), /* diff --git a/drivers/usb/typec/ucsi/psy.c b/drivers/usb/typec/ucsi/psy.c index 62a9d68bb66d..8ae900c8c132 100644 --- a/drivers/usb/typec/ucsi/psy.c +++ b/drivers/usb/typec/ucsi/psy.c @@ -145,6 +145,11 @@ static int ucsi_psy_get_current_max(struct ucsi_connector *con, { u32 pdo; + if (!UCSI_CONSTAT(con, CONNECTED)) { + val->intval = 0; + return 0; + } + switch (UCSI_CONSTAT(con, PWR_OPMODE)) { case UCSI_CONSTAT_PWR_OPMODE_PD: if (con->num_pdos > 0) { diff --git a/drivers/vfio/group.c b/drivers/vfio/group.c index c376a6279de0..d47ffada6912 100644 --- a/drivers/vfio/group.c +++ b/drivers/vfio/group.c @@ -299,10 +299,8 @@ static int vfio_group_ioctl_get_device_fd(struct vfio_group *group, char __user *arg) { struct vfio_device *device; - struct file *filep; char *buf; - int fdno; - int ret; + int fd; buf = strndup_user(arg, PAGE_SIZE); if (IS_ERR(buf)) @@ -313,26 +311,10 @@ static int vfio_group_ioctl_get_device_fd(struct vfio_group *group, if (IS_ERR(device)) return PTR_ERR(device); - fdno = get_unused_fd_flags(O_CLOEXEC); - if (fdno < 0) { - ret = fdno; - goto err_put_device; - } - - filep = vfio_device_open_file(device); - if (IS_ERR(filep)) { - ret = PTR_ERR(filep); - goto err_put_fdno; - } - - fd_install(fdno, filep); - return fdno; - -err_put_fdno: - put_unused_fd(fdno); -err_put_device: - vfio_device_put_registration(device); - return ret; + fd = FD_ADD(O_CLOEXEC, vfio_device_open_file(device)); + if (fd < 0) + vfio_device_put_registration(device); + return fd; } static int vfio_group_ioctl_get_status(struct vfio_group *group, diff --git a/drivers/video/fbdev/core/fbcon.c b/drivers/video/fbdev/core/fbcon.c index 9bd3c3814b5c..e7e07eb2142e 100644 --- a/drivers/video/fbdev/core/fbcon.c +++ b/drivers/video/fbdev/core/fbcon.c @@ -66,6 +66,7 @@ #include <linux/string.h> #include <linux/kd.h> #include <linux/panic.h> +#include <linux/pci.h> #include <linux/printk.h> #include <linux/slab.h> #include <linux/fb.h> @@ -78,6 +79,7 @@ #include <linux/interrupt.h> #include <linux/crc32.h> /* For counting font checksums */ #include <linux/uaccess.h> +#include <linux/vga_switcheroo.h> #include <asm/irq.h> #include "fbcon.h" @@ -2899,6 +2901,9 @@ void fbcon_fb_unregistered(struct fb_info *info) console_lock(); + if (info->device && dev_is_pci(info->device)) + vga_switcheroo_client_fb_set(to_pci_dev(info->device), NULL); + fbcon_registered_fb[info->node] = NULL; fbcon_num_registered_fb--; @@ -3032,6 +3037,10 @@ static int do_fb_registered(struct fb_info *info) } } + /* Set the fb info for vga_switcheroo clients. Does nothing otherwise. */ + if (info->device && dev_is_pci(info->device)) + vga_switcheroo_client_fb_set(to_pci_dev(info->device), info); + return ret; } diff --git a/drivers/watchdog/diag288_wdt.c b/drivers/watchdog/diag288_wdt.c index 887d5a6c155b..9daed2758ae5 100644 --- a/drivers/watchdog/diag288_wdt.c +++ b/drivers/watchdog/diag288_wdt.c @@ -18,8 +18,7 @@ * */ -#define KMSG_COMPONENT "diag288_wdt" -#define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#define pr_fmt(fmt) "diag288_wdt: " fmt #include <linux/init.h> #include <linux/kernel.h> diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c index 528682bf0c7f..f794014814be 100644 --- a/drivers/xen/xenbus/xenbus_xs.c +++ b/drivers/xen/xenbus/xenbus_xs.c @@ -410,7 +410,7 @@ static char *join(const char *dir, const char *name) return (!buffer) ? ERR_PTR(-ENOMEM) : buffer; } -static char **split(char *strings, unsigned int len, unsigned int *num) +static char **split_strings(char *strings, unsigned int len, unsigned int *num) { char *p, **ret; @@ -448,7 +448,7 @@ char **xenbus_directory(struct xenbus_transaction t, if (IS_ERR(strings)) return ERR_CAST(strings); - return split(strings, len, num); + return split_strings(strings, len, num); } EXPORT_SYMBOL_GPL(xenbus_directory); |