diff options
720 files changed, 24311 insertions, 4693 deletions
@@ -303,6 +303,7 @@ Hans de Goede <hansg@kernel.org> <hdegoede@redhat.com> Hans Verkuil <hverkuil@kernel.org> <hverkuil@xs4all.nl> Hans Verkuil <hverkuil@kernel.org> <hverkuil-cisco@xs4all.nl> Hans Verkuil <hverkuil@kernel.org> <hansverk@cisco.com> +Hao Ge <hao.ge@linux.dev> <gehao@kylinos.cn> Harry Yoo <harry.yoo@oracle.com> <42.hyeyoo@gmail.com> Heiko Carstens <hca@linux.ibm.com> <h.carstens@de.ibm.com> Heiko Carstens <hca@linux.ibm.com> <heiko.carstens@de.ibm.com> @@ -503,9 +504,7 @@ Mark Brown <broonie@sirena.org.uk> Mark Starovoytov <mstarovo@pm.me> <mstarovoitov@marvell.com> Markus Schneider-Pargmann <msp@baylibre.com> <mpa@pengutronix.de> Mark Yao <markyao0591@gmail.com> <mark.yao@rock-chips.com> -Martin Kepplinger <martink@posteo.de> <martin.kepplinger@ginzinger.com> -Martin Kepplinger <martink@posteo.de> <martin.kepplinger@puri.sm> -Martin Kepplinger <martink@posteo.de> <martin.kepplinger@theobroma-systems.com> +Martin Kepplinger-Novakovic <martink@posteo.de> <martin.kepplinger-novakovic@ginzinger.com> Martyna Szapar-Mudlaw <martyna.szapar-mudlaw@linux.intel.com> <martyna.szapar-mudlaw@intel.com> Mathieu Othacehe <othacehe@gnu.org> <m.othacehe@gmail.com> Mat Martineau <martineau@kernel.org> <mathew.j.martineau@linux.intel.com> @@ -856,6 +855,9 @@ Vivien Didelot <vivien.didelot@gmail.com> <vivien.didelot@savoirfairelinux.com> Vlad Dogaru <ddvlad@gmail.com> <vlad.dogaru@intel.com> Vladimir Davydov <vdavydov.dev@gmail.com> <vdavydov@parallels.com> Vladimir Davydov <vdavydov.dev@gmail.com> <vdavydov@virtuozzo.com> +WangYuli <wangyuli@aosc.io> <wangyl5933@chinaunicom.cn> +WangYuli <wangyuli@aosc.io> <wangyuli@deepin.org> +WangYuli <wangyuli@aosc.io> <wangyuli@uniontech.com> Weiwen Hu <huweiwen@linux.alibaba.com> <sehuww@mail.scut.edu.cn> WeiXiong Liao <gmpy.liaowx@gmail.com> <liaoweixiong@allwinnertech.com> Wen Gong <quic_wgong@quicinc.com> <wgong@codeaurora.org> @@ -867,6 +869,7 @@ Yakir Yang <kuankuan.y@gmail.com> <ykk@rock-chips.com> Yanteng Si <si.yanteng@linux.dev> <siyanteng@loongson.cn> Ying Huang <huang.ying.caritas@gmail.com> <ying.huang@intel.com> Yosry Ahmed <yosry.ahmed@linux.dev> <yosryahmed@google.com> +Yu-Chun Lin <eleanor.lin@realtek.com> <eleanor15x@gmail.com> Yusuke Goda <goda.yusuke@renesas.com> Zack Rusin <zack.rusin@broadcom.com> <zackr@vmware.com> Zhu Yanjun <zyjzyj2000@gmail.com> <yanjunz@nvidia.com> @@ -2056,16 +2056,15 @@ S: Korte Heul 95 S: 1403 ND BUSSUM S: The Netherlands -N: Martin Kepplinger +N: Martin Kepplinger-Novakovic E: martink@posteo.de -E: martin.kepplinger@puri.sm -W: http://www.martinkepplinger.com P: 4096R/5AB387D3 F208 2B88 0F9E 4239 3468 6E3F 5003 98DF 5AB3 87D3 D: mma8452 accelerators iio driver D: pegasus_notetaker input driver +D: imx8m media and hi846 sensor driver D: Kernel fixes and cleanups -S: Garnisonstraße 26 -S: 4020 Linz +S: Keplerstr. 6 +S: 4050 Traun S: Austria N: Karl Keyte diff --git a/Documentation/ABI/obsolete/sysfs-kernel-kexec-kdump b/Documentation/ABI/obsolete/sysfs-kernel-kexec-kdump new file mode 100644 index 000000000000..ba26a6a1d2be --- /dev/null +++ b/Documentation/ABI/obsolete/sysfs-kernel-kexec-kdump @@ -0,0 +1,71 @@ +NOTE: all the ABIs listed in this file are deprecated and will be removed after 2028. + +Here are the alternative ABIs: ++------------------------------------+-----------------------------------------+ +| Deprecated | Alternative | ++------------------------------------+-----------------------------------------+ +| /sys/kernel/kexec_loaded | /sys/kernel/kexec/loaded | ++------------------------------------+-----------------------------------------+ +| /sys/kernel/kexec_crash_loaded | /sys/kernel/kexec/crash_loaded | ++------------------------------------+-----------------------------------------+ +| /sys/kernel/kexec_crash_size | /sys/kernel/kexec/crash_size | ++------------------------------------+-----------------------------------------+ +| /sys/kernel/crash_elfcorehdr_size | /sys/kernel/kexec/crash_elfcorehdr_size | ++------------------------------------+-----------------------------------------+ +| /sys/kernel/kexec_crash_cma_ranges | /sys/kernel/kexec/crash_cma_ranges | ++------------------------------------+-----------------------------------------+ + + +What: /sys/kernel/kexec_loaded +Date: Jun 2006 +Contact: kexec@lists.infradead.org +Description: read only + Indicates whether a new kernel image has been loaded + into memory using the kexec system call. It shows 1 if + a kexec image is present and ready to boot, or 0 if none + is loaded. +User: kexec tools, kdump service + +What: /sys/kernel/kexec_crash_loaded +Date: Jun 2006 +Contact: kexec@lists.infradead.org +Description: read only + Indicates whether a crash (kdump) kernel is currently + loaded into memory. It shows 1 if a crash kernel has been + successfully loaded for panic handling, or 0 if no crash + kernel is present. +User: Kexec tools, Kdump service + +What: /sys/kernel/kexec_crash_size +Date: Dec 2009 +Contact: kexec@lists.infradead.org +Description: read/write + Shows the amount of memory reserved for loading the crash + (kdump) kernel. It reports the size, in bytes, of the + crash kernel area defined by the crashkernel= parameter. + This interface also allows reducing the crashkernel + reservation by writing a smaller value, and the reclaimed + space is added back to the system RAM. +User: Kdump service + +What: /sys/kernel/crash_elfcorehdr_size +Date: Aug 2023 +Contact: kexec@lists.infradead.org +Description: read only + Indicates the preferred size of the memory buffer for the + ELF core header used by the crash (kdump) kernel. It defines + how much space is needed to hold metadata about the crashed + system, including CPU and memory information. This information + is used by the user space utility kexec to support updating the + in-kernel kdump image during hotplug operations. +User: Kexec tools + +What: /sys/kernel/kexec_crash_cma_ranges +Date: Nov 2025 +Contact: kexec@lists.infradead.org +Description: read only + Provides information about the memory ranges reserved from + the Contiguous Memory Allocator (CMA) area that are allocated + to the crash (kdump) kernel. It lists the start and end physical + addresses of CMA regions assigned for crashkernel use. +User: kdump service diff --git a/Documentation/ABI/testing/sysfs-bus-pci b/Documentation/ABI/testing/sysfs-bus-pci index 92debe879ffb..b767db2c52cb 100644 --- a/Documentation/ABI/testing/sysfs-bus-pci +++ b/Documentation/ABI/testing/sysfs-bus-pci @@ -621,3 +621,84 @@ Description: number extended capability. The file is read only and due to the possible sensitivity of accessible serial numbers, admin only. + +What: /sys/bus/pci/devices/.../tsm/ +Contact: linux-coco@lists.linux.dev +Description: + This directory only appears if a physical device function + supports authentication (PCIe CMA-SPDM), interface security + (PCIe TDISP), and is accepted for secure operation by the + platform TSM driver. This attribute directory appears + dynamically after the platform TSM driver loads. So, only after + the /sys/class/tsm/tsm0 device arrives can tools assume that + devices without a tsm/ attribute directory will never have one; + before that, the security capabilities of the device relative to + the platform TSM are unknown. See + Documentation/ABI/testing/sysfs-class-tsm. + +What: /sys/bus/pci/devices/.../tsm/connect +Contact: linux-coco@lists.linux.dev +Description: + (RW) Write the name of a TSM (TEE Security Manager) device from + /sys/class/tsm to this file to establish a connection with the + device. This typically includes an SPDM (DMTF Security + Protocols and Data Models) session over PCIe DOE (Data Object + Exchange) and may also include PCIe IDE (Integrity and Data + Encryption) establishment. Reads from this attribute return the + name of the connected TSM or the empty string if not + connected. A TSM device signals its readiness to accept PCI + connection via a KOBJ_CHANGE event. + +What: /sys/bus/pci/devices/.../tsm/disconnect +Contact: linux-coco@lists.linux.dev +Description: + (WO) Write the name of the TSM device that was specified + to 'connect' to teardown the connection. + +What: /sys/bus/pci/devices/.../tsm/dsm +Contact: linux-coco@lists.linux.dev +Description: (RO) Return PCI device name of this device's DSM (Device + Security Manager). When a device is in the connected state it + indicates that the platform TSM (TEE Security Manager) has made + a secure-session connection with a device's DSM. A DSM is always + physical function 0 and when the device supports TDISP (TEE + Device Interface Security Protocol) its managed functions also + populate this tsm/dsm attribute. The managed functions of a DSM + are SR-IOV (Single Root I/O Virtualization) virtual functions, + non-zero functions of a multi-function device, or downstream + endpoints depending on whether the DSM is an SR-IOV physical + function, function0 of a multi-function device, or an upstream + PCIe switch port. This is a "link" TSM attribute, see + Documentation/ABI/testing/sysfs-class-tsm. + +What: /sys/bus/pci/devices/.../tsm/bound +Contact: linux-coco@lists.linux.dev +Description: (RO) Return the device name of the TSM when the device is in a + TDISP (TEE Device Interface Security Protocol) operational state + (LOCKED, RUN, or ERROR, not UNLOCKED). Bound devices consume + platform TSM resources and depend on the device's configuration + (e.g. BME (Bus Master Enable) and MSE (Memory Space Enable) + among other settings) to remain stable for the duration of the + bound state. This attribute is only visible for devices that + support TDISP operation, and it is only populated after + successful connect and TSM bind. The TSM bind operation is + initiated by VFIO/IOMMUFD. This is a "link" TSM attribute, see + Documentation/ABI/testing/sysfs-class-tsm. + +What: /sys/bus/pci/devices/.../authenticated +Contact: linux-pci@vger.kernel.org +Description: + When the device's tsm/ directory is present device + authentication (PCIe CMA-SPDM) and link encryption (PCIe IDE) + are handled by the platform TSM (TEE Security Manager). When the + tsm/ directory is not present this attribute reflects only the + native CMA-SPDM authentication state with the kernel's + certificate store. + + If the attribute is not present, it indicates that + authentication is unsupported by the device, or the TSM has no + available authentication methods for the device. + + When present and the tsm/ attribute directory is present, the + authenticated attribute is an alias for the device 'connect' + state. See the 'tsm/connect' attribute for more details. diff --git a/Documentation/ABI/testing/sysfs-class-tsm b/Documentation/ABI/testing/sysfs-class-tsm new file mode 100644 index 000000000000..6fc1a5ac6da1 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-class-tsm @@ -0,0 +1,19 @@ +What: /sys/class/tsm/tsmN +Contact: linux-coco@lists.linux.dev +Description: + "tsmN" is a device that represents the generic attributes of a + platform TEE Security Manager. It is typically a child of a + platform enumerated TSM device. /sys/class/tsm/tsmN/uevent + signals when the PCI layer is able to support establishment of + link encryption and other device-security features coordinated + through a platform tsm. + +What: /sys/class/tsm/tsmN/streamH.R.E +Contact: linux-pci@vger.kernel.org +Description: + (RO) When a host bridge has established a secure connection via + the platform TSM, symlink appears. The primary function of this + is have a system global review of TSM resource consumption + across host bridges. The link points to the endpoint PCI device + and matches the same link published by the host bridge. See + Documentation/ABI/testing/sysfs-devices-pci-host-bridge. diff --git a/Documentation/ABI/testing/sysfs-devices-pci-host-bridge b/Documentation/ABI/testing/sysfs-devices-pci-host-bridge new file mode 100644 index 000000000000..b91ec3450811 --- /dev/null +++ b/Documentation/ABI/testing/sysfs-devices-pci-host-bridge @@ -0,0 +1,45 @@ +What: /sys/devices/pciDDDD:BB + /sys/devices/.../pciDDDD:BB +Contact: linux-pci@vger.kernel.org +Description: + A PCI host bridge device parents a PCI bus device topology. PCI + controllers may also parent host bridges. The DDDD:BB format + conveys the PCI domain (ACPI segment) number and root bus number + (in hexadecimal) of the host bridge. Note that the domain number + may be larger than the 16-bits that the "DDDD" format implies + for emulated host-bridges. + +What: pciDDDD:BB/firmware_node +Contact: linux-pci@vger.kernel.org +Description: + (RO) Symlink to the platform firmware device object "companion" + of the host bridge. For example, an ACPI device with an _HID of + PNP0A08 (/sys/devices/LNXSYSTM:00/LNXSYBUS:00/PNP0A08:00). See + /sys/devices/pciDDDD:BB entry for details about the DDDD:BB + format. + +What: pciDDDD:BB/streamH.R.E +Contact: linux-pci@vger.kernel.org +Description: + (RO) When a platform has established a secure connection, PCIe + IDE, between two Partner Ports, this symlink appears. A stream + consumes a Stream ID slot in each of the Host bridge (H), Root + Port (R) and Endpoint (E). The link points to the Endpoint PCI + device in the Selective IDE Stream pairing. Specifically, "R" + and "E" represent the assigned Selective IDE Stream Register + Block in the Root Port and Endpoint, and "H" represents a + platform specific pool of stream resources shared by the Root + Ports in a host bridge. See /sys/devices/pciDDDD:BB entry for + details about the DDDD:BB format. + +What: pciDDDD:BB/available_secure_streams +Contact: linux-pci@vger.kernel.org +Description: + (RO) When a host bridge has Root Ports that support PCIe IDE + (link encryption and integrity protection) there may be a + limited number of Selective IDE Streams that can be used for + establishing new end-to-end secure links. This attribute + decrements upon secure link setup, and increments upon secure + link teardown. The in-use stream count is determined by counting + stream symlinks. See /sys/devices/pciDDDD:BB entry for details + about the DDDD:BB format. diff --git a/Documentation/ABI/testing/sysfs-kernel-kexec-kdump b/Documentation/ABI/testing/sysfs-kernel-kexec-kdump new file mode 100644 index 000000000000..f59051b5d96d --- /dev/null +++ b/Documentation/ABI/testing/sysfs-kernel-kexec-kdump @@ -0,0 +1,61 @@ +What: /sys/kernel/kexec/* +Date: Nov 2025 +Contact: kexec@lists.infradead.org +Description: + The /sys/kernel/kexec/* directory contains sysfs files + that provide information about the configuration status + of kexec and kdump. + +What: /sys/kernel/kexec/loaded +Date: Nov 2025 +Contact: kexec@lists.infradead.org +Description: read only + Indicates whether a new kernel image has been loaded + into memory using the kexec system call. It shows 1 if + a kexec image is present and ready to boot, or 0 if none + is loaded. +User: kexec tools, kdump service + +What: /sys/kernel/kexec/crash_loaded +Date: Nov 2025 +Contact: kexec@lists.infradead.org +Description: read only + Indicates whether a crash (kdump) kernel is currently + loaded into memory. It shows 1 if a crash kernel has been + successfully loaded for panic handling, or 0 if no crash + kernel is present. +User: Kexec tools, Kdump service + +What: /sys/kernel/kexec/crash_size +Date: Nov 2025 +Contact: kexec@lists.infradead.org +Description: read/write + Shows the amount of memory reserved for loading the crash + (kdump) kernel. It reports the size, in bytes, of the + crash kernel area defined by the crashkernel= parameter. + This interface also allows reducing the crashkernel + reservation by writing a smaller value, and the reclaimed + space is added back to the system RAM. +User: Kdump service + +What: /sys/kernel/kexec/crash_elfcorehdr_size +Date: Nov 2025 +Contact: kexec@lists.infradead.org +Description: read only + Indicates the preferred size of the memory buffer for the + ELF core header used by the crash (kdump) kernel. It defines + how much space is needed to hold metadata about the crashed + system, including CPU and memory information. This information + is used by the user space utility kexec to support updating the + in-kernel kdump image during hotplug operations. +User: Kexec tools + +What: /sys/kernel/kexec/crash_cma_ranges +Date: Nov 2025 +Contact: kexec@lists.infradead.org +Description: read only + Provides information about the memory ranges reserved from + the Contiguous Memory Allocator (CMA) area that are allocated + to the crash (kdump) kernel. It lists the start and end physical + addresses of CMA regions assigned for crashkernel use. +User: kdump service diff --git a/Documentation/admin-guide/dynamic-debug-howto.rst b/Documentation/admin-guide/dynamic-debug-howto.rst index 7c036590cd07..095a63892257 100644 --- a/Documentation/admin-guide/dynamic-debug-howto.rst +++ b/Documentation/admin-guide/dynamic-debug-howto.rst @@ -223,12 +223,13 @@ The flags are:: f Include the function name s Include the source file name l Include line number + d Include call trace For ``print_hex_dump_debug()`` and ``print_hex_dump_bytes()``, only the ``p`` flag has meaning, other flags are ignored. -Note the regexp ``^[-+=][fslmpt_]+$`` matches a flags specification. -To clear all flags at once, use ``=_`` or ``-fslmpt``. +Note the regexp ``^[-+=][fslmptd_]+$`` matches a flags specification. +To clear all flags at once, use ``=_`` or ``-fslmptd``. Debug messages during Boot Process diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index b242519f57da..b86b6d946d88 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2114,14 +2114,20 @@ Kernel parameters the added memory block itself do not be affected. hung_task_panic= - [KNL] Should the hung task detector generate panics. - Format: 0 | 1 + [KNL] Number of hung tasks to trigger kernel panic. + Format: <int> + + When set to a non-zero value, a kernel panic will be triggered if + the number of detected hung tasks reaches this value. + + 0: don't panic + 1: panic immediately on first hung task + N: panic after N hung tasks are detected in a single scan - A value of 1 instructs the kernel to panic when a - hung task is detected. The default value is controlled - by the CONFIG_BOOTPARAM_HUNG_TASK_PANIC build-time - option. The value selected by this boot parameter can - be changed later by the kernel.hung_task_panic sysctl. + The default value is controlled by the + CONFIG_BOOTPARAM_HUNG_TASK_PANIC build-time option. The value + selected by this boot parameter can be changed later by the + kernel.hung_task_panic sysctl. hvc_iucv= [S390] Number of z/VM IUCV hypervisor console (HVC) terminal devices. Valid values: 0..8 diff --git a/Documentation/admin-guide/sysctl/kernel.rst b/Documentation/admin-guide/sysctl/kernel.rst index f3ee807b5d8b..239da22c4e28 100644 --- a/Documentation/admin-guide/sysctl/kernel.rst +++ b/Documentation/admin-guide/sysctl/kernel.rst @@ -397,13 +397,14 @@ a hung task is detected. hung_task_panic =============== -Controls the kernel's behavior when a hung task is detected. +When set to a non-zero value, a kernel panic will be triggered if the +number of hung tasks found during a single scan reaches this value. This file shows up if ``CONFIG_DETECT_HUNG_TASK`` is enabled. -= ================================================= += ======================================================= 0 Continue operation. This is the default behavior. -1 Panic immediately. -= ================================================= +N Panic when N hung tasks are found during a single scan. += ======================================================= hung_task_check_count @@ -421,6 +422,11 @@ the system boot. This file shows up if ``CONFIG_DETECT_HUNG_TASK`` is enabled. +hung_task_sys_info +================== +A comma separated list of extra system information to be dumped when +hung task is detected, for example, "tasks,mem,timers,locks,...". +Refer 'panic_sys_info' section below for more details. hung_task_timeout_secs ====================== @@ -515,6 +521,15 @@ default), only processes with the CAP_SYS_ADMIN capability may create io_uring instances. +kernel_sys_info +=============== +A comma separated list of extra system information to be dumped when +soft/hard lockup is detected, for example, "tasks,mem,timers,locks,...". +Refer 'panic_sys_info' section below for more details. + +It serves as the default kernel control knob, which will take effect +when a kernel module calls sys_info() with parameter==0. + kexec_load_disabled =================== @@ -576,6 +591,11 @@ if leaking kernel pointer values to unprivileged users is a concern. When ``kptr_restrict`` is set to 2, kernel pointers printed using %pK will be replaced with 0s regardless of privileges. +softlockup_sys_info & hardlockup_sys_info +========================================= +A comma separated list of extra system information to be dumped when +soft/hard lockup is detected, for example, "tasks,mem,timers,locks,...". +Refer 'panic_sys_info' section below for more details. modprobe ======== @@ -910,8 +930,8 @@ to 'panic_print'. Possible values are: ============= =================================================== tasks print all tasks info mem print system memory info -timer print timers info -lock print locks info if CONFIG_LOCKDEP is on +timers print timers info +locks print locks info if CONFIG_LOCKDEP is on ftrace print ftrace buffer all_bt print all CPUs backtrace (if available in the arch) blocked_tasks print only tasks in uninterruptible (blocked) state diff --git a/Documentation/arch/x86/boot.rst b/Documentation/arch/x86/boot.rst index 32eea3d2807e..6d36ce86fd8e 100644 --- a/Documentation/arch/x86/boot.rst +++ b/Documentation/arch/x86/boot.rst @@ -416,7 +416,7 @@ Offset/size: 0x210/1 Protocol: 2.00+ ============ ================== - If your boot loader has an assigned id (see table below), enter + If your boot loader has an assigned ID (see table below), enter 0xTV here, where T is an identifier for the boot loader and V is a version number. Otherwise, enter 0xFF here. @@ -431,31 +431,31 @@ Protocol: 2.00+ ext_loader_type <- 0x05 ext_loader_ver <- 0x23 - Assigned boot loader ids (hexadecimal): + Assigned boot loader IDs: == ======================================= - 0 LILO - (0x00 reserved for pre-2.00 bootloader) - 1 Loadlin - 2 bootsect-loader - (0x20, all other values reserved) - 3 Syslinux - 4 Etherboot/gPXE/iPXE - 5 ELILO - 7 GRUB - 8 U-Boot - 9 Xen - A Gujin - B Qemu - C Arcturus Networks uCbootloader - D kexec-tools - E Extended (see ext_loader_type) - F Special (0xFF = undefined) - 10 Reserved - 11 Minimal Linux Bootloader - <http://sebastian-plotz.blogspot.de> - 12 OVMF UEFI virtualization stack - 13 barebox + 0x0 LILO + (0x00 reserved for pre-2.00 bootloader) + 0x1 Loadlin + 0x2 bootsect-loader + (0x20, all other values reserved) + 0x3 Syslinux + 0x4 Etherboot/gPXE/iPXE + 0x5 ELILO + 0x7 GRUB + 0x8 U-Boot + 0x9 Xen + 0xA Gujin + 0xB Qemu + 0xC Arcturus Networks uCbootloader + 0xD kexec-tools + 0xE Extended (see ext_loader_type) + 0xF Special (0xFF = undefined) + 0x10 Reserved + 0x11 Minimal Linux Bootloader + <http://sebastian-plotz.blogspot.de> + 0x12 OVMF UEFI virtualization stack + 0x13 barebox == ======================================= Please contact <hpa@zytor.com> if you need a bootloader ID value assigned. diff --git a/Documentation/core-api/index.rst b/Documentation/core-api/index.rst index 6cbdcbfa79c3..5eb0fbbbc323 100644 --- a/Documentation/core-api/index.rst +++ b/Documentation/core-api/index.rst @@ -138,6 +138,7 @@ Documents that don't fit elsewhere or which have yet to be categorized. :maxdepth: 1 librs + liveupdate netlink .. only:: subproject and html diff --git a/Documentation/core-api/kho/concepts.rst b/Documentation/core-api/kho/concepts.rst index 36d5c05cfb30..d626d1dbd678 100644 --- a/Documentation/core-api/kho/concepts.rst +++ b/Documentation/core-api/kho/concepts.rst @@ -70,5 +70,5 @@ in the FDT. That state is called the KHO finalization phase. Public API ========== -.. kernel-doc:: kernel/kexec_handover.c +.. kernel-doc:: kernel/liveupdate/kexec_handover.c :export: diff --git a/Documentation/core-api/liveupdate.rst b/Documentation/core-api/liveupdate.rst new file mode 100644 index 000000000000..7960eb15a81f --- /dev/null +++ b/Documentation/core-api/liveupdate.rst @@ -0,0 +1,61 @@ +.. SPDX-License-Identifier: GPL-2.0 + +======================== +Live Update Orchestrator +======================== +:Author: Pasha Tatashin <pasha.tatashin@soleen.com> + +.. kernel-doc:: kernel/liveupdate/luo_core.c + :doc: Live Update Orchestrator (LUO) + +LUO Sessions +============ +.. kernel-doc:: kernel/liveupdate/luo_session.c + :doc: LUO Sessions + +LUO Preserving File Descriptors +=============================== +.. kernel-doc:: kernel/liveupdate/luo_file.c + :doc: LUO File Descriptors + +Live Update Orchestrator ABI +============================ +.. kernel-doc:: include/linux/kho/abi/luo.h + :doc: Live Update Orchestrator ABI + +The following types of file descriptors can be preserved + +.. toctree:: + :maxdepth: 1 + + ../mm/memfd_preservation + +Public API +========== +.. kernel-doc:: include/linux/liveupdate.h + +.. kernel-doc:: include/linux/kho/abi/luo.h + :functions: + +.. kernel-doc:: kernel/liveupdate/luo_core.c + :export: + +.. kernel-doc:: kernel/liveupdate/luo_file.c + :export: + +Internal API +============ +.. kernel-doc:: kernel/liveupdate/luo_core.c + :internal: + +.. kernel-doc:: kernel/liveupdate/luo_session.c + :internal: + +.. kernel-doc:: kernel/liveupdate/luo_file.c + :internal: + +See Also +======== + +- :doc:`Live Update uAPI </userspace-api/liveupdate>` +- :doc:`/core-api/kho/concepts` diff --git a/Documentation/dev-tools/checkpatch.rst b/Documentation/dev-tools/checkpatch.rst index dfaad0a279ff..fa2988dd4657 100644 --- a/Documentation/dev-tools/checkpatch.rst +++ b/Documentation/dev-tools/checkpatch.rst @@ -1238,6 +1238,16 @@ Others The patch file does not appear to be in unified-diff format. Please regenerate the patch file before sending it to the maintainer. + **PLACEHOLDER_USE** + Detects unhandled placeholder text left in cover letters or commit headers/logs. + Common placeholders include lines like:: + + *** SUBJECT HERE *** + *** BLURB HERE *** + + These typically come from autogenerated templates. Replace them with a proper + subject and description before sending. + **PRINTF_0XDECIMAL** Prefixing 0x with decimal output is defective and should be corrected. diff --git a/Documentation/devicetree/bindings/mfd/rohm,bd96801-pmic.yaml b/Documentation/devicetree/bindings/mfd/rohm,bd96801-pmic.yaml index 0e06570483ae..adb491bcc8dc 100644 --- a/Documentation/devicetree/bindings/mfd/rohm,bd96801-pmic.yaml +++ b/Documentation/devicetree/bindings/mfd/rohm,bd96801-pmic.yaml @@ -57,8 +57,7 @@ properties: - prstb - intb-only - timeout-sec: - maxItems: 2 + timeout-sec: true regulators: $ref: /schemas/regulator/rohm,bd96801-regulator.yaml @@ -72,7 +71,10 @@ required: - interrupt-names - regulators -additionalProperties: false +allOf: + - $ref: /schemas/watchdog/watchdog.yaml + +unevaluatedProperties: false examples: - | diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml index 661c2b425da3..137f95028313 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.yaml @@ -24,6 +24,7 @@ properties: - qcom,msm8998-adsp-pas - qcom,msm8998-slpi-pas - qcom,sdm660-adsp-pas + - qcom,sdm660-cdsp-pas - qcom,sdm845-adsp-pas - qcom,sdm845-cdsp-pas - qcom,sdm845-slpi-pas @@ -31,9 +32,6 @@ properties: reg: maxItems: 1 - cx-supply: - description: Phandle to the CX regulator - px-supply: description: Phandle to the PX regulator @@ -69,6 +67,8 @@ allOf: - qcom,msm8996-slpi-pil - qcom,msm8998-adsp-pas - qcom,msm8998-slpi-pas + - qcom,sdm660-adsp-pas + - qcom,sdm660-cdsp-pas - qcom,sdm845-adsp-pas - qcom,sdm845-cdsp-pas - qcom,sdm845-slpi-pas @@ -93,6 +93,8 @@ allOf: - qcom,msm8996-slpi-pil - qcom,msm8998-adsp-pas - qcom,msm8998-slpi-pas + - qcom,sdm660-adsp-pas + - qcom,sdm660-cdsp-pas - qcom,sdm845-adsp-pas - qcom,sdm845-cdsp-pas - qcom,sdm845-slpi-pas @@ -108,20 +110,13 @@ allOf: compatible: contains: enum: - - qcom,msm8974-adsp-pil - then: - required: - - cx-supply - - - if: - properties: - compatible: - contains: - enum: - qcom,msm8226-adsp-pil - qcom,msm8953-adsp-pil + - qcom,msm8974-adsp-pil - qcom,msm8996-adsp-pil - qcom,msm8998-adsp-pas + - qcom,sdm660-adsp-pas + - qcom,sdm660-cdsp-pas then: properties: power-domains: @@ -178,6 +173,7 @@ allOf: - qcom,msm8998-adsp-pas - qcom,msm8998-slpi-pas - qcom,sdm660-adsp-pas + - qcom,sdm660-cdsp-pas then: properties: qcom,qmp: false @@ -187,6 +183,7 @@ examples: #include <dt-bindings/clock/qcom,rpmcc.h> #include <dt-bindings/interrupt-controller/arm-gic.h> #include <dt-bindings/interrupt-controller/irq.h> + #include <dt-bindings/power/qcom-rpmpd.h> adsp { compatible = "qcom,msm8974-adsp-pil"; @@ -204,7 +201,8 @@ examples: clocks = <&rpmcc RPM_CXO_CLK>; clock-names = "xo"; - cx-supply = <&pm8841_s2>; + power-domains = <&rpmpd MSM8974_VDDCX>; + power-domain-names = "cx"; memory-region = <&adsp_region>; diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml b/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml index 96d53baf6e00..5dbda3a55047 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml +++ b/Documentation/devicetree/bindings/remoteproc/qcom,sc8280xp-pas.yaml @@ -91,9 +91,13 @@ allOf: power-domains: items: - description: NSP power domain + - description: CX power domain + - description: MXC power domain power-domain-names: items: - const: nsp + - const: cx + - const: mxc unevaluatedProperties: false diff --git a/Documentation/devicetree/bindings/watchdog/airoha,en7581-wdt.yaml b/Documentation/devicetree/bindings/watchdog/airoha,en7581-wdt.yaml index 6bbab3cb28e5..6259478bdae5 100644 --- a/Documentation/devicetree/bindings/watchdog/airoha,en7581-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/airoha,en7581-wdt.yaml @@ -14,7 +14,11 @@ allOf: properties: compatible: - const: airoha,en7581-wdt + oneOf: + - items: + - const: airoha,an7583-wdt + - const: airoha,en7581-wdt + - const: airoha,en7581-wdt reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/watchdog/aspeed,ast2400-wdt.yaml b/Documentation/devicetree/bindings/watchdog/aspeed,ast2400-wdt.yaml index be78a9865584..9322cb5b462a 100644 --- a/Documentation/devicetree/bindings/watchdog/aspeed,ast2400-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/aspeed,ast2400-wdt.yaml @@ -15,6 +15,7 @@ properties: - aspeed,ast2400-wdt - aspeed,ast2500-wdt - aspeed,ast2600-wdt + - aspeed,ast2700-wdt reg: maxItems: 1 @@ -87,13 +88,15 @@ properties: aspeed,reset-mask: $ref: /schemas/types.yaml#/definitions/uint32-array minItems: 1 - maxItems: 2 + maxItems: 5 description: > A bitmask indicating which peripherals will be reset if the watchdog timer expires. On AST2500 SoCs this should be a single word defined using the AST2500_WDT_RESET_* macros; on AST2600 SoCs this should be a two-word array with the first word defined using the AST2600_WDT_RESET1_* macros, - and the second word defined using the AST2600_WDT_RESET2_* macros. + and the second word defined using the AST2600_WDT_RESET2_* macros; on + AST2700 SoCs, this should be five-word array from AST2700_WDT_RESET1_* + macros to AST2700_WDT_RESET5_* macros. required: - compatible @@ -114,6 +117,7 @@ allOf: enum: - aspeed,ast2500-wdt - aspeed,ast2600-wdt + - aspeed,ast2700-wdt - if: required: - aspeed,ext-active-high diff --git a/Documentation/devicetree/bindings/watchdog/lantiq,wdt.yaml b/Documentation/devicetree/bindings/watchdog/lantiq,wdt.yaml new file mode 100644 index 000000000000..a7edae9ca05a --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/lantiq,wdt.yaml @@ -0,0 +1,57 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/lantiq,wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Lantiq WTD watchdog + +maintainers: + - Hauke Mehrtens <hauke@hauke-m.de> + +properties: + compatible: + oneOf: + - enum: + - lantiq,falcon-wdt + - lantiq,wdt + - lantiq,xrx100-wdt + - items: + - enum: + - lantiq,xrx200-wdt + - const: lantiq,xrx100-wdt + + reg: + maxItems: 1 + + lantiq,rcu: + $ref: /schemas/types.yaml#/definitions/phandle + description: Phandle to the RCU syscon node + +required: + - compatible + - reg + +allOf: + - $ref: watchdog.yaml# + - if: + properties: + compatible: + contains: + enum: + - lantiq,xrx100-wdt + - lantiq,falcon-wdt + then: + required: + - lantiq,rcu + +unevaluatedProperties: false + +examples: + - | + watchdog@803f0 { + compatible = "lantiq,xrx200-wdt", "lantiq,xrx100-wdt"; + reg = <0x803f0 0x10>; + + lantiq,rcu = <&rcu0>; + }; diff --git a/Documentation/devicetree/bindings/watchdog/lantiq-wdt.txt b/Documentation/devicetree/bindings/watchdog/lantiq-wdt.txt deleted file mode 100644 index 18d4d8302702..000000000000 --- a/Documentation/devicetree/bindings/watchdog/lantiq-wdt.txt +++ /dev/null @@ -1,24 +0,0 @@ -Lantiq WTD watchdog binding -============================ - -This describes the binding of the Lantiq watchdog driver. - -------------------------------------------------------------------------------- -Required properties: -- compatible : Should be one of - "lantiq,wdt" - "lantiq,xrx100-wdt" - "lantiq,xrx200-wdt", "lantiq,xrx100-wdt" - "lantiq,falcon-wdt" -- reg : Address of the watchdog block -- lantiq,rcu : A phandle to the RCU syscon (required for - "lantiq,falcon-wdt" and "lantiq,xrx100-wdt") - -------------------------------------------------------------------------------- -Example for the watchdog on the xRX200 SoCs: - watchdog@803f0 { - compatible = "lantiq,xrx200-wdt", "lantiq,xrx100-wdt"; - reg = <0x803f0 0x10>; - - lantiq,rcu = <&rcu0>; - }; diff --git a/Documentation/devicetree/bindings/watchdog/loongson,ls1x-wdt.yaml b/Documentation/devicetree/bindings/watchdog/loongson,ls1x-wdt.yaml index 81690d4b62a6..50a9b468c4a3 100644 --- a/Documentation/devicetree/bindings/watchdog/loongson,ls1x-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/loongson,ls1x-wdt.yaml @@ -4,7 +4,7 @@ $id: http://devicetree.org/schemas/watchdog/loongson,ls1x-wdt.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: Loongson-1 Watchdog Timer +title: Loongson Watchdog Timer maintainers: - Keguang Zhang <keguang.zhang@gmail.com> @@ -17,6 +17,7 @@ properties: enum: - loongson,ls1b-wdt - loongson,ls1c-wdt + - loongson,ls2k0300-wdt reg: maxItems: 1 diff --git a/Documentation/devicetree/bindings/watchdog/marvel.txt b/Documentation/devicetree/bindings/watchdog/marvel.txt deleted file mode 100644 index c1b67a78f00c..000000000000 --- a/Documentation/devicetree/bindings/watchdog/marvel.txt +++ /dev/null @@ -1,45 +0,0 @@ -* Marvell Orion Watchdog Time - -Required Properties: - -- Compatibility : "marvell,orion-wdt" - "marvell,armada-370-wdt" - "marvell,armada-xp-wdt" - "marvell,armada-375-wdt" - "marvell,armada-380-wdt" - -- reg : Should contain two entries: first one with the - timer control address, second one with the - rstout enable address. - -For "marvell,armada-375-wdt" and "marvell,armada-380-wdt": - -- reg : A third entry is mandatory and should contain the - shared mask/unmask RSTOUT address. - -Clocks required for compatibles = "marvell,orion-wdt", - "marvell,armada-370-wdt": -- clocks : Must contain a single entry describing the clock input - -Clocks required for compatibles = "marvell,armada-xp-wdt" - "marvell,armada-375-wdt" - "marvell,armada-380-wdt": -- clocks : Must contain an entry for each entry in clock-names. -- clock-names : Must include the following entries: - "nbclk" (L2/coherency fabric clock), - "fixed" (Reference 25 MHz fixed-clock). - -Optional properties: - -- interrupts : Contains the IRQ for watchdog expiration -- timeout-sec : Contains the watchdog timeout in seconds - -Example: - - wdt@20300 { - compatible = "marvell,orion-wdt"; - reg = <0x20300 0x28>, <0x20108 0x4>; - interrupts = <3>; - timeout-sec = <10>; - clocks = <&gate_clk 7>; - }; diff --git a/Documentation/devicetree/bindings/watchdog/marvell,orion-wdt.yaml b/Documentation/devicetree/bindings/watchdog/marvell,orion-wdt.yaml new file mode 100644 index 000000000000..fdc7bc45dfde --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/marvell,orion-wdt.yaml @@ -0,0 +1,100 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/marvell,orion-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Marvell Orion Watchdog Timer + +maintainers: + - Andrew Lunn <andrew@lunn.ch> + - Gregory Clement <gregory.clement@bootlin.com> + +properties: + compatible: + enum: + - marvell,orion-wdt + - marvell,armada-370-wdt + - marvell,armada-xp-wdt + - marvell,armada-375-wdt + - marvell,armada-380-wdt + + reg: + minItems: 2 + items: + - description: Timer control register address + - description: RSTOUT enable register address + - description: Shared mask/unmask RSTOUT register address + + clocks: + minItems: 1 + items: + - description: L2/coherency fabric clock input + - description: Reference 25 MHz fixed-clock supply + + clock-names: + minItems: 1 + items: + - const: nbclk + - const: fixed + + interrupts: + minItems: 1 + items: + - description: timeout + - description: pre-timeout + +allOf: + - $ref: watchdog.yaml# + - if: + properties: + compatible: + contains: + enum: + - marvell,armada-375-wdt + - marvell,armada-380-wdt + then: + properties: + reg: + minItems: 3 + else: + properties: + reg: + maxItems: 2 + + - if: + properties: + compatible: + contains: + enum: + - marvell,armada-xp-wdt + - marvell,armada-375-wdt + - marvell,armada-380-wdt + then: + properties: + clocks: + minItems: 2 + clock-names: + minItems: 2 + interrupts: + minItems: 2 + + required: + - clock-names + +required: + - compatible + - reg + - clocks + +unevaluatedProperties: false + +examples: + - | + watchdog@20300 { + compatible = "marvell,orion-wdt"; + reg = <0x20300 0x28>, <0x20108 0x4>; + interrupts = <3>; + timeout-sec = <10>; + clocks = <&gate_clk 7>; + }; diff --git a/Documentation/devicetree/bindings/watchdog/mediatek,mtk-wdt.yaml b/Documentation/devicetree/bindings/watchdog/mediatek,mtk-wdt.yaml index ba0bfd73ab62..953629cb9558 100644 --- a/Documentation/devicetree/bindings/watchdog/mediatek,mtk-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/mediatek,mtk-wdt.yaml @@ -41,6 +41,8 @@ properties: - mediatek,mt7623-wdt - mediatek,mt7629-wdt - mediatek,mt8173-wdt + - mediatek,mt8188-wdt + - mediatek,mt8189-wdt - mediatek,mt8365-wdt - mediatek,mt8516-wdt - const: mediatek,mt6589-wdt diff --git a/Documentation/devicetree/bindings/watchdog/omap-wdt.txt b/Documentation/devicetree/bindings/watchdog/omap-wdt.txt deleted file mode 100644 index 1fa20e453a2d..000000000000 --- a/Documentation/devicetree/bindings/watchdog/omap-wdt.txt +++ /dev/null @@ -1,15 +0,0 @@ -TI Watchdog Timer (WDT) Controller for OMAP - -Required properties: -- compatible : "ti,omap3-wdt" for OMAP3 or "ti,omap4-wdt" for OMAP4 -- ti,hwmods : Name of the hwmod associated to the WDT - -Optional properties: -- timeout-sec : default watchdog timeout in seconds - -Examples: - -wdt2: wdt@4a314000 { - compatible = "ti,omap4-wdt", "ti,omap3-wdt"; - ti,hwmods = "wd_timer2"; -}; diff --git a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml index 49e2b807db0b..54f5311ed016 100644 --- a/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/qcom-wdt.yaml @@ -22,6 +22,7 @@ properties: - qcom,apss-wdt-ipq5332 - qcom,apss-wdt-ipq5424 - qcom,apss-wdt-ipq9574 + - qcom,apss-wdt-kaanapali - qcom,apss-wdt-msm8226 - qcom,apss-wdt-msm8974 - qcom,apss-wdt-msm8994 diff --git a/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml new file mode 100644 index 000000000000..099200c4f136 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/renesas,r9a09g057-wdt.yaml @@ -0,0 +1,99 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/renesas,r9a09g057-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas RZ/V2H(P) Watchdog Timer (WDT) Controller + +maintainers: + - Lad Prabhakar <prabhakar.mahadev-lad.rj@bp.renesas.com> + +properties: + compatible: + oneOf: + - items: + - enum: + - renesas,r9a09g047-wdt # RZ/G3E + - renesas,r9a09g056-wdt # RZ/V2N + - const: renesas,r9a09g057-wdt # RZ/V2H(P) + + - items: + - const: renesas,r9a09g087-wdt # RZ/N2H + - const: renesas,r9a09g077-wdt # RZ/T2H + + - enum: + - renesas,r9a09g057-wdt # RZ/V2H(P) + - renesas,r9a09g077-wdt # RZ/T2H + + reg: + minItems: 1 + maxItems: 2 + + clocks: + minItems: 1 + items: + - description: Register access clock + - description: Main clock + + clock-names: + minItems: 1 + items: + - const: pclk + - const: oscclk + + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + + timeout-sec: true + +required: + - compatible + - reg + - clocks + - clock-names + - power-domains + +allOf: + - $ref: watchdog.yaml# + + - if: + properties: + compatible: + contains: + const: renesas,r9a09g057-wdt + then: + properties: + reg: + maxItems: 1 + clocks: + minItems: 2 + clock-names: + minItems: 2 + else: + properties: + clocks: + maxItems: 1 + clock-names: + maxItems: 1 + reg: + minItems: 2 + resets: false + +additionalProperties: false + +examples: + - | + #include <dt-bindings/clock/renesas,r9a09g057-cpg.h> + + watchdog@11c00400 { + compatible = "renesas,r9a09g057-wdt"; + reg = <0x11c00400 0x400>; + clocks = <&cpg CPG_MOD 0x4b>, <&cpg CPG_MOD 0x4c>; + clock-names = "pclk", "oscclk"; + resets = <&cpg 0x75>; + power-domains = <&cpg>; + }; diff --git a/Documentation/devicetree/bindings/watchdog/renesas,rcar-gen3-wwdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,rcar-gen3-wwdt.yaml new file mode 100644 index 000000000000..ffafe9a6d3f5 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/renesas,rcar-gen3-wwdt.yaml @@ -0,0 +1,114 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/renesas,rcar-gen3-wwdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas Window Watchdog Timer (WWDT) Controller + +maintainers: + - Wolfram Sang <wsa+renesas@sang-engineering.com> + +properties: + compatible: + oneOf: + - items: + - enum: + - renesas,r8a77970-wwdt # R-Car V3M + - renesas,r8a77980-wwdt # R-Car V3H + - const: renesas,rcar-gen3-wwdt + + - items: + - enum: + - renesas,r8a779a0-wwdt # R-Car V3U + - renesas,r8a779f0-wwdt # R-Car S4 + - renesas,r8a779g0-wwdt # R-Car V4H + - renesas,r8a779h0-wwdt # R-Car V4M + - const: renesas,rcar-gen4-wwdt + + reg: + maxItems: 1 + + interrupts: + items: + - description: Pretimeout, 75% of overflow reached + - description: Error occurred + + interrupt-names: + items: + - const: pretimeout + - const: error + + clocks: + items: + - description: Counting clock + - description: Bus clock + + clock-names: + items: + - const: cnt + - const: bus + + resets: + minItems: 1 + maxItems: 2 + + reset-names: + minItems: 1 + items: + - const: cnt + - const: bus + + power-domains: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + - interrupt-names + - clocks + - clock-names + - resets + - reset-names + - power-domains + +allOf: + - $ref: watchdog.yaml# + + - if: + properties: + compatible: + contains: + enum: + - renesas,r8a779a0-wwdt + - renesas,r8a779f0-wwdt + then: + properties: + resets: + minItems: 2 + reset-names: + minItems: 2 + +additionalProperties: false + +examples: + - | + #include <dt-bindings/clock/r8a779g0-cpg-mssr.h> + #include <dt-bindings/power/r8a779g0-sysc.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + + watchdog@ffc90000 { + compatible = "renesas,r8a779g0-wwdt", + "renesas,rcar-gen4-wwdt"; + reg = <0xffc90000 0x10>; + interrupts = <GIC_SPI 310 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 311 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "pretimeout", "error"; + clocks = <&cpg CPG_CORE R8A779G0_CLK_R>, + <&cpg CPG_CORE R8A779G0_CLK_SASYNCRT>; + clock-names = "cnt", "bus"; + power-domains = <&sysc R8A779G0_PD_ALWAYS_ON>; + resets = <&cpg 1200>; + reset-names = "cnt"; + }; diff --git a/Documentation/devicetree/bindings/watchdog/renesas,rza-wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,rza-wdt.yaml new file mode 100644 index 000000000000..ba922c3f7b10 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/renesas,rza-wdt.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/renesas,rza-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas RZ/A Watchdog Timer (WDT) Controller + +maintainers: + - Wolfram Sang <wsa+renesas@sang-engineering.com> + +properties: + compatible: + items: + - enum: + - renesas,r7s72100-wdt # RZ/A1 + - renesas,r7s9210-wdt # RZ/A2 + - const: renesas,rza-wdt # RZ/A + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + timeout-sec: true + +required: + - compatible + - reg + - clocks + +allOf: + - $ref: watchdog.yaml# + +additionalProperties: false + +examples: + - | + #include <dt-bindings/clock/r7s72100-clock.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + + watchdog@fcfe0000 { + compatible = "renesas,r7s72100-wdt", "renesas,rza-wdt"; + reg = <0xfcfe0000 0x6>; + interrupts = <GIC_SPI 106 IRQ_TYPE_LEVEL_HIGH>; + clocks = <&p0_clk>; + }; diff --git a/Documentation/devicetree/bindings/watchdog/renesas,rzg2l-wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,rzg2l-wdt.yaml new file mode 100644 index 000000000000..a4d06c9c8b86 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/renesas,rzg2l-wdt.yaml @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/renesas,rzg2l-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas RZ/G2L Watchdog Timer (WDT) Controller + +maintainers: + - Biju Das <biju.das.jz@bp.renesas.com> + +properties: + compatible: + oneOf: + - items: + - enum: + - renesas,r9a07g043-wdt # RZ/G2UL and RZ/Five + - renesas,r9a07g044-wdt # RZ/G2{L,LC} + - renesas,r9a07g054-wdt # RZ/V2L + - renesas,r9a08g045-wdt # RZ/G3S + - const: renesas,rzg2l-wdt + + - items: + - const: renesas,r9a09g011-wdt # RZ/V2M + - const: renesas,rzv2m-wdt # RZ/V2M + + reg: + maxItems: 1 + + interrupts: + minItems: 1 + items: + - description: Timeout + - description: Parity error + + interrupt-names: + minItems: 1 + items: + - const: wdt + - const: perrout + + clocks: + items: + - description: Register access clock + - description: Main clock + + clock-names: + items: + - const: pclk + - const: oscclk + + power-domains: + maxItems: 1 + + resets: + maxItems: 1 + + timeout-sec: true + +required: + - compatible + - reg + - interrupts + - clocks + - clock-names + - power-domains + - resets + +allOf: + - $ref: watchdog.yaml# + + - if: + properties: + compatible: + contains: + const: renesas,rzg2l-wdt + then: + properties: + interrupts: + minItems: 2 + interrupt-names: + minItems: 2 + required: + - interrupt-names + else: + properties: + interrupts: + maxItems: 1 + interrupt-names: + maxItems: 1 + +additionalProperties: false + +examples: + - | + #include <dt-bindings/clock/r9a07g044-cpg.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + + watchdog@12800800 { + compatible = "renesas,r9a07g044-wdt", + "renesas,rzg2l-wdt"; + reg = <0x12800800 0x400>; + clocks = <&cpg CPG_MOD R9A07G044_WDT0_PCLK>, + <&cpg CPG_MOD R9A07G044_WDT0_CLK>; + clock-names = "pclk", "oscclk"; + interrupts = <GIC_SPI 49 IRQ_TYPE_LEVEL_HIGH>, + <GIC_SPI 50 IRQ_TYPE_LEVEL_HIGH>; + interrupt-names = "wdt", "perrout"; + resets = <&cpg R9A07G044_WDT0_PRESETN>; + power-domains = <&cpg>; + }; diff --git a/Documentation/devicetree/bindings/watchdog/renesas,rzn1-wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,rzn1-wdt.yaml new file mode 100644 index 000000000000..7e3ee533cd56 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/renesas,rzn1-wdt.yaml @@ -0,0 +1,50 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/renesas,rzn1-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Renesas RZ/N1 Watchdog Timer (WDT) Controller + +maintainers: + - Wolfram Sang <wsa+renesas@sang-engineering.com> + +properties: + compatible: + items: + - const: renesas,r9a06g032-wdt # RZ/N1D + - const: renesas,rzn1-wdt # RZ/N1 + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + clocks: + maxItems: 1 + + timeout-sec: true + +required: + - compatible + - reg + - interrupts + - clocks + +allOf: + - $ref: watchdog.yaml# + +additionalProperties: false + +examples: + - | + #include <dt-bindings/clock/r9a06g032-sysctrl.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + + watchdog@40008000 { + compatible = "renesas,r9a06g032-wdt", "renesas,rzn1-wdt"; + reg = <0x40008000 0x1000>; + interrupts = <GIC_SPI 73 IRQ_TYPE_EDGE_RISING>; + clocks = <&sysctrl R9A06G032_CLK_WATCHDOG>; + }; diff --git a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml index b6e60162c263..7aebc5a5cf17 100644 --- a/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/renesas,wdt.yaml @@ -15,30 +15,6 @@ properties: oneOf: - items: - enum: - - renesas,r7s72100-wdt # RZ/A1 - - renesas,r7s9210-wdt # RZ/A2 - - const: renesas,rza-wdt # RZ/A - - - items: - - enum: - - renesas,r9a06g032-wdt # RZ/N1D - - const: renesas,rzn1-wdt # RZ/N1 - - - items: - - enum: - - renesas,r9a07g043-wdt # RZ/G2UL and RZ/Five - - renesas,r9a07g044-wdt # RZ/G2{L,LC} - - renesas,r9a07g054-wdt # RZ/V2L - - renesas,r9a08g045-wdt # RZ/G3S - - const: renesas,rzg2l-wdt - - - items: - - enum: - - renesas,r9a09g011-wdt # RZ/V2M - - const: renesas,rzv2m-wdt # RZ/V2M - - - items: - - enum: - renesas,r8a7742-wdt # RZ/G1H - renesas,r8a7743-wdt # RZ/G1M - renesas,r8a7744-wdt # RZ/G1N @@ -75,47 +51,14 @@ properties: - renesas,r8a779h0-wdt # R-Car V4M - const: renesas,rcar-gen4-wdt # R-Car Gen4 - - items: - - enum: - - renesas,r9a09g047-wdt # RZ/G3E - - renesas,r9a09g056-wdt # RZ/V2N - - const: renesas,r9a09g057-wdt # RZ/V2H(P) - - - enum: - - renesas,r9a09g057-wdt # RZ/V2H(P) - - renesas,r9a09g077-wdt # RZ/T2H - - - items: - - const: renesas,r9a09g087-wdt # RZ/N2H - - const: renesas,r9a09g077-wdt # RZ/T2H - reg: - minItems: 1 - maxItems: 2 + maxItems: 1 interrupts: - minItems: 1 - items: - - description: Timeout - - description: Parity error - - interrupt-names: - minItems: 1 - items: - - const: wdt - - const: perrout + maxItems: 1 clocks: - minItems: 1 - items: - - description: Register access clock - - description: Main clock - - clock-names: - minItems: 1 - items: - - const: pclk - - const: oscclk + maxItems: 1 power-domains: maxItems: 1 @@ -129,6 +72,8 @@ required: - compatible - reg - clocks + - interrupts + - power-domains allOf: - $ref: watchdog.yaml# @@ -138,90 +83,11 @@ allOf: properties: compatible: contains: - enum: - - renesas,r9a09g077-wdt - - renesas,rza-wdt - - renesas,rzn1-wdt + const: renesas,r8a77980-wdt then: required: - - power-domains - resets - - if: - properties: - compatible: - contains: - enum: - - renesas,r9a09g057-wdt - - renesas,rzg2l-wdt - - renesas,rzv2m-wdt - then: - properties: - clocks: - minItems: 2 - clock-names: - minItems: 2 - required: - - clock-names - else: - properties: - clocks: - maxItems: 1 - - - if: - properties: - compatible: - contains: - enum: - - renesas,rzg2l-wdt - then: - properties: - interrupts: - minItems: 2 - interrupt-names: - minItems: 2 - required: - - interrupt-names - else: - properties: - interrupts: - maxItems: 1 - - - if: - properties: - compatible: - contains: - enum: - - renesas,r9a09g057-wdt - - renesas,r9a09g077-wdt - then: - properties: - interrupts: false - interrupt-names: false - else: - required: - - interrupts - - - if: - properties: - compatible: - contains: - const: renesas,r9a09g077-wdt - then: - properties: - resets: false - clock-names: - maxItems: 1 - reg: - minItems: 2 - required: - - clock-names - - power-domains - else: - properties: - reg: - maxItems: 1 - additionalProperties: false examples: diff --git a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml index ef088e0f6917..609e98cdaaff 100644 --- a/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml +++ b/Documentation/devicetree/bindings/watchdog/snps,dw-wdt.yaml @@ -28,6 +28,7 @@ properties: - rockchip,rk3328-wdt - rockchip,rk3368-wdt - rockchip,rk3399-wdt + - rockchip,rk3506-wdt - rockchip,rk3562-wdt - rockchip,rk3568-wdt - rockchip,rk3576-wdt diff --git a/Documentation/devicetree/bindings/watchdog/ti,omap2-wdt.yaml b/Documentation/devicetree/bindings/watchdog/ti,omap2-wdt.yaml new file mode 100644 index 000000000000..913b55222f29 --- /dev/null +++ b/Documentation/devicetree/bindings/watchdog/ti,omap2-wdt.yaml @@ -0,0 +1,51 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/watchdog/ti,omap2-wdt.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: TI OMAP Watchdog Timer Controller + +maintainers: + - Aaro Koskinen <aaro.koskinen@iki.fi> + +allOf: + - $ref: watchdog.yaml# + +properties: + compatible: + oneOf: + - enum: + - ti,omap2-wdt + - ti,omap3-wdt + - items: + - enum: + - ti,am4372-wdt + - ti,omap4-wdt + - ti,omap5-wdt + - const: ti,omap3-wdt + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + + ti,hwmods: + description: Name of the hardware module associated with the watchdog. + $ref: /schemas/types.yaml#/definitions/string + deprecated: true + +required: + - compatible + - reg + +unevaluatedProperties: false + +examples: + - | + watchdog@48314000 { + compatible = "ti,omap3-wdt"; + reg = <0x48314000 0x80>; + ti,hwmods = "wd_timer2"; + }; diff --git a/Documentation/devicetree/bindings/watchdog/watchdog.yaml b/Documentation/devicetree/bindings/watchdog/watchdog.yaml index f0a584af1223..77ac23516d6d 100644 --- a/Documentation/devicetree/bindings/watchdog/watchdog.yaml +++ b/Documentation/devicetree/bindings/watchdog/watchdog.yaml @@ -21,9 +21,10 @@ select: properties: $nodename: - pattern: "^(timer|watchdog)(@.*|-([0-9]|[1-9][0-9]+))?$" + pattern: "^(pmic|timer|watchdog)(@.*|-([0-9]|[1-9][0-9]+))?$" timeout-sec: + maxItems: 1 description: Contains the watchdog timeout in seconds. diff --git a/Documentation/driver-api/hw-recoverable-errors.rst b/Documentation/driver-api/hw-recoverable-errors.rst new file mode 100644 index 000000000000..fc526c3454bd --- /dev/null +++ b/Documentation/driver-api/hw-recoverable-errors.rst @@ -0,0 +1,60 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================================================= +Recoverable Hardware Error Tracking in vmcoreinfo +================================================= + +Overview +-------- + +This feature provides a generic infrastructure within the Linux kernel to track +and log recoverable hardware errors. These are hardware recoverable errors +visible that might not cause immediate panics but may influence health, mainly +because new code path will be executed in the kernel. + +By recording counts and timestamps of recoverable errors into the vmcoreinfo +crash dump notes, this infrastructure aids post-mortem crash analysis tools in +correlating hardware events with kernel failures. This enables faster triage +and better understanding of root causes, especially in large-scale cloud +environments where hardware issues are common. + +Benefits +-------- + +- Facilitates correlation of hardware recoverable errors with kernel panics or + unusual code paths that lead to system crashes. +- Provides operators and cloud providers quick insights, improving reliability + and reducing troubleshooting time. +- Complements existing full hardware diagnostics without replacing them. + +Data Exposure and Consumption +----------------------------- + +- The tracked error data consists of per-error-type counts and timestamps of + last occurrence. +- This data is stored in the `hwerror_data` array, categorized by error source + types like CPU, memory, PCI, CXL, and others. +- It is exposed via vmcoreinfo crash dump notes and can be read using tools + like `crash`, `drgn`, or other kernel crash analysis utilities. +- There is no other way to read these data other than from crash dumps. +- These errors are divided by area, which includes CPU, Memory, PCI, CXL and + others. + +Typical usage example (in drgn REPL): + +.. code-block:: python + + >>> prog['hwerror_data'] + (struct hwerror_info[HWERR_RECOV_MAX]){ + { + .count = (int)844, + .timestamp = (time64_t)1752852018, + }, + ... + } + +Enabling +-------- + +- This feature is enabled when CONFIG_VMCORE_INFO is set. + diff --git a/Documentation/driver-api/index.rst b/Documentation/driver-api/index.rst index baff96b5cf0b..1833e6a0687e 100644 --- a/Documentation/driver-api/index.rst +++ b/Documentation/driver-api/index.rst @@ -97,6 +97,7 @@ Subsystem-specific APIs gpio/index hsi hte/index + hw-recoverable-errors i2c iio/index infiniband diff --git a/Documentation/driver-api/nvdimm/btt.rst b/Documentation/driver-api/nvdimm/btt.rst index 107395c042ae..2d8269f834bd 100644 --- a/Documentation/driver-api/nvdimm/btt.rst +++ b/Documentation/driver-api/nvdimm/btt.rst @@ -83,7 +83,7 @@ flags, and the remaining form the internal block number. ======== ============================================================= Bit Description ======== ============================================================= -31 - 30 Error and Zero flags - Used in the following way:: +31 - 30 Error and Zero flags - Used in the following way: == == ==================================================== 31 30 Description diff --git a/Documentation/driver-api/pci/index.rst b/Documentation/driver-api/pci/index.rst index a38e475cdbe3..9e1b801d0f74 100644 --- a/Documentation/driver-api/pci/index.rst +++ b/Documentation/driver-api/pci/index.rst @@ -10,6 +10,7 @@ The Linux PCI driver implementer's API guide pci p2pdma + tsm .. only:: subproject and html diff --git a/Documentation/driver-api/pci/tsm.rst b/Documentation/driver-api/pci/tsm.rst new file mode 100644 index 000000000000..232b92bec93f --- /dev/null +++ b/Documentation/driver-api/pci/tsm.rst @@ -0,0 +1,21 @@ +.. SPDX-License-Identifier: GPL-2.0 +.. include:: <isonum.txt> + +======================================================== +PCI Trusted Execution Environment Security Manager (TSM) +======================================================== + +Subsystem Interfaces +==================== + +.. kernel-doc:: include/linux/pci-ide.h + :internal: + +.. kernel-doc:: drivers/pci/ide.c + :export: + +.. kernel-doc:: include/linux/pci-tsm.h + :internal: + +.. kernel-doc:: drivers/pci/tsm.c + :export: diff --git a/Documentation/filesystems/nfs/index.rst b/Documentation/filesystems/nfs/index.rst index 95c2c009874c..a29a212b5b4d 100644 --- a/Documentation/filesystems/nfs/index.rst +++ b/Documentation/filesystems/nfs/index.rst @@ -13,5 +13,6 @@ NFS rpc-cache rpc-server-gss nfs41-server + nfsd-io-modes knfsd-stats reexport diff --git a/Documentation/filesystems/nfs/nfsd-io-modes.rst b/Documentation/filesystems/nfs/nfsd-io-modes.rst new file mode 100644 index 000000000000..0fd6e82478fe --- /dev/null +++ b/Documentation/filesystems/nfs/nfsd-io-modes.rst @@ -0,0 +1,153 @@ +.. SPDX-License-Identifier: GPL-2.0 + +============= +NFSD IO MODES +============= + +Overview +======== + +NFSD has historically always used buffered IO when servicing READ and +WRITE operations. BUFFERED is NFSD's default IO mode, but it is possible +to override that default to use either DONTCACHE or DIRECT IO modes. + +Experimental NFSD debugfs interfaces are available to allow the NFSD IO +mode used for READ and WRITE to be configured independently. See both: + +- /sys/kernel/debug/nfsd/io_cache_read +- /sys/kernel/debug/nfsd/io_cache_write + +The default value for both io_cache_read and io_cache_write reflects +NFSD's default IO mode (which is NFSD_IO_BUFFERED=0). + +Based on the configured settings, NFSD's IO will either be: + +- cached using page cache (NFSD_IO_BUFFERED=0) +- cached but removed from page cache on completion (NFSD_IO_DONTCACHE=1) +- not cached stable_how=NFS_UNSTABLE (NFSD_IO_DIRECT=2) + +To set an NFSD IO mode, write a supported value (0 - 2) to the +corresponding IO operation's debugfs interface, e.g.:: + + echo 2 > /sys/kernel/debug/nfsd/io_cache_read + echo 2 > /sys/kernel/debug/nfsd/io_cache_write + +To check which IO mode NFSD is using for READ or WRITE, simply read the +corresponding IO operation's debugfs interface, e.g.:: + + cat /sys/kernel/debug/nfsd/io_cache_read + cat /sys/kernel/debug/nfsd/io_cache_write + +If you experiment with NFSD's IO modes on a recent kernel and have +interesting results, please report them to linux-nfs@vger.kernel.org + +NFSD DONTCACHE +============== + +DONTCACHE offers a hybrid approach to servicing IO that aims to offer +the benefits of using DIRECT IO without any of the strict alignment +requirements that DIRECT IO imposes. To achieve this buffered IO is used +but the IO is flagged to "drop behind" (meaning associated pages are +dropped from the page cache) when IO completes. + +DONTCACHE aims to avoid what has proven to be a fairly significant +limition of Linux's memory management subsystem if/when large amounts of +data is infrequently accessed (e.g. read once _or_ written once but not +read until much later). Such use-cases are particularly problematic +because the page cache will eventually become a bottleneck to servicing +new IO requests. + +For more context on DONTCACHE, please see these Linux commit headers: + +- Overview: 9ad6344568cc3 ("mm/filemap: change filemap_create_folio() + to take a struct kiocb") +- for READ: 8026e49bff9b1 ("mm/filemap: add read support for + RWF_DONTCACHE") +- for WRITE: 974c5e6139db3 ("xfs: flag as supporting FOP_DONTCACHE") + +NFSD_IO_DONTCACHE will fall back to NFSD_IO_BUFFERED if the underlying +filesystem doesn't indicate support by setting FOP_DONTCACHE. + +NFSD DIRECT +=========== + +DIRECT IO doesn't make use of the page cache, as such it is able to +avoid the Linux memory management's page reclaim scalability problems +without resorting to the hybrid use of page cache that DONTCACHE does. + +Some workloads benefit from NFSD avoiding the page cache, particularly +those with a working set that is significantly larger than available +system memory. The pathological worst-case workload that NFSD DIRECT has +proven to help most is: NFS client issuing large sequential IO to a file +that is 2-3 times larger than the NFS server's available system memory. +The reason for such improvement is NFSD DIRECT eliminates a lot of work +that the memory management subsystem would otherwise be required to +perform (e.g. page allocation, dirty writeback, page reclaim). When +using NFSD DIRECT, kswapd and kcompactd are no longer commanding CPU +time trying to find adequate free pages so that forward IO progress can +be made. + +The performance win associated with using NFSD DIRECT was previously +discussed on linux-nfs, see: +https://lore.kernel.org/linux-nfs/aEslwqa9iMeZjjlV@kernel.org/ + +But in summary: + +- NFSD DIRECT can significantly reduce memory requirements +- NFSD DIRECT can reduce CPU load by avoiding costly page reclaim work +- NFSD DIRECT can offer more deterministic IO performance + +As always, your mileage may vary and so it is important to carefully +consider if/when it is beneficial to make use of NFSD DIRECT. When +assessing comparative performance of your workload please be sure to log +relevant performance metrics during testing (e.g. memory usage, cpu +usage, IO performance). Using perf to collect perf data that may be used +to generate a "flamegraph" for work Linux must perform on behalf of your +test is a really meaningful way to compare the relative health of the +system and how switching NFSD's IO mode changes what is observed. + +If NFSD_IO_DIRECT is specified by writing 2 (or 3 and 4 for WRITE) to +NFSD's debugfs interfaces, ideally the IO will be aligned relative to +the underlying block device's logical_block_size. Also the memory buffer +used to store the READ or WRITE payload must be aligned relative to the +underlying block device's dma_alignment. + +But NFSD DIRECT does handle misaligned IO in terms of O_DIRECT as best +it can: + +Misaligned READ: + If NFSD_IO_DIRECT is used, expand any misaligned READ to the next + DIO-aligned block (on either end of the READ). The expanded READ is + verified to have proper offset/len (logical_block_size) and + dma_alignment checking. + +Misaligned WRITE: + If NFSD_IO_DIRECT is used, split any misaligned WRITE into a start, + middle and end as needed. The large middle segment is DIO-aligned + and the start and/or end are misaligned. Buffered IO is used for the + misaligned segments and O_DIRECT is used for the middle DIO-aligned + segment. DONTCACHE buffered IO is _not_ used for the misaligned + segments because using normal buffered IO offers significant RMW + performance benefit when handling streaming misaligned WRITEs. + +Tracing: + The nfsd_read_direct trace event shows how NFSD expands any + misaligned READ to the next DIO-aligned block (on either end of the + original READ, as needed). + + This combination of trace events is useful for READs:: + + echo 1 > /sys/kernel/tracing/events/nfsd/nfsd_read_vector/enable + echo 1 > /sys/kernel/tracing/events/nfsd/nfsd_read_direct/enable + echo 1 > /sys/kernel/tracing/events/nfsd/nfsd_read_io_done/enable + echo 1 > /sys/kernel/tracing/events/xfs/xfs_file_direct_read/enable + + The nfsd_write_direct trace event shows how NFSD splits a given + misaligned WRITE into a DIO-aligned middle segment. + + This combination of trace events is useful for WRITEs:: + + echo 1 > /sys/kernel/tracing/events/nfsd/nfsd_write_opened/enable + echo 1 > /sys/kernel/tracing/events/nfsd/nfsd_write_direct/enable + echo 1 > /sys/kernel/tracing/events/nfsd/nfsd_write_io_done/enable + echo 1 > /sys/kernel/tracing/events/xfs/xfs_file_direct_write/enable diff --git a/Documentation/filesystems/nfs/nfsd-maintainer-entry-profile.rst b/Documentation/filesystems/nfs/nfsd-maintainer-entry-profile.rst new file mode 100644 index 000000000000..4d6b57dbab2a --- /dev/null +++ b/Documentation/filesystems/nfs/nfsd-maintainer-entry-profile.rst @@ -0,0 +1,547 @@ +NFSD Maintainer Entry Profile +============================= + +A Maintainer Entry Profile supplements the top-level process +documents (found in Documentation/process/) with customs that are +specific to a subsystem and its maintainers. A contributor may use +this document to set their expectations and avoid common mistakes. +A maintainer may use these profiles to look across subsystems for +opportunities to converge on best common practices. + +Overview +-------- +The Network File System (NFS) is a standardized family of network +protocols that enable access to files across a set of network- +connected peer hosts. Applications on NFS clients access files that +reside on file systems that are shared by NFS servers. A single +network peer can act as both an NFS client and an NFS server. + +NFSD refers to the NFS server implementation included in the Linux +kernel. An in-kernel NFS server has fast access to files stored +in file systems local to that server. NFSD can share files stored +on most of the file system types native to Linux, including xfs, +ext4, btrfs, and tmpfs. + +Mailing list +------------ +The linux-nfs@vger.kernel.org mailing list is a public list. Its +purpose is to enable collaboration among developers working on the +Linux NFS stack, both client and server. It is not a place for +conversations that are not related directly to the Linux NFS stack. + +The linux-nfs mailing list is archived on `lore.kernel.org <https://lore.kernel.org/linux-nfs/>`_. + +The Linux NFS community does not have any chat room. + +Reporting bugs +-------------- +If you experience an NFSD-related bug on a distribution-built +kernel, please start by working with your Linux distributor. + +Bug reports against upstream Linux code bases are welcome on the +linux-nfs@vger.kernel.org mailing list, where some active triage +can be done. NFSD bugs may also be reported in the Linux kernel +community's bugzilla at: + + https://bugzilla.kernel.org + +Please file NFSD-related bugs under the "Filesystems/NFSD" +component. In general, including as much detail as possible is a +good start, including pertinent system log messages from both +the client and server. + +User space software related to NFSD, such as mountd or the exportfs +command, is contained in the nfs-utils package. Report problems +with those components to linux-nfs@vger.kernel.org. You might be +directed to move the report to a specific bug tracker. + +Contributor's Guide +------------------- + +Standards compliance +~~~~~~~~~~~~~~~~~~~~ +The priority is for NFSD to interoperate fully with the Linux NFS +client. We also test against other popular NFS client implementa- +tions regularly at NFS bake-a-thon events (also known as plug- +fests). Non-Linux NFS clients are not part of upstream NFSD CI/CD. + +The NFSD community strives to provide an NFS server implementation +that interoperates with all standards-compliant NFS client +implementations. This is done by staying as close as is sensible to +the normative mandates in the IETF's published NFS, RPC, and GSS-API +standards. + +It is always useful to reference an RFC and section number in a code +comment where behavior deviates from the standard (and even when the +behavior is compliant but the implementation is obfuscatory). + +On the rare occasion when a deviation from standard-mandated +behavior is needed, brief documentation of the use case or +deficiencies in the standard is a required part of in-code +documentation. + +Care must always be taken to avoid leaking local error codes (ie, +errnos) to clients of NFSD. A proper NFS status code is always +required in NFS protocol replies. + +NFSD administrative interfaces +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +NFSD administrative interfaces include: + +- an NFSD or SUNRPC module parameter + +- export options in /etc/exports + +- files under /proc/fs/nfsd/ or /proc/sys/sunrpc/ + +- the NFSD netlink protocol + +Frequently, a request is made to introduce or modify one of NFSD's +traditional administrative interfaces. Certainly it is technically +easy to introduce a new administrative setting. However, there are +good reasons why the NFSD maintainers prefer to leave that as a last +resort: + +- As with any API, administrative interfaces are difficult to get + right. + +- Once they are documented and have a legacy of use, administrative + interfaces become difficult to modify or remove. + +- Every new administrative setting multiplies the NFSD test matrix. + +- The cost of one administrative interface is incremental, but costs + add up across all of the existing interfaces. + +It is often better for everyone if effort is made up front to +understanding the underlying requirement of the new setting, and +then trying to make it tune itself (or to become otherwise +unnecessary). + +If a new setting is indeed necessary, first consider adding it to +the NFSD netlink protocol. Or if it doesn't need to be a reliable +long term user space feature, it can be added to NFSD's menagerie of +experimental settings which reside under /sys/kernel/debug/nfsd/ . + +Field observability +~~~~~~~~~~~~~~~~~~~ +NFSD employs several different mechanisms for observing operation, +including counters, printks, WARNings, and static trace points. Each +have their strengths and weaknesses. Contributors should select the +most appropriate tool for their task. + +- BUG must be avoided if at all possible, as it will frequently + result in a full system crash. + +- WARN is appropriate only when a full stack trace is useful. + +- printk can show detailed information. These must not be used + in code paths where they can be triggered repeatedly by remote + users. + +- dprintk can show detailed information, but can be enabled only + in pre-set groups. The overhead of emitting output makes dprintk + inappropriate for frequent operations like I/O. + +- Counters are always on, but provide little information about + individual events other than how frequently they occur. + +- static trace points can be enabled individually or in groups + (via a glob). These are generally low overhead, and thus are + favored for use in hot paths. + +- dynamic tracing, such as kprobes or eBPF, are quite flexible but + cannot be used in certain environments (eg, full kernel lock- + down). + +Testing +~~~~~~~ +The kdevops project + + https://github.com/linux-kdevops/kdevops + +contains several NFS-specific workflows, as well as the community +standard fstests suite. These workflows are based on open source +testing tools such as ltp and fio. Contributors are encouraged to +use these tools without kdevops, or contributors should install and +use kdevops themselves to verify their patches before submission. + +Coding style +~~~~~~~~~~~~ +Follow the coding style preferences described in + + Documentation/process/coding-style.rst + +with the following exceptions: + +- Add new local variables to a function in reverse Christmas tree + order + +- Use the kdoc comment style for + + non-static functions + + static inline functions + + static functions that are callbacks/virtual functions + +- All new function names start with ``nfsd_`` for non-NFS-version- + specific functions. + +- New function names that are specific to NFSv2 or NFSv3, or are + used by all minor versions of NFSv4, use ``nfsdN_`` where N is + the version. + +- New function names specific to an NFSv4 minor version can be + named with ``nfsd4M_`` where M is the minor version. + +Patch preparation +~~~~~~~~~~~~~~~~~ +Read and follow all guidelines in + + Documentation/process/submitting-patches.rst + +Use tagging to identify all patch authors. However, reviewers and +testers should be added by replying to the email patch submission. +Email is extensively used in order to publicly archive review and +testing attributions. These tags are automatically inserted into +your patches when they are applied. + +The code in the body of the diff already shows /what/ is being +changed. Thus it is not necessary to repeat that in the patch +description. Instead, the description should contain one or more +of: + +- A brief problem statement ("what is this patch trying to fix?") + with a root-cause analysis. + +- End-user visible symptoms or items that a support engineer might + use to search for the patch, like stack traces. + +- A brief explanation of why the patch is the best way to address + the problem. + +- Any context that reviewers might need to understand the changes + made by the patch. + +- Any relevant benchmarking results, and/or functional test results. + +As detailed in Documentation/process/submitting-patches.rst, +identify the point in history that the issue being addressed was +introduced by using a Fixes: tag. + +Mention in the patch description if that point in history cannot be +determined -- that is, no Fixes: tag can be provided. In this case, +please make it clear to maintainers whether an LTS backport is +needed even though there is no Fixes: tag. + +The NFSD maintainers prefer to add stable tagging themselves, after +public discussion in response to the patch submission. Contributors +may suggest stable tagging, but be aware that many version +management tools add such stable Cc's when you post your patches. +Don't add "Cc: stable" unless you are absolutely sure the patch +needs to go to stable during the initial submission process. + +Patch submission +~~~~~~~~~~~~~~~~ +Patches to NFSD are submitted via the kernel's email-based review +process that is common to most other kernel subsystems. + +Just before each submission, rebase your patch or series on the +nfsd-testing branch at + + https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux.git + +The NFSD subsystem is maintained separately from the Linux in-kernel +NFS client. The NFSD maintainers do not normally take submissions +for client changes, nor can they respond authoritatively to bug +reports or feature requests for NFS client code. + +This means that contributors might be asked to resubmit patches if +they were emailed to the incorrect set of maintainers and reviewers. +This is not a rejection, but simply a correction of the submission +process. + +When in doubt, consult the NFSD entry in the MAINTAINERS file to +see which files and directories fall under the NFSD subsystem. + +The proper set of email addresses for NFSD patches are: + +To: the NFSD maintainers and reviewers listed in MAINTAINERS +Cc: linux-nfs@vger.kernel.org and optionally linux-kernel@ + +If there are other subsystems involved in the patches (for example +MM or RDMA) their primary mailing list address can be included in +the Cc: field. Other contributors and interested parties may be +included there as well. + +In general we prefer that contributors use common patch email tools +such as "git send-email" or "stg email format/send", which tend to +get the details right without a lot of fuss. + +A series consisting of a single patch is not required to have a +cover letter. However, a cover letter can be included if there is +substantial context that is not appropriate to include in the +patch description. + +Please note that, with an e-mail based submission process, series +cover letters are not part of the work that is committed to the +kernel source code base or its commit history. Therefore always try +to keep pertinent information in the patch descriptions. + +Design documentation is welcome, but as cover letters are not +preserved, a perhaps better option is to include a patch that adds +such documentation under Documentation/filesystems/nfs/. + +Reviewers will ask about test coverage and what use cases the +patches are expected to address. Please be prepared to answer these +questions. + +Review comments from maintainers might be politely stated, but in +general, these are not optional to address when they are actionable. +If necessary, the maintainers retain the right to not apply patches +when contributors refuse to address reasonable requests. + +Post changes to kernel source code and user space source code as +separate series. You can connect the two series with comments in +your cover letters. + +Generally the NFSD maintainers ask for a reposts even for simple +modifications in order to publicly archive the request and the +resulting repost before it is pulled into the NFSD trees. This +also enables us to rebuild a patch series quickly without missing +changes that might have been discussed via email. + +Avoid frequently reposting large series with only small changes. As +a rule of thumb, posting substantial changes more than once a week +will result in reviewer overload. + +Remember, there are only a handful of subsystem maintainers and +reviewers, but potentially many sources of contributions. The +maintainers and reviewers, therefore, are always the less scalable +resource. Be kind to your friendly neighborhood maintainer. + +Patch Acceptance +~~~~~~~~~~~~~~~~ +There isn't a formal review process for NFSD, but we like to see +at least two Reviewed-by: notices for patches that are more than +simple clean-ups. Reviews are done in public on +linux-nfs@vger.kernel.org and are archived on lore.kernel.org. + +Currently the NFSD patch queues are maintained in branches here: + + https://git.kernel.org/pub/scm/linux/kernel/git/cel/linux.git + +The NFSD maintainers apply patches initially to the nfsd-testing +branch, which is always open to new submissions. Patches can be +applied while review is ongoing. nfsd-testing is a topic branch, +so it can change frequently, it will be rebased, and your patch +might get dropped if there is a problem with it. + +Generally a script-generated "thank you" email will indicate when +your patch has been added to the nfsd-testing branch. You can track +the progress of your patch using the linux-nfs patchworks instance: + + https://patchwork.kernel.org/project/linux-nfs/list/ + +While your patch is in nfsd-testing, it is exposed to a variety of +test environments, including community zero-day bots, static +analysis tools, and NFSD continuous integration testing. The soak +period is three to four weeks. + +Each patch that survives in nfsd-testing for the soak period without +changes is moved to the nfsd-next branch. + +The nfsd-next branch is automatically merged into linux-next and +fs-next on a nightly basis. + +Patches that survive in nfsd-next are included in the next NFSD +merge window pull request. These windows typically occur once every +63 days (nine weeks). + +When the upstream merge window closes, the nfsd-next branch is +renamed nfsd-fixes, and a new nfsd-next branch is created, based on +the upstream -rc1 tag. + +Fixes that are destined for an upstream -rc release also run the +nfsd-testing gauntlet, but are then applied to the nfsd-fixes +branch. That branch is made available for Linus to pull after a +short time. In order to limit the risk of introducing regressions, +we limit such fixes to emergency situations or fixes to breakage +that occurred during the most recent upstream merge. + +Please make it clear when submitting an emergency patch that +immediate action (either application to -rc or LTS backport) is +needed. + +Sensitive patch submissions and bug reports +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +CVEs are generated by specific members of the Linux kernel community +and several external entities. The Linux NFS community does not emit +or assign CVEs. CVEs are assigned after an issue and its fix are +known. + +However, the NFSD maintainers sometimes receive sensitive security +reports, and at times these are significant enough to need to be +embargoed. In such rare cases, fixes can be developed and reviewed +out of the public eye. + +Please be aware that many version management tools add the stable +Cc's when you post your patches. This is generally a nuisance, but +it can result in outing an embargoed security issue accidentally. +Don't add "Cc: stable" unless you are absolutely sure the patch +needs to go to stable@ during the initial submission process. + +Patches that are merged without ever appearing on any list, and +which carry a Reported-by: or Fixes: tag are detected as suspicious +by security-focused people. We encourage that, after any private +review, security-sensitive patches should be posted to linux-nfs@ +for the usual public review, archiving, and test period. + +LLM-generated submissions +~~~~~~~~~~~~~~~~~~~~~~~~~ +The Linux kernel community as a whole is still exploring the new +world of LLM-generated code. The NFSD maintainers will entertain +submission of patches that are partially or wholly generated by +LLM-based development tools. Such submissions are held to the +same standards as submissions created entirely by human authors: + +- The human contributor identifies themselves via a Signed-off-by: + tag. This tag counts as a DoC. + +- The human contributor is solely responsible for code provenance + and any contamination by inadvertently-included code with a + conflicting license, as usual. + +- The human contributor must be able to answer and address review + questions. A patch description such as "This fixed my problem + but I don't know why" is not acceptable. + +- The contribution is subjected to the same test regimen as all + other submissions. + +- An indication (via a Generated-by: tag or otherwise) that the + contribution is LLM-generated is not required. + +It is easy to address review comments and fix requests in LLM +generated code. So easy, in fact, that it becomes tempting to repost +refreshed code immediately. Please resist that temptation. + +As always, please avoid reposting series revisions more than once +every 24 hours. + +Clean-up patches +~~~~~~~~~~~~~~~~ +The NFSD maintainers discourage patches which perform simple clean- +ups, which are not in the context of other work. For example: + +* Addressing ``checkpatch.pl`` warnings after merge +* Addressing :ref:`Local variable ordering<rcs>` issues +* Addressing long-standing whitespace damage + +This is because it is felt that the churn that such changes produce +comes at a greater cost than the value of such clean-ups. + +Conversely, spelling and grammar fixes are encouraged. + +Stable and LTS support +---------------------- +Upstream NFSD continuous integration testing runs against LTS trees +whenever they are updated. + +Please indicate when a patch containing a fix needs to be considered +for LTS kernels, either via a Fixes: tag or explicit mention. + +Feature requests +---------------- +There is no one way to make an official feature request, but +discussion about the request should eventually make its way to +the linux-nfs@vger.kernel.org mailing list for public review by +the community. + +Subsystem boundaries +~~~~~~~~~~~~~~~~~~~~ +NFSD itself is not much more than a protocol engine. This means its +primary responsibility is to translate the NFS protocol into API +calls in the Linux kernel. For example, NFSD is not responsible for +knowing exactly how bytes or file attributes are managed on a block +device. It relies on other kernel subsystems for that. + +If the subsystems on which NFSD relies do not implement a particular +feature, even if the standard NFS protocols do support that feature, +that usually means NFSD cannot provide that feature without +substantial development work in other areas of the kernel. + +Specificity +~~~~~~~~~~~ +Feature requests can come from anywhere, and thus can often be +nebulous. A requester might not understand what a "use case" or +"user story" is. These descriptive paradigms are often used by +developers and architects to understand what is required of a +design, but are terms of art in the software trade, not used in +the everyday world. + +In order to prevent contributors and maintainers from becoming +overwhelmed, we won't be afraid of saying "no" politely to +underspecified requests. + +Community roles and their authority +----------------------------------- +The purpose of Linux subsystem communities is to provide expertise +and active stewardship of a narrow set of source files in the Linux +kernel. This can include managing user space tooling as well. + +To contextualize the structure of the Linux NFS community that +is responsible for stewardship of the NFS server code base, we +define the community roles here. + +- **Contributor** : Anyone who submits a code change, bug fix, + recommendation, documentation fix, and so on. A contributor can + submit regularly or infrequently. + +- **Outside Contributor** : A contributor who is not a regular actor + in the Linux NFS community. This can mean someone who contributes + to other parts of the kernel, or someone who just noticed a + misspelling in a comment and sent a patch. + +- **Reviewer** : Someone who is named in the MAINTAINERS file as a + reviewer is an area expert who can request changes to contributed + code, and expects that contributors will address the request. + +- **External Reviewer** : Someone who is not named in the + MAINTAINERS file as a reviewer, but who is an area expert. + Examples include Linux kernel contributors with networking, + security, or persistent storage expertise, or developers who + contribute primarily to other NFS implementations. + +One or more people will take on the following roles. These people +are often generically referred to as "maintainers", and are +identified in the MAINTAINERS file with the "M:" tag under the NFSD +subsystem. + +- **Upstream Release Manager** : This role is responsible for + curating contributions into a branch, reviewing test results, and + then sending a pull request during merge windows. There is a + trust relationship between the release manager and Linus. + +- **Bug Triager** : Someone who is a first responder to bug reports + submitted to the linux-nfs mailing list or bug trackers, and helps + troubleshoot and identify next steps. + +- **Security Lead** : The security lead handles contacts from the + security community to resolve immediate issues, as well as dealing + with long-term security issues such as supply chain concerns. For + upstream, that's usually whether contributions violate licensing + or other intellectual property agreements. + +- **Testing Lead** : The testing lead builds and runs the test + infrastructure for the subsystem. The testing lead may ask for + patches to be dropped because of ongoing high defect rates. + +- **LTS Maintainer** : The LTS maintainer is responsible for managing + the Fixes: and Cc: stable annotations on patches, and seeing that + patches that cannot be automatically applied to LTS kernels get + proper manual backports as necessary. + +- **Community Manager** : This umpire role can be asked to call balls + and strikes during conflicts, but is also responsible for ensuring + the health of the relationships within the community and for + facilitating discussions on long-term topics such as how to manage + growing technical debt. diff --git a/Documentation/maintainer/maintainer-entry-profile.rst b/Documentation/maintainer/maintainer-entry-profile.rst index d36dd892a78a..6020d188e13d 100644 --- a/Documentation/maintainer/maintainer-entry-profile.rst +++ b/Documentation/maintainer/maintainer-entry-profile.rst @@ -110,5 +110,6 @@ to do something different in the near future. ../process/maintainer-netdev ../driver-api/vfio-pci-device-specific-driver-acceptance ../nvme/feature-and-quirk-policy + ../filesystems/nfs/nfsd-maintainer-entry-profile ../filesystems/xfs/xfs-maintainer-entry-profile ../mm/damon/maintainer-profile diff --git a/Documentation/mm/index.rst b/Documentation/mm/index.rst index ba6a8872849b..7aa2a8886908 100644 --- a/Documentation/mm/index.rst +++ b/Documentation/mm/index.rst @@ -48,6 +48,7 @@ documentation, or deleted if it has served its purpose. hugetlbfs_reserv ksm memory-model + memfd_preservation mmu_notifier multigen_lru numa diff --git a/Documentation/mm/memfd_preservation.rst b/Documentation/mm/memfd_preservation.rst new file mode 100644 index 000000000000..66e0fb6d5ef0 --- /dev/null +++ b/Documentation/mm/memfd_preservation.rst @@ -0,0 +1,23 @@ +.. SPDX-License-Identifier: GPL-2.0-or-later + +========================== +Memfd Preservation via LUO +========================== + +.. kernel-doc:: mm/memfd_luo.c + :doc: Memfd Preservation via LUO + +Memfd Preservation ABI +====================== + +.. kernel-doc:: include/linux/kho/abi/memfd.h + :doc: DOC: memfd Live Update ABI + +.. kernel-doc:: include/linux/kho/abi/memfd.h + :internal: + +See Also +======== + +- :doc:`/core-api/liveupdate` +- :doc:`/core-api/kho/concepts` diff --git a/Documentation/security/landlock.rst b/Documentation/security/landlock.rst index e0fc54aff09e..3e4d4d04cfae 100644 --- a/Documentation/security/landlock.rst +++ b/Documentation/security/landlock.rst @@ -7,7 +7,7 @@ Landlock LSM: kernel documentation ================================== :Author: Mickaël Salaün -:Date: March 2025 +:Date: September 2025 Landlock's goal is to create scoped access-control (i.e. sandboxing). To harden a whole system, this feature should be available to any process, @@ -110,6 +110,12 @@ Filesystem .. kernel-doc:: security/landlock/fs.h :identifiers: +Process credential +------------------ + +.. kernel-doc:: security/landlock/cred.h + :identifiers: + Ruleset and domain ------------------ @@ -128,6 +134,9 @@ makes the reasoning much easier and helps avoid pitfalls. .. kernel-doc:: security/landlock/ruleset.h :identifiers: +.. kernel-doc:: security/landlock/domain.h + :identifiers: + Additional documentation ======================== diff --git a/Documentation/userspace-api/index.rst b/Documentation/userspace-api/index.rst index b8c73be4fb11..8a61ac4c1bf1 100644 --- a/Documentation/userspace-api/index.rst +++ b/Documentation/userspace-api/index.rst @@ -61,6 +61,7 @@ Everything else :maxdepth: 1 ELF + liveupdate netlink/index sysfs-platform_profile vduse diff --git a/Documentation/userspace-api/ioctl/ioctl-number.rst b/Documentation/userspace-api/ioctl/ioctl-number.rst index 7c527a01d1cf..7232b3544cec 100644 --- a/Documentation/userspace-api/ioctl/ioctl-number.rst +++ b/Documentation/userspace-api/ioctl/ioctl-number.rst @@ -385,6 +385,8 @@ Code Seq# Include File Comments 0xB8 01-02 uapi/misc/mrvl_cn10k_dpi.h Marvell CN10K DPI driver 0xB8 all uapi/linux/mshv.h Microsoft Hyper-V /dev/mshv driver <mailto:linux-hyperv@vger.kernel.org> +0xBA 00-0F uapi/linux/liveupdate.h Pasha Tatashin + <mailto:pasha.tatashin@soleen.com> 0xC0 00-0F linux/usb/iowarrior.h 0xCA 00-0F uapi/misc/cxl.h Dead since 6.15 0xCA 10-2F uapi/misc/ocxl.h diff --git a/Documentation/userspace-api/liveupdate.rst b/Documentation/userspace-api/liveupdate.rst new file mode 100644 index 000000000000..41c0473e4f16 --- /dev/null +++ b/Documentation/userspace-api/liveupdate.rst @@ -0,0 +1,20 @@ +.. SPDX-License-Identifier: GPL-2.0 + +================ +Live Update uAPI +================ +:Author: Pasha Tatashin <pasha.tatashin@soleen.com> + +ioctl interface +=============== +.. kernel-doc:: kernel/liveupdate/luo_core.c + :doc: LUO ioctl Interface + +ioctl uAPI +=========== +.. kernel-doc:: include/uapi/linux/liveupdate.h + +See Also +======== + +- :doc:`Live Update Orchestrator </core-api/liveupdate>` diff --git a/MAINTAINERS b/MAINTAINERS index 05db8b2df438..9a3657a40000 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4432,6 +4432,7 @@ F: arch/*/lib/bitops.c F: include/asm-generic/bitops F: include/asm-generic/bitops.h F: include/linux/bitops.h +F: lib/hweight.c F: lib/test_bitops.c F: tools/*/bitops* @@ -11658,7 +11659,7 @@ T: git git://linuxtv.org/media.git F: drivers/media/i2c/hi556.c HYNIX HI846 SENSOR DRIVER -M: Martin Kepplinger <martin.kepplinger@puri.sm> +M: Martin Kepplinger-Novakovic <martink@posteo.de> L: linux-media@vger.kernel.org S: Maintained F: drivers/media/i2c/hi846.c @@ -11743,6 +11744,7 @@ HUNG TASK DETECTOR M: Andrew Morton <akpm@linux-foundation.org> R: Lance Yang <lance.yang@linux.dev> R: Masami Hiramatsu <mhiramat@kernel.org> +R: Petr Mladek <pmladek@suse.com> L: linux-kernel@vger.kernel.org S: Maintained F: include/linux/hung_task.h @@ -13653,6 +13655,7 @@ R: Dai Ngo <Dai.Ngo@oracle.com> R: Tom Talpey <tom@talpey.com> L: linux-nfs@vger.kernel.org S: Supported +P: Documentation/filesystems/nfs/nfsd-maintainer-entry-profile.rst B: https://bugzilla.kernel.org T: git git://git.kernel.org/pub/scm/linux/kernel/git/cel/linux.git F: Documentation/filesystems/nfs/ @@ -13672,6 +13675,10 @@ F: include/uapi/linux/sunrpc/ F: net/sunrpc/ F: tools/net/sunrpc/ +KERNEL NFSD BLOCK and SCSI LAYOUT DRIVER +R: Christoph Hellwig <hch@lst.de> +F: fs/nfsd/blocklayout* + KERNEL PACMAN PACKAGING (in addition to generic KERNEL BUILD) M: Thomas Weißschuh <linux@weissschuh.net> R: Christian Heusel <christian@heusel.eu> @@ -13885,14 +13892,15 @@ F: kernel/kexec* KEXEC HANDOVER (KHO) M: Alexander Graf <graf@amazon.com> M: Mike Rapoport <rppt@kernel.org> -M: Changyuan Lyu <changyuanl@google.com> +M: Pasha Tatashin <pasha.tatashin@soleen.com> +R: Pratyush Yadav <pratyush@kernel.org> L: kexec@lists.infradead.org L: linux-mm@kvack.org S: Maintained F: Documentation/admin-guide/mm/kho.rst F: Documentation/core-api/kho/* F: include/linux/kexec_handover.h -F: kernel/kexec_handover.c +F: kernel/liveupdate/kexec_handover* F: lib/test_kho.c F: tools/testing/selftests/kho/ @@ -14561,6 +14569,22 @@ F: samples/livepatch/ F: scripts/livepatch/ F: tools/testing/selftests/livepatch/ +LIVE UPDATE +M: Pasha Tatashin <pasha.tatashin@soleen.com> +M: Mike Rapoport <rppt@kernel.org> +R: Pratyush Yadav <pratyush@kernel.org> +L: linux-kernel@vger.kernel.org +S: Maintained +F: Documentation/core-api/liveupdate.rst +F: Documentation/mm/memfd_preservation.rst +F: Documentation/userspace-api/liveupdate.rst +F: include/linux/liveupdate.h +F: include/linux/liveupdate/ +F: include/uapi/linux/liveupdate.h +F: kernel/liveupdate/ +F: mm/memfd_luo.c +F: tools/testing/selftests/liveupdate/ + LLC (802.2) L: netdev@vger.kernel.org S: Odd fixes @@ -15662,7 +15686,7 @@ F: include/media/imx.h MEDIA DRIVERS FOR FREESCALE IMX7/8 M: Rui Miguel Silva <rmfrfs@gmail.com> M: Laurent Pinchart <laurent.pinchart@ideasonboard.com> -M: Martin Kepplinger <martin.kepplinger@puri.sm> +M: Martin Kepplinger-Novakovic <martink@posteo.de> R: Purism Kernel Team <kernel@puri.sm> R: Frank Li <Frank.Li@nxp.com> L: imx@lists.linux.dev @@ -18414,10 +18438,11 @@ F: net/sunrpc/ NILFS2 FILESYSTEM M: Ryusuke Konishi <konishi.ryusuke@gmail.com> +M: Viacheslav Dubeyko <slava@dubeyko.com> L: linux-nilfs@vger.kernel.org -S: Supported +S: Maintained W: https://nilfs.sourceforge.io/ -T: git https://github.com/konis/nilfs2.git +T: git git://git.kernel.org/pub/scm/linux/kernel/git/vdubeyko/nilfs2.git F: Documentation/filesystems/nilfs2.rst F: fs/nilfs2/ F: include/trace/events/nilfs2.h @@ -20092,6 +20117,7 @@ Q: https://patchwork.kernel.org/project/linux-pci/list/ B: https://bugzilla.kernel.org C: irc://irc.oftc.net/linux-pci T: git git://git.kernel.org/pub/scm/linux/kernel/git/pci/pci.git +F: Documentation/ABI/testing/sysfs-devices-pci-host-bridge F: Documentation/PCI/ F: Documentation/devicetree/bindings/pci/ F: arch/x86/kernel/early-quirks.c @@ -25096,7 +25122,6 @@ F: drivers/regulator/sy8106a-regulator.c SYNC FILE FRAMEWORK M: Sumit Semwal <sumit.semwal@linaro.org> -R: Gustavo Padovan <gustavo@padovan.org> L: linux-media@vger.kernel.org L: dri-devel@lists.freedesktop.org S: Maintained @@ -26301,7 +26326,7 @@ M: Jarkko Sakkinen <jarkko@kernel.org> R: Jason Gunthorpe <jgg@ziepe.ca> L: linux-integrity@vger.kernel.org S: Maintained -W: https://codeberg.org/jarkko/linux-tpmdd-test +W: https://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd-test.git/about/ Q: https://patchwork.kernel.org/project/linux-integrity/list/ T: git git://git.kernel.org/pub/scm/linux/kernel/git/jarkko/linux-tpmdd.git F: Documentation/devicetree/bindings/tpm/ @@ -26391,14 +26416,16 @@ M: David Lechner <dlechner@baylibre.com> S: Maintained F: Documentation/devicetree/bindings/trigger-source/* -TRUSTED SECURITY MODULE (TSM) INFRASTRUCTURE +TRUSTED EXECUTION ENVIRONMENT SECURITY MANAGER (TSM) M: Dan Williams <dan.j.williams@intel.com> L: linux-coco@lists.linux.dev S: Maintained F: Documentation/ABI/testing/configfs-tsm-report F: Documentation/driver-api/coco/ +F: Documentation/driver-api/pci/tsm.rst +F: drivers/pci/tsm.c F: drivers/virt/coco/guest/ -F: include/linux/tsm*.h +F: include/linux/*tsm*.h F: samples/tsm-mr/ TRUSTED SERVICES TEE DRIVER diff --git a/arch/Kconfig b/arch/Kconfig index 61130b88964b..31220f512b16 100644 --- a/arch/Kconfig +++ b/arch/Kconfig @@ -232,17 +232,14 @@ config HAVE_EFFICIENT_UNALIGNED_ACCESS config ARCH_USE_BUILTIN_BSWAP bool help - Modern versions of GCC (since 4.4) have builtin functions - for handling byte-swapping. Using these, instead of the old - inline assembler that the architecture code provides in the - __arch_bswapXX() macros, allows the compiler to see what's - happening and offers more opportunity for optimisation. In - particular, the compiler will be able to combine the byteswap - with a nearby load or store and use load-and-swap or - store-and-swap instructions if the architecture has them. It - should almost *never* result in code which is worse than the - hand-coded assembler in <asm/swab.h>. But just in case it - does, the use of the builtins is optional. + GCC and Clang have builtin functions for handling byte-swapping. + Using these allows the compiler to see what's happening and + offers more opportunity for optimisation. In particular, the + compiler will be able to combine the byteswap with a nearby load + or store and use load-and-swap or store-and-swap instructions if + the architecture has them. It should almost *never* result in code + which is worse than the hand-coded assembler in <asm/swab.h>. + But just in case it does, the use of the builtins is optional. Any architecture with load-and-swap or store-and-swap instructions should set this. And it shouldn't hurt to set it diff --git a/arch/alpha/kernel/pci_iommu.c b/arch/alpha/kernel/pci_iommu.c index dc91de50f906..955b6ca61627 100644 --- a/arch/alpha/kernel/pci_iommu.c +++ b/arch/alpha/kernel/pci_iommu.c @@ -224,28 +224,26 @@ static int pci_dac_dma_supported(struct pci_dev *dev, u64 mask) until either pci_unmap_single or pci_dma_sync_single is performed. */ static dma_addr_t -pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size, +pci_map_single_1(struct pci_dev *pdev, phys_addr_t paddr, size_t size, int dac_allowed) { struct pci_controller *hose = pdev ? pdev->sysdata : pci_isa_hose; dma_addr_t max_dma = pdev ? pdev->dma_mask : ISA_DMA_MASK; + unsigned long offset = offset_in_page(paddr); struct pci_iommu_arena *arena; long npages, dma_ofs, i; - unsigned long paddr; dma_addr_t ret; unsigned int align = 0; struct device *dev = pdev ? &pdev->dev : NULL; - paddr = __pa(cpu_addr); - #if !DEBUG_NODIRECT /* First check to see if we can use the direct map window. */ if (paddr + size + __direct_map_base - 1 <= max_dma && paddr + size <= __direct_map_size) { ret = paddr + __direct_map_base; - DBGA2("pci_map_single: [%p,%zx] -> direct %llx from %ps\n", - cpu_addr, size, ret, __builtin_return_address(0)); + DBGA2("pci_map_single: [%pa,%zx] -> direct %llx from %ps\n", + &paddr, size, ret, __builtin_return_address(0)); return ret; } @@ -255,8 +253,8 @@ pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size, if (dac_allowed) { ret = paddr + alpha_mv.pci_dac_offset; - DBGA2("pci_map_single: [%p,%zx] -> DAC %llx from %ps\n", - cpu_addr, size, ret, __builtin_return_address(0)); + DBGA2("pci_map_single: [%pa,%zx] -> DAC %llx from %ps\n", + &paddr, size, ret, __builtin_return_address(0)); return ret; } @@ -290,10 +288,10 @@ pci_map_single_1(struct pci_dev *pdev, void *cpu_addr, size_t size, arena->ptes[i + dma_ofs] = mk_iommu_pte(paddr); ret = arena->dma_base + dma_ofs * PAGE_SIZE; - ret += (unsigned long)cpu_addr & ~PAGE_MASK; + ret += offset; - DBGA2("pci_map_single: [%p,%zx] np %ld -> sg %llx from %ps\n", - cpu_addr, size, npages, ret, __builtin_return_address(0)); + DBGA2("pci_map_single: [%pa,%zx] np %ld -> sg %llx from %ps\n", + &paddr, size, npages, ret, __builtin_return_address(0)); return ret; } @@ -322,19 +320,18 @@ static struct pci_dev *alpha_gendev_to_pci(struct device *dev) return NULL; } -static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, +static dma_addr_t alpha_pci_map_phys(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, unsigned long attrs) { struct pci_dev *pdev = alpha_gendev_to_pci(dev); int dac_allowed; - BUG_ON(dir == DMA_NONE); + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; - dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0; - return pci_map_single_1(pdev, (char *)page_address(page) + offset, - size, dac_allowed); + dac_allowed = pdev ? pci_dac_dma_supported(pdev, pdev->dma_mask) : 0; + return pci_map_single_1(pdev, phys, size, dac_allowed); } /* Unmap a single streaming mode DMA translation. The DMA_ADDR and @@ -343,7 +340,7 @@ static dma_addr_t alpha_pci_map_page(struct device *dev, struct page *page, the cpu to the buffer are guaranteed to see whatever the device wrote there. */ -static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr, +static void alpha_pci_unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { @@ -353,8 +350,6 @@ static void alpha_pci_unmap_page(struct device *dev, dma_addr_t dma_addr, struct pci_iommu_arena *arena; long dma_ofs, npages; - BUG_ON(dir == DMA_NONE); - if (dma_addr >= __direct_map_base && dma_addr < __direct_map_base + __direct_map_size) { /* Nothing to do. */ @@ -429,7 +424,7 @@ try_again: } memset(cpu_addr, 0, size); - *dma_addrp = pci_map_single_1(pdev, cpu_addr, size, 0); + *dma_addrp = pci_map_single_1(pdev, virt_to_phys(cpu_addr), size, 0); if (*dma_addrp == DMA_MAPPING_ERROR) { free_pages((unsigned long)cpu_addr, order); if (alpha_mv.mv_pci_tbi || (gfp & GFP_DMA)) @@ -643,9 +638,8 @@ static int alpha_pci_map_sg(struct device *dev, struct scatterlist *sg, /* Fast path single entry scatterlists. */ if (nents == 1) { sg->dma_length = sg->length; - sg->dma_address - = pci_map_single_1(pdev, SG_ENT_VIRT_ADDRESS(sg), - sg->length, dac_allowed); + sg->dma_address = pci_map_single_1(pdev, sg_phys(sg), + sg->length, dac_allowed); if (sg->dma_address == DMA_MAPPING_ERROR) return -EIO; return 1; @@ -917,8 +911,8 @@ iommu_unbind(struct pci_iommu_arena *arena, long pg_start, long pg_count) const struct dma_map_ops alpha_pci_ops = { .alloc = alpha_pci_alloc_coherent, .free = alpha_pci_free_coherent, - .map_page = alpha_pci_map_page, - .unmap_page = alpha_pci_unmap_page, + .map_phys = alpha_pci_map_phys, + .unmap_phys = alpha_pci_unmap_phys, .map_sg = alpha_pci_map_sg, .unmap_sg = alpha_pci_unmap_sg, .dma_supported = alpha_pci_supported, diff --git a/arch/arm/Kconfig b/arch/arm/Kconfig index 4fb985b76e97..ff61891abe53 100644 --- a/arch/arm/Kconfig +++ b/arch/arm/Kconfig @@ -1161,8 +1161,6 @@ config AEABI disambiguate both ABIs and allow for backward compatibility support (selected with CONFIG_OABI_COMPAT). - To use this you need GCC version 4.0.0 or later. - config OABI_COMPAT bool "Allow old ABI binaries to run with this kernel (EXPERIMENTAL)" depends on AEABI && !THUMB2_KERNEL diff --git a/arch/arm/configs/aspeed_g5_defconfig b/arch/arm/configs/aspeed_g5_defconfig index 61cee1e7ebea..c3b0d5f06889 100644 --- a/arch/arm/configs/aspeed_g5_defconfig +++ b/arch/arm/configs/aspeed_g5_defconfig @@ -308,7 +308,7 @@ CONFIG_PANIC_ON_OOPS=y CONFIG_PANIC_TIMEOUT=-1 CONFIG_SOFTLOCKUP_DETECTOR=y CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y -CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y +CONFIG_BOOTPARAM_HUNG_TASK_PANIC=1 CONFIG_WQ_WATCHDOG=y # CONFIG_SCHED_DEBUG is not set CONFIG_FUNCTION_TRACER=y diff --git a/arch/arm/mm/dma-mapping.c b/arch/arm/mm/dma-mapping.c index 08641a936394..a4c765d24692 100644 --- a/arch/arm/mm/dma-mapping.c +++ b/arch/arm/mm/dma-mapping.c @@ -624,16 +624,14 @@ static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, kfree(buf); } -static void dma_cache_maint_page(struct page *page, unsigned long offset, - size_t size, enum dma_data_direction dir, +static void dma_cache_maint_page(phys_addr_t phys, size_t size, + enum dma_data_direction dir, void (*op)(const void *, size_t, int)) { - unsigned long pfn; + unsigned long offset = offset_in_page(phys); + unsigned long pfn = __phys_to_pfn(phys); size_t left = size; - pfn = page_to_pfn(page) + offset / PAGE_SIZE; - offset %= PAGE_SIZE; - /* * A single sg entry may refer to multiple physically contiguous * pages. But we still need to process highmem pages individually. @@ -644,17 +642,18 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, size_t len = left; void *vaddr; - page = pfn_to_page(pfn); - - if (PageHighMem(page)) { + phys = __pfn_to_phys(pfn); + if (PhysHighMem(phys)) { if (len + offset > PAGE_SIZE) len = PAGE_SIZE - offset; if (cache_is_vipt_nonaliasing()) { - vaddr = kmap_atomic(page); + vaddr = kmap_atomic_pfn(pfn); op(vaddr + offset, len, dir); kunmap_atomic(vaddr); } else { + struct page *page = phys_to_page(phys); + vaddr = kmap_high_get(page); if (vaddr) { op(vaddr + offset, len, dir); @@ -662,7 +661,8 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, } } } else { - vaddr = page_address(page) + offset; + phys += offset; + vaddr = phys_to_virt(phys); op(vaddr, len, dir); } offset = 0; @@ -676,14 +676,11 @@ static void dma_cache_maint_page(struct page *page, unsigned long offset, * Note: Drivers should NOT use this function directly. * Use the driver DMA support - see dma-mapping.h (dma_sync_*) */ -static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { - phys_addr_t paddr; + dma_cache_maint_page(paddr, size, dir, dmac_map_area); - dma_cache_maint_page(page, off, size, dir, dmac_map_area); - - paddr = page_to_phys(page) + off; if (dir == DMA_FROM_DEVICE) { outer_inv_range(paddr, paddr + size); } else { @@ -692,17 +689,15 @@ static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, /* FIXME: non-speculating: flush on bidirectional mappings? */ } -static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, - size_t size, enum dma_data_direction dir) +void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { - phys_addr_t paddr = page_to_phys(page) + off; - /* FIXME: non-speculating: not required */ /* in any case, don't bother invalidating if DMA to device */ if (dir != DMA_TO_DEVICE) { outer_inv_range(paddr, paddr + size); - dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); + dma_cache_maint_page(paddr, size, dir, dmac_unmap_area); } /* @@ -737,6 +732,9 @@ static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs) if (attrs & DMA_ATTR_PRIVILEGED) prot |= IOMMU_PRIV; + if (attrs & DMA_ATTR_MMIO) + prot |= IOMMU_MMIO; + switch (dir) { case DMA_BIDIRECTIONAL: return prot | IOMMU_READ | IOMMU_WRITE; @@ -1205,7 +1203,7 @@ static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, unsigned int len = PAGE_ALIGN(s->offset + s->length); if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); + arch_sync_dma_for_device(sg_phys(s), s->length, dir); prot = __dma_info_to_prot(dir, attrs); @@ -1307,8 +1305,7 @@ static void arm_iommu_unmap_sg(struct device *dev, __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - __dma_page_dev_to_cpu(sg_page(s), s->offset, - s->length, dir); + arch_sync_dma_for_cpu(sg_phys(s), s->length, dir); } } @@ -1330,7 +1327,7 @@ static void arm_iommu_sync_sg_for_cpu(struct device *dev, return; for_each_sg(sg, s, nents, i) - __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); + arch_sync_dma_for_cpu(sg_phys(s), s->length, dir); } @@ -1352,29 +1349,31 @@ static void arm_iommu_sync_sg_for_device(struct device *dev, return; for_each_sg(sg, s, nents, i) - __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); + arch_sync_dma_for_device(sg_phys(s), s->length, dir); } /** - * arm_iommu_map_page + * arm_iommu_map_phys * @dev: valid struct device pointer - * @page: page that buffer resides in - * @offset: offset into page for start of buffer + * @phys: physical address that buffer resides in * @size: size of buffer to map * @dir: DMA transfer direction + * @attrs: DMA mapping attributes * * IOMMU aware version of arm_dma_map_page() */ -static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) +static dma_addr_t arm_iommu_map_phys(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, unsigned long attrs) { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); + int len = PAGE_ALIGN(size + offset_in_page(phys)); + phys_addr_t addr = phys & PAGE_MASK; dma_addr_t dma_addr; - int ret, prot, len = PAGE_ALIGN(size + offset); + int ret, prot; - if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - __dma_page_cpu_to_dev(page, offset, size, dir); + if (!dev->dma_coherent && + !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) + arch_sync_dma_for_device(phys, size, dir); dma_addr = __alloc_iova(mapping, len); if (dma_addr == DMA_MAPPING_ERROR) @@ -1382,12 +1381,11 @@ static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, prot = __dma_info_to_prot(dir, attrs); - ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, - prot, GFP_KERNEL); + ret = iommu_map(mapping->domain, dma_addr, addr, len, prot, GFP_KERNEL); if (ret < 0) goto fail; - return dma_addr + offset; + return dma_addr + offset_in_page(phys); fail: __free_iova(mapping, dma_addr, len); return DMA_MAPPING_ERROR; @@ -1399,82 +1397,27 @@ fail: * @handle: DMA address of buffer * @size: size of buffer (same as passed to dma_map_page) * @dir: DMA transfer direction (same as passed to dma_map_page) + * @attrs: DMA mapping attributes * - * IOMMU aware version of arm_dma_unmap_page() + * IOMMU aware version of arm_dma_unmap_phys() */ -static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, +static void arm_iommu_unmap_phys(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; - struct page *page; int offset = handle & ~PAGE_MASK; int len = PAGE_ALIGN(size + offset); if (!iova) return; - if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { - page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); - __dma_page_dev_to_cpu(page, offset, size, dir); - } - - iommu_unmap(mapping->domain, iova, len); - __free_iova(mapping, iova, len); -} - -/** - * arm_iommu_map_resource - map a device resource for DMA - * @dev: valid struct device pointer - * @phys_addr: physical address of resource - * @size: size of resource to map - * @dir: DMA transfer direction - */ -static dma_addr_t arm_iommu_map_resource(struct device *dev, - phys_addr_t phys_addr, size_t size, - enum dma_data_direction dir, unsigned long attrs) -{ - struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); - dma_addr_t dma_addr; - int ret, prot; - phys_addr_t addr = phys_addr & PAGE_MASK; - unsigned int offset = phys_addr & ~PAGE_MASK; - size_t len = PAGE_ALIGN(size + offset); - - dma_addr = __alloc_iova(mapping, len); - if (dma_addr == DMA_MAPPING_ERROR) - return dma_addr; - - prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO; - - ret = iommu_map(mapping->domain, dma_addr, addr, len, prot, GFP_KERNEL); - if (ret < 0) - goto fail; - - return dma_addr + offset; -fail: - __free_iova(mapping, dma_addr, len); - return DMA_MAPPING_ERROR; -} + if (!dev->dma_coherent && + !(attrs & (DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_MMIO))) { + phys_addr_t phys = iommu_iova_to_phys(mapping->domain, iova); -/** - * arm_iommu_unmap_resource - unmap a device DMA resource - * @dev: valid struct device pointer - * @dma_handle: DMA address to resource - * @size: size of resource to map - * @dir: DMA transfer direction - */ -static void arm_iommu_unmap_resource(struct device *dev, dma_addr_t dma_handle, - size_t size, enum dma_data_direction dir, - unsigned long attrs) -{ - struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); - dma_addr_t iova = dma_handle & PAGE_MASK; - unsigned int offset = dma_handle & ~PAGE_MASK; - size_t len = PAGE_ALIGN(size + offset); - - if (!iova) - return; + arch_sync_dma_for_cpu(phys + offset, size, dir); + } iommu_unmap(mapping->domain, iova, len); __free_iova(mapping, iova, len); @@ -1485,14 +1428,14 @@ static void arm_iommu_sync_single_for_cpu(struct device *dev, { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; - struct page *page; unsigned int offset = handle & ~PAGE_MASK; + phys_addr_t phys; if (dev->dma_coherent || !iova) return; - page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); - __dma_page_dev_to_cpu(page, offset, size, dir); + phys = iommu_iova_to_phys(mapping->domain, iova); + arch_sync_dma_for_cpu(phys + offset, size, dir); } static void arm_iommu_sync_single_for_device(struct device *dev, @@ -1500,14 +1443,14 @@ static void arm_iommu_sync_single_for_device(struct device *dev, { struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); dma_addr_t iova = handle & PAGE_MASK; - struct page *page; unsigned int offset = handle & ~PAGE_MASK; + phys_addr_t phys; if (dev->dma_coherent || !iova) return; - page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); - __dma_page_cpu_to_dev(page, offset, size, dir); + phys = iommu_iova_to_phys(mapping->domain, iova); + arch_sync_dma_for_device(phys + offset, size, dir); } static const struct dma_map_ops iommu_ops = { @@ -1516,8 +1459,8 @@ static const struct dma_map_ops iommu_ops = { .mmap = arm_iommu_mmap_attrs, .get_sgtable = arm_iommu_get_sgtable, - .map_page = arm_iommu_map_page, - .unmap_page = arm_iommu_unmap_page, + .map_phys = arm_iommu_map_phys, + .unmap_phys = arm_iommu_unmap_phys, .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, .sync_single_for_device = arm_iommu_sync_single_for_device, @@ -1525,9 +1468,6 @@ static const struct dma_map_ops iommu_ops = { .unmap_sg = arm_iommu_unmap_sg, .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, .sync_sg_for_device = arm_iommu_sync_sg_for_device, - - .map_resource = arm_iommu_map_resource, - .unmap_resource = arm_iommu_unmap_resource, }; /** @@ -1794,20 +1734,6 @@ void arch_teardown_dma_ops(struct device *dev) set_dma_ops(dev, NULL); } -void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, - enum dma_data_direction dir) -{ - __dma_page_cpu_to_dev(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), - size, dir); -} - -void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, - enum dma_data_direction dir) -{ - __dma_page_dev_to_cpu(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), - size, dir); -} - void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) { diff --git a/arch/mips/jazz/jazzdma.c b/arch/mips/jazz/jazzdma.c index c97b089b9902..eb9fb2f2a720 100644 --- a/arch/mips/jazz/jazzdma.c +++ b/arch/mips/jazz/jazzdma.c @@ -521,18 +521,24 @@ static void jazz_dma_free(struct device *dev, size_t size, void *vaddr, __free_pages(virt_to_page(vaddr), get_order(size)); } -static dma_addr_t jazz_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) +static dma_addr_t jazz_dma_map_phys(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, unsigned long attrs) { - phys_addr_t phys = page_to_phys(page) + offset; + if (unlikely(attrs & DMA_ATTR_MMIO)) + /* + * This check is included because older versions of the code lacked + * MMIO path support, and my ability to test this path is limited. + * However, from a software technical standpoint, there is no restriction, + * as the following code operates solely on physical addresses. + */ + return DMA_MAPPING_ERROR; if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) arch_sync_dma_for_device(phys, size, dir); return vdma_alloc(phys, size); } -static void jazz_dma_unmap_page(struct device *dev, dma_addr_t dma_addr, +static void jazz_dma_unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) @@ -607,8 +613,8 @@ static void jazz_dma_sync_sg_for_cpu(struct device *dev, const struct dma_map_ops jazz_dma_ops = { .alloc = jazz_dma_alloc, .free = jazz_dma_free, - .map_page = jazz_dma_map_page, - .unmap_page = jazz_dma_unmap_page, + .map_phys = jazz_dma_map_phys, + .unmap_phys = jazz_dma_unmap_phys, .map_sg = jazz_dma_map_sg, .unmap_sg = jazz_dma_unmap_sg, .sync_single_for_cpu = jazz_dma_sync_single_for_cpu, diff --git a/arch/parisc/include/asm/bug.h b/arch/parisc/include/asm/bug.h index 5aa1623e4f2f..5cf35489ad80 100644 --- a/arch/parisc/include/asm/bug.h +++ b/arch/parisc/include/asm/bug.h @@ -2,8 +2,6 @@ #ifndef _PARISC_BUG_H #define _PARISC_BUG_H -#include <linux/kernel.h> /* for BUGFLAG_TAINT */ - /* * Tell the user there is some problem. * The offending file and line are encoded in the __bug_table section. diff --git a/arch/parisc/kernel/asm-offsets.c b/arch/parisc/kernel/asm-offsets.c index 9abfe65492c6..3de4b5933b10 100644 --- a/arch/parisc/kernel/asm-offsets.c +++ b/arch/parisc/kernel/asm-offsets.c @@ -258,6 +258,8 @@ int main(void) BLANK(); DEFINE(TIF_BLOCKSTEP_PA_BIT, 31-TIF_BLOCKSTEP); DEFINE(TIF_SINGLESTEP_PA_BIT, 31-TIF_SINGLESTEP); + DEFINE(TIF_32BIT_PA_BIT, 31-TIF_32BIT); + BLANK(); DEFINE(ASM_PMD_SHIFT, PMD_SHIFT); DEFINE(ASM_PGDIR_SHIFT, PGDIR_SHIFT); diff --git a/arch/parisc/kernel/drivers.c b/arch/parisc/kernel/drivers.c index 1f8936fc2292..8d23fe42b0ce 100644 --- a/arch/parisc/kernel/drivers.c +++ b/arch/parisc/kernel/drivers.c @@ -1043,11 +1043,7 @@ static __init int qemu_print_iodc_data(struct device *lin_dev, void *data) (unsigned char)mod_path.path.bc[3], (unsigned char)mod_path.path.bc[4], (unsigned char)mod_path.path.bc[5]); - pr_cont(".mod = 0x%x ", mod_path.path.mod); - pr_cont(" },\n"); - pr_cont("\t.layers = { 0x%x, 0x%x, 0x%x, 0x%x, 0x%x, 0x%x }\n", - mod_path.layers[0], mod_path.layers[1], mod_path.layers[2], - mod_path.layers[3], mod_path.layers[4], mod_path.layers[5]); + pr_cont(".mod = 0x%x }\n", mod_path.path.mod); pr_cont("};\n"); pr_info("static struct pdc_iodc iodc_data_hpa_%08lx = {\n", hpa); @@ -1067,8 +1063,6 @@ static __init int qemu_print_iodc_data(struct device *lin_dev, void *data) DO(checksum); DO(length); #undef DO - pr_cont("\t/* pad: 0x%04x, 0x%04x */\n", - iodc_data.pad[0], iodc_data.pad[1]); pr_cont("};\n"); pr_info("#define HPA_%08lx_num_addr %d\n", hpa, dev->num_addrs); diff --git a/arch/parisc/kernel/entry.S b/arch/parisc/kernel/entry.S index f4bf61a34701..e04c5d806c10 100644 --- a/arch/parisc/kernel/entry.S +++ b/arch/parisc/kernel/entry.S @@ -1059,8 +1059,6 @@ ENTRY_CFI(intr_save) /* for os_hpmc */ STREG %r17, PT_IOR(%r29) #if defined(CONFIG_64BIT) - b,n intr_save2 - skip_save_ior: /* We have a itlb miss, and when executing code above 4 Gb on ILP64, we * need to adjust iasq/iaoq here in the same way we adjusted isr/ior @@ -1069,10 +1067,17 @@ skip_save_ior: bb,COND(>=),n %r8,PSW_W_BIT,intr_save2 LDREG PT_IASQ0(%r29), %r16 LDREG PT_IAOQ0(%r29), %r17 - /* adjust iasq/iaoq */ + /* adjust iasq0/iaoq0 */ space_adjust %r16,%r17,%r1 STREG %r16, PT_IASQ0(%r29) STREG %r17, PT_IAOQ0(%r29) + + LDREG PT_IASQ1(%r29), %r16 + LDREG PT_IAOQ1(%r29), %r17 + /* adjust iasq1/iaoq1 */ + space_adjust %r16,%r17,%r1 + STREG %r16, PT_IASQ1(%r29) + STREG %r17, PT_IAOQ1(%r29) #else skip_save_ior: #endif @@ -1841,6 +1846,10 @@ syscall_restore_rfi: extru,= %r19,TIF_BLOCKSTEP_PA_BIT,1,%r0 depi -1,7,1,%r20 /* T bit */ +#ifdef CONFIG_64BIT + extru,<> %r19,TIF_32BIT_PA_BIT,1,%r0 + depi -1,4,1,%r20 /* W bit */ +#endif STREG %r20,TASK_PT_PSW(%r1) /* Always store space registers, since sr3 can be changed (e.g. fork) */ @@ -1854,7 +1863,6 @@ syscall_restore_rfi: STREG %r25,TASK_PT_IASQ0(%r1) STREG %r25,TASK_PT_IASQ1(%r1) - /* XXX W bit??? */ /* Now if old D bit is clear, it means we didn't save all registers * on syscall entry, so do that now. This only happens on TRACEME * calls, or if someone attached to us while we were on a syscall. diff --git a/arch/parisc/kernel/perf_regs.c b/arch/parisc/kernel/perf_regs.c index 68458e2f6197..10a1a5f06a18 100644 --- a/arch/parisc/kernel/perf_regs.c +++ b/arch/parisc/kernel/perf_regs.c @@ -27,7 +27,7 @@ u64 perf_reg_value(struct pt_regs *regs, int idx) return regs->ior; case PERF_REG_PARISC_IPSW: /* CR22 */ return regs->ipsw; - }; + } WARN_ON_ONCE((u32)idx >= PERF_REG_PARISC_MAX); return 0; } diff --git a/arch/powerpc/include/asm/crash_reserve.h b/arch/powerpc/include/asm/crash_reserve.h index 6467ce29b1fa..d1b570ddbf98 100644 --- a/arch/powerpc/include/asm/crash_reserve.h +++ b/arch/powerpc/include/asm/crash_reserve.h @@ -5,4 +5,12 @@ /* crash kernel regions are Page size agliged */ #define CRASH_ALIGN PAGE_SIZE +#ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION +static inline bool arch_add_crash_res_to_iomem(void) +{ + return false; +} +#define arch_add_crash_res_to_iomem arch_add_crash_res_to_iomem +#endif + #endif /* _ASM_POWERPC_CRASH_RESERVE_H */ diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h index b410021ad4c6..eafdd63cd6c4 100644 --- a/arch/powerpc/include/asm/iommu.h +++ b/arch/powerpc/include/asm/iommu.h @@ -274,12 +274,12 @@ extern void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl, unsigned long mask, gfp_t flag, int node); extern void iommu_free_coherent(struct iommu_table *tbl, size_t size, void *vaddr, dma_addr_t dma_handle); -extern dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, - struct page *page, unsigned long offset, - size_t size, unsigned long mask, +extern dma_addr_t iommu_map_phys(struct device *dev, struct iommu_table *tbl, + phys_addr_t phys, size_t size, + unsigned long mask, enum dma_data_direction direction, unsigned long attrs); -extern void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle, +extern void iommu_unmap_phys(struct iommu_table *tbl, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, unsigned long attrs); diff --git a/arch/powerpc/kernel/dma-iommu.c b/arch/powerpc/kernel/dma-iommu.c index 0359ab72cd3b..aa3689d61917 100644 --- a/arch/powerpc/kernel/dma-iommu.c +++ b/arch/powerpc/kernel/dma-iommu.c @@ -93,28 +93,26 @@ static void dma_iommu_free_coherent(struct device *dev, size_t size, /* Creates TCEs for a user provided buffer. The user buffer must be * contiguous real kernel storage (not vmalloc). The address passed here - * comprises a page address and offset into that page. The dma_addr_t - * returned will point to the same byte within the page as was passed in. + * is a physical address to that page. The dma_addr_t returned will point + * to the same byte within the page as was passed in. */ -static dma_addr_t dma_iommu_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, +static dma_addr_t dma_iommu_map_phys(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction direction, unsigned long attrs) { - return iommu_map_page(dev, get_iommu_table_base(dev), page, offset, - size, dma_get_mask(dev), direction, attrs); + return iommu_map_phys(dev, get_iommu_table_base(dev), phys, size, + dma_get_mask(dev), direction, attrs); } - -static void dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, +static void dma_iommu_unmap_phys(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, unsigned long attrs) { - iommu_unmap_page(get_iommu_table_base(dev), dma_handle, size, direction, + iommu_unmap_phys(get_iommu_table_base(dev), dma_handle, size, direction, attrs); } - static int dma_iommu_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, unsigned long attrs) @@ -211,8 +209,8 @@ const struct dma_map_ops dma_iommu_ops = { .map_sg = dma_iommu_map_sg, .unmap_sg = dma_iommu_unmap_sg, .dma_supported = dma_iommu_dma_supported, - .map_page = dma_iommu_map_page, - .unmap_page = dma_iommu_unmap_page, + .map_phys = dma_iommu_map_phys, + .unmap_phys = dma_iommu_unmap_phys, .get_required_mask = dma_iommu_get_required_mask, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c index b7dcf07b2499..0ce71310b7d9 100644 --- a/arch/powerpc/kernel/iommu.c +++ b/arch/powerpc/kernel/iommu.c @@ -848,12 +848,12 @@ EXPORT_SYMBOL_GPL(iommu_tce_table_put); /* Creates TCEs for a user provided buffer. The user buffer must be * contiguous real kernel storage (not vmalloc). The address passed here - * comprises a page address and offset into that page. The dma_addr_t - * returned will point to the same byte within the page as was passed in. + * is physical address into that page. The dma_addr_t returned will point + * to the same byte within the page as was passed in. */ -dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, - struct page *page, unsigned long offset, size_t size, - unsigned long mask, enum dma_data_direction direction, +dma_addr_t iommu_map_phys(struct device *dev, struct iommu_table *tbl, + phys_addr_t phys, size_t size, unsigned long mask, + enum dma_data_direction direction, unsigned long attrs) { dma_addr_t dma_handle = DMA_MAPPING_ERROR; @@ -863,7 +863,7 @@ dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, BUG_ON(direction == DMA_NONE); - vaddr = page_address(page) + offset; + vaddr = phys_to_virt(phys); uaddr = (unsigned long)vaddr; if (tbl) { @@ -890,7 +890,7 @@ dma_addr_t iommu_map_page(struct device *dev, struct iommu_table *tbl, return dma_handle; } -void iommu_unmap_page(struct iommu_table *tbl, dma_addr_t dma_handle, +void iommu_unmap_phys(struct iommu_table *tbl, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, unsigned long attrs) { diff --git a/arch/powerpc/platforms/ps3/system-bus.c b/arch/powerpc/platforms/ps3/system-bus.c index afbaabf182d0..f4f3477d3a23 100644 --- a/arch/powerpc/platforms/ps3/system-bus.c +++ b/arch/powerpc/platforms/ps3/system-bus.c @@ -551,18 +551,20 @@ static void ps3_free_coherent(struct device *_dev, size_t size, void *vaddr, /* Creates TCEs for a user provided buffer. The user buffer must be * contiguous real kernel storage (not vmalloc). The address passed here - * comprises a page address and offset into that page. The dma_addr_t - * returned will point to the same byte within the page as was passed in. + * is physical address to that hat page. The dma_addr_t returned will point + * to the same byte within the page as was passed in. */ -static dma_addr_t ps3_sb_map_page(struct device *_dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction direction, - unsigned long attrs) +static dma_addr_t ps3_sb_map_phys(struct device *_dev, phys_addr_t phys, + size_t size, enum dma_data_direction direction, unsigned long attrs) { struct ps3_system_bus_device *dev = ps3_dev_to_system_bus_dev(_dev); int result; dma_addr_t bus_addr; - void *ptr = page_address(page) + offset; + void *ptr = phys_to_virt(phys); + + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; result = ps3_dma_map(dev->d_region, (unsigned long)ptr, size, &bus_addr, @@ -577,8 +579,8 @@ static dma_addr_t ps3_sb_map_page(struct device *_dev, struct page *page, return bus_addr; } -static dma_addr_t ps3_ioc0_map_page(struct device *_dev, struct page *page, - unsigned long offset, size_t size, +static dma_addr_t ps3_ioc0_map_phys(struct device *_dev, phys_addr_t phys, + size_t size, enum dma_data_direction direction, unsigned long attrs) { @@ -586,7 +588,10 @@ static dma_addr_t ps3_ioc0_map_page(struct device *_dev, struct page *page, int result; dma_addr_t bus_addr; u64 iopte_flag; - void *ptr = page_address(page) + offset; + void *ptr = phys_to_virt(phys); + + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; iopte_flag = CBE_IOPTE_M; switch (direction) { @@ -613,7 +618,7 @@ static dma_addr_t ps3_ioc0_map_page(struct device *_dev, struct page *page, return bus_addr; } -static void ps3_unmap_page(struct device *_dev, dma_addr_t dma_addr, +static void ps3_unmap_phys(struct device *_dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction direction, unsigned long attrs) { struct ps3_system_bus_device *dev = ps3_dev_to_system_bus_dev(_dev); @@ -690,8 +695,8 @@ static const struct dma_map_ops ps3_sb_dma_ops = { .map_sg = ps3_sb_map_sg, .unmap_sg = ps3_sb_unmap_sg, .dma_supported = ps3_dma_supported, - .map_page = ps3_sb_map_page, - .unmap_page = ps3_unmap_page, + .map_phys = ps3_sb_map_phys, + .unmap_phys = ps3_unmap_phys, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, .alloc_pages_op = dma_common_alloc_pages, @@ -704,8 +709,8 @@ static const struct dma_map_ops ps3_ioc0_dma_ops = { .map_sg = ps3_ioc0_map_sg, .unmap_sg = ps3_ioc0_unmap_sg, .dma_supported = ps3_dma_supported, - .map_page = ps3_ioc0_map_page, - .unmap_page = ps3_unmap_page, + .map_phys = ps3_ioc0_map_phys, + .unmap_phys = ps3_unmap_phys, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, .alloc_pages_op = dma_common_alloc_pages, diff --git a/arch/powerpc/platforms/pseries/ibmebus.c b/arch/powerpc/platforms/pseries/ibmebus.c index 3436b0af795e..cad2deb7e70d 100644 --- a/arch/powerpc/platforms/pseries/ibmebus.c +++ b/arch/powerpc/platforms/pseries/ibmebus.c @@ -86,17 +86,18 @@ static void ibmebus_free_coherent(struct device *dev, kfree(vaddr); } -static dma_addr_t ibmebus_map_page(struct device *dev, - struct page *page, - unsigned long offset, +static dma_addr_t ibmebus_map_phys(struct device *dev, phys_addr_t phys, size_t size, enum dma_data_direction direction, unsigned long attrs) { - return (dma_addr_t)(page_address(page) + offset); + if (attrs & DMA_ATTR_MMIO) + return DMA_MAPPING_ERROR; + + return (dma_addr_t)(phys_to_virt(phys)); } -static void ibmebus_unmap_page(struct device *dev, +static void ibmebus_unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction direction, @@ -146,8 +147,8 @@ static const struct dma_map_ops ibmebus_dma_ops = { .unmap_sg = ibmebus_unmap_sg, .dma_supported = ibmebus_dma_supported, .get_required_mask = ibmebus_dma_get_required_mask, - .map_page = ibmebus_map_page, - .unmap_page = ibmebus_unmap_page, + .map_phys = ibmebus_map_phys, + .unmap_phys = ibmebus_unmap_phys, }; static int ibmebus_match_path(struct device *dev, const void *data) diff --git a/arch/powerpc/platforms/pseries/vio.c b/arch/powerpc/platforms/pseries/vio.c index ac1d2d2c9a88..18cffac5468f 100644 --- a/arch/powerpc/platforms/pseries/vio.c +++ b/arch/powerpc/platforms/pseries/vio.c @@ -512,18 +512,21 @@ static void vio_dma_iommu_free_coherent(struct device *dev, size_t size, vio_cmo_dealloc(viodev, roundup(size, PAGE_SIZE)); } -static dma_addr_t vio_dma_iommu_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction direction, - unsigned long attrs) +static dma_addr_t vio_dma_iommu_map_phys(struct device *dev, phys_addr_t phys, + size_t size, + enum dma_data_direction direction, + unsigned long attrs) { struct vio_dev *viodev = to_vio_dev(dev); struct iommu_table *tbl = get_iommu_table_base(dev); dma_addr_t ret = DMA_MAPPING_ERROR; + if (unlikely(attrs & DMA_ATTR_MMIO)) + return ret; + if (vio_cmo_alloc(viodev, roundup(size, IOMMU_PAGE_SIZE(tbl)))) goto out_fail; - ret = iommu_map_page(dev, tbl, page, offset, size, dma_get_mask(dev), + ret = iommu_map_phys(dev, tbl, phys, size, dma_get_mask(dev), direction, attrs); if (unlikely(ret == DMA_MAPPING_ERROR)) goto out_deallocate; @@ -536,7 +539,7 @@ out_fail: return DMA_MAPPING_ERROR; } -static void vio_dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, +static void vio_dma_iommu_unmap_phys(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction direction, unsigned long attrs) @@ -544,7 +547,7 @@ static void vio_dma_iommu_unmap_page(struct device *dev, dma_addr_t dma_handle, struct vio_dev *viodev = to_vio_dev(dev); struct iommu_table *tbl = get_iommu_table_base(dev); - iommu_unmap_page(tbl, dma_handle, size, direction, attrs); + iommu_unmap_phys(tbl, dma_handle, size, direction, attrs); vio_cmo_dealloc(viodev, roundup(size, IOMMU_PAGE_SIZE(tbl))); } @@ -605,8 +608,8 @@ static const struct dma_map_ops vio_dma_mapping_ops = { .free = vio_dma_iommu_free_coherent, .map_sg = vio_dma_iommu_map_sg, .unmap_sg = vio_dma_iommu_unmap_sg, - .map_page = vio_dma_iommu_map_page, - .unmap_page = vio_dma_iommu_unmap_page, + .map_phys = vio_dma_iommu_map_phys, + .unmap_phys = vio_dma_iommu_unmap_phys, .dma_supported = dma_iommu_dma_supported, .get_required_mask = dma_iommu_get_required_mask, .mmap = dma_common_mmap, diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c index da0363692528..46ef88bc9c26 100644 --- a/arch/sparc/kernel/iommu.c +++ b/arch/sparc/kernel/iommu.c @@ -260,26 +260,35 @@ static void dma_4u_free_coherent(struct device *dev, size_t size, free_pages((unsigned long)cpu, order); } -static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t sz, - enum dma_data_direction direction, +static dma_addr_t dma_4u_map_phys(struct device *dev, phys_addr_t phys, + size_t sz, enum dma_data_direction direction, unsigned long attrs) { struct iommu *iommu; struct strbuf *strbuf; iopte_t *base; unsigned long flags, npages, oaddr; - unsigned long i, base_paddr, ctx; + unsigned long i, ctx; u32 bus_addr, ret; unsigned long iopte_protection; + if (unlikely(attrs & DMA_ATTR_MMIO)) + /* + * This check is included because older versions of the code + * lacked MMIO path support, and my ability to test this path + * is limited. However, from a software technical standpoint, + * there is no restriction, as the following code operates + * solely on physical addresses. + */ + goto bad_no_ctx; + iommu = dev->archdata.iommu; strbuf = dev->archdata.stc; if (unlikely(direction == DMA_NONE)) goto bad_no_ctx; - oaddr = (unsigned long)(page_address(page) + offset); + oaddr = (unsigned long)(phys_to_virt(phys)); npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; @@ -296,7 +305,6 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, bus_addr = (iommu->tbl.table_map_base + ((base - iommu->page_table) << IO_PAGE_SHIFT)); ret = bus_addr | (oaddr & ~IO_PAGE_MASK); - base_paddr = __pa(oaddr & IO_PAGE_MASK); if (strbuf->strbuf_enabled) iopte_protection = IOPTE_STREAMING(ctx); else @@ -304,8 +312,8 @@ static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, if (direction != DMA_TO_DEVICE) iopte_protection |= IOPTE_WRITE; - for (i = 0; i < npages; i++, base++, base_paddr += IO_PAGE_SIZE) - iopte_val(*base) = iopte_protection | base_paddr; + for (i = 0; i < npages; i++, base++, phys += IO_PAGE_SIZE) + iopte_val(*base) = iopte_protection | phys; return ret; @@ -383,7 +391,7 @@ do_flush_sync: vaddr, ctx, npages); } -static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, +static void dma_4u_unmap_phys(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction, unsigned long attrs) { @@ -753,8 +761,8 @@ static int dma_4u_supported(struct device *dev, u64 device_mask) static const struct dma_map_ops sun4u_dma_ops = { .alloc = dma_4u_alloc_coherent, .free = dma_4u_free_coherent, - .map_page = dma_4u_map_page, - .unmap_page = dma_4u_unmap_page, + .map_phys = dma_4u_map_phys, + .unmap_phys = dma_4u_unmap_phys, .map_sg = dma_4u_map_sg, .unmap_sg = dma_4u_unmap_sg, .sync_single_for_cpu = dma_4u_sync_single_for_cpu, diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index b720b21ccfbd..791f0a76665f 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -352,9 +352,8 @@ static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, free_pages((unsigned long)cpu, order); } -static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t sz, - enum dma_data_direction direction, +static dma_addr_t dma_4v_map_phys(struct device *dev, phys_addr_t phys, + size_t sz, enum dma_data_direction direction, unsigned long attrs) { struct iommu *iommu; @@ -362,18 +361,27 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, struct iommu_map_table *tbl; u64 mask; unsigned long flags, npages, oaddr; - unsigned long i, base_paddr; - unsigned long prot; + unsigned long i, prot; dma_addr_t bus_addr, ret; long entry; + if (unlikely(attrs & DMA_ATTR_MMIO)) + /* + * This check is included because older versions of the code + * lacked MMIO path support, and my ability to test this path + * is limited. However, from a software technical standpoint, + * there is no restriction, as the following code operates + * solely on physical addresses. + */ + goto bad; + iommu = dev->archdata.iommu; atu = iommu->atu; if (unlikely(direction == DMA_NONE)) goto bad; - oaddr = (unsigned long)(page_address(page) + offset); + oaddr = (unsigned long)(phys_to_virt(phys)); npages = IO_PAGE_ALIGN(oaddr + sz) - (oaddr & IO_PAGE_MASK); npages >>= IO_PAGE_SHIFT; @@ -391,7 +399,6 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, bus_addr = (tbl->table_map_base + (entry << IO_PAGE_SHIFT)); ret = bus_addr | (oaddr & ~IO_PAGE_MASK); - base_paddr = __pa(oaddr & IO_PAGE_MASK); prot = HV_PCI_MAP_ATTR_READ; if (direction != DMA_TO_DEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; @@ -403,8 +410,8 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, iommu_batch_start(dev, prot, entry); - for (i = 0; i < npages; i++, base_paddr += IO_PAGE_SIZE) { - long err = iommu_batch_add(base_paddr, mask); + for (i = 0; i < npages; i++, phys += IO_PAGE_SIZE) { + long err = iommu_batch_add(phys, mask); if (unlikely(err < 0L)) goto iommu_map_fail; } @@ -426,7 +433,7 @@ iommu_map_fail: return DMA_MAPPING_ERROR; } -static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, +static void dma_4v_unmap_phys(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction, unsigned long attrs) { @@ -686,8 +693,8 @@ static int dma_4v_supported(struct device *dev, u64 device_mask) static const struct dma_map_ops sun4v_dma_ops = { .alloc = dma_4v_alloc_coherent, .free = dma_4v_free_coherent, - .map_page = dma_4v_map_page, - .unmap_page = dma_4v_unmap_page, + .map_phys = dma_4v_map_phys, + .unmap_phys = dma_4v_unmap_phys, .map_sg = dma_4v_map_sg, .unmap_sg = dma_4v_unmap_sg, .dma_supported = dma_4v_supported, diff --git a/arch/sparc/mm/io-unit.c b/arch/sparc/mm/io-unit.c index d8376f61b4d0..d409cb450de4 100644 --- a/arch/sparc/mm/io-unit.c +++ b/arch/sparc/mm/io-unit.c @@ -94,13 +94,14 @@ static int __init iounit_init(void) subsys_initcall(iounit_init); /* One has to hold iounit->lock to call this */ -static unsigned long iounit_get_area(struct iounit_struct *iounit, unsigned long vaddr, int size) +static dma_addr_t iounit_get_area(struct iounit_struct *iounit, + phys_addr_t phys, int size) { int i, j, k, npages; unsigned long rotor, scan, limit; iopte_t iopte; - npages = ((vaddr & ~PAGE_MASK) + size + (PAGE_SIZE-1)) >> PAGE_SHIFT; + npages = (offset_in_page(phys) + size + (PAGE_SIZE - 1)) >> PAGE_SHIFT; /* A tiny bit of magic ingredience :) */ switch (npages) { @@ -109,7 +110,7 @@ static unsigned long iounit_get_area(struct iounit_struct *iounit, unsigned long default: i = 0x0213; break; } - IOD(("iounit_get_area(%08lx,%d[%d])=", vaddr, size, npages)); + IOD(("%s(%pa,%d[%d])=", __func__, &phys, size, npages)); next: j = (i & 15); rotor = iounit->rotor[j - 1]; @@ -124,7 +125,8 @@ nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan); } i >>= 4; if (!(i & 15)) - panic("iounit_get_area: Couldn't find free iopte slots for (%08lx,%d)\n", vaddr, size); + panic("iounit_get_area: Couldn't find free iopte slots for (%pa,%d)\n", + &phys, size); goto next; } for (k = 1, scan++; k < npages; k++) @@ -132,30 +134,29 @@ nexti: scan = find_next_zero_bit(iounit->bmap, limit, scan); goto nexti; iounit->rotor[j - 1] = (scan < limit) ? scan : iounit->limit[j - 1]; scan -= npages; - iopte = MKIOPTE(__pa(vaddr & PAGE_MASK)); - vaddr = IOUNIT_DMA_BASE + (scan << PAGE_SHIFT) + (vaddr & ~PAGE_MASK); + iopte = MKIOPTE(phys & PAGE_MASK); + phys = IOUNIT_DMA_BASE + (scan << PAGE_SHIFT) + offset_in_page(phys); for (k = 0; k < npages; k++, iopte = __iopte(iopte_val(iopte) + 0x100), scan++) { set_bit(scan, iounit->bmap); sbus_writel(iopte_val(iopte), &iounit->page_table[scan]); } - IOD(("%08lx\n", vaddr)); - return vaddr; + IOD(("%pa\n", &phys)); + return phys; } -static dma_addr_t iounit_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t len, enum dma_data_direction dir, - unsigned long attrs) +static dma_addr_t iounit_map_phys(struct device *dev, phys_addr_t phys, + size_t len, enum dma_data_direction dir, unsigned long attrs) { - void *vaddr = page_address(page) + offset; struct iounit_struct *iounit = dev->archdata.iommu; - unsigned long ret, flags; + unsigned long flags; + dma_addr_t ret; /* XXX So what is maxphys for us and how do drivers know it? */ if (!len || len > 256 * 1024) return DMA_MAPPING_ERROR; spin_lock_irqsave(&iounit->lock, flags); - ret = iounit_get_area(iounit, (unsigned long)vaddr, len); + ret = iounit_get_area(iounit, phys, len); spin_unlock_irqrestore(&iounit->lock, flags); return ret; } @@ -171,14 +172,15 @@ static int iounit_map_sg(struct device *dev, struct scatterlist *sgl, int nents, /* FIXME: Cache some resolved pages - often several sg entries are to the same page */ spin_lock_irqsave(&iounit->lock, flags); for_each_sg(sgl, sg, nents, i) { - sg->dma_address = iounit_get_area(iounit, (unsigned long) sg_virt(sg), sg->length); + sg->dma_address = + iounit_get_area(iounit, sg_phys(sg), sg->length); sg->dma_length = sg->length; } spin_unlock_irqrestore(&iounit->lock, flags); return nents; } -static void iounit_unmap_page(struct device *dev, dma_addr_t vaddr, size_t len, +static void iounit_unmap_phys(struct device *dev, dma_addr_t vaddr, size_t len, enum dma_data_direction dir, unsigned long attrs) { struct iounit_struct *iounit = dev->archdata.iommu; @@ -279,8 +281,8 @@ static const struct dma_map_ops iounit_dma_ops = { .alloc = iounit_alloc, .free = iounit_free, #endif - .map_page = iounit_map_page, - .unmap_page = iounit_unmap_page, + .map_phys = iounit_map_phys, + .unmap_phys = iounit_unmap_phys, .map_sg = iounit_map_sg, .unmap_sg = iounit_unmap_sg, }; diff --git a/arch/sparc/mm/iommu.c b/arch/sparc/mm/iommu.c index 5a5080db800f..f48adf62724a 100644 --- a/arch/sparc/mm/iommu.c +++ b/arch/sparc/mm/iommu.c @@ -181,18 +181,20 @@ static void iommu_flush_iotlb(iopte_t *iopte, unsigned int niopte) } } -static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t len, bool per_page_flush) +static dma_addr_t __sbus_iommu_map_phys(struct device *dev, phys_addr_t paddr, + size_t len, bool per_page_flush, unsigned long attrs) { struct iommu_struct *iommu = dev->archdata.iommu; - phys_addr_t paddr = page_to_phys(page) + offset; - unsigned long off = paddr & ~PAGE_MASK; + unsigned long off = offset_in_page(paddr); unsigned long npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT; unsigned long pfn = __phys_to_pfn(paddr); unsigned int busa, busa0; iopte_t *iopte, *iopte0; int ioptex, i; + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; + /* XXX So what is maxphys for us and how do drivers know it? */ if (!len || len > 256 * 1024) return DMA_MAPPING_ERROR; @@ -202,10 +204,10 @@ static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page, * XXX Is this a good assumption? * XXX What if someone else unmaps it here and races us? */ - if (per_page_flush && !PageHighMem(page)) { + if (per_page_flush && !PhysHighMem(paddr)) { unsigned long vaddr, p; - vaddr = (unsigned long)page_address(page) + offset; + vaddr = (unsigned long)phys_to_virt(paddr); for (p = vaddr & PAGE_MASK; p < vaddr + len; p += PAGE_SIZE) flush_page_for_dma(p); } @@ -231,19 +233,19 @@ static dma_addr_t __sbus_iommu_map_page(struct device *dev, struct page *page, return busa0 + off; } -static dma_addr_t sbus_iommu_map_page_gflush(struct device *dev, - struct page *page, unsigned long offset, size_t len, - enum dma_data_direction dir, unsigned long attrs) +static dma_addr_t sbus_iommu_map_phys_gflush(struct device *dev, + phys_addr_t phys, size_t len, enum dma_data_direction dir, + unsigned long attrs) { flush_page_for_dma(0); - return __sbus_iommu_map_page(dev, page, offset, len, false); + return __sbus_iommu_map_phys(dev, phys, len, false, attrs); } -static dma_addr_t sbus_iommu_map_page_pflush(struct device *dev, - struct page *page, unsigned long offset, size_t len, - enum dma_data_direction dir, unsigned long attrs) +static dma_addr_t sbus_iommu_map_phys_pflush(struct device *dev, + phys_addr_t phys, size_t len, enum dma_data_direction dir, + unsigned long attrs) { - return __sbus_iommu_map_page(dev, page, offset, len, true); + return __sbus_iommu_map_phys(dev, phys, len, true, attrs); } static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl, @@ -254,8 +256,8 @@ static int __sbus_iommu_map_sg(struct device *dev, struct scatterlist *sgl, int j; for_each_sg(sgl, sg, nents, j) { - sg->dma_address =__sbus_iommu_map_page(dev, sg_page(sg), - sg->offset, sg->length, per_page_flush); + sg->dma_address = __sbus_iommu_map_phys(dev, sg_phys(sg), + sg->length, per_page_flush, attrs); if (sg->dma_address == DMA_MAPPING_ERROR) return -EIO; sg->dma_length = sg->length; @@ -277,7 +279,7 @@ static int sbus_iommu_map_sg_pflush(struct device *dev, struct scatterlist *sgl, return __sbus_iommu_map_sg(dev, sgl, nents, dir, attrs, true); } -static void sbus_iommu_unmap_page(struct device *dev, dma_addr_t dma_addr, +static void sbus_iommu_unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t len, enum dma_data_direction dir, unsigned long attrs) { struct iommu_struct *iommu = dev->archdata.iommu; @@ -303,7 +305,7 @@ static void sbus_iommu_unmap_sg(struct device *dev, struct scatterlist *sgl, int i; for_each_sg(sgl, sg, nents, i) { - sbus_iommu_unmap_page(dev, sg->dma_address, sg->length, dir, + sbus_iommu_unmap_phys(dev, sg->dma_address, sg->length, dir, attrs); sg->dma_address = 0x21212121; } @@ -426,8 +428,8 @@ static const struct dma_map_ops sbus_iommu_dma_gflush_ops = { .alloc = sbus_iommu_alloc, .free = sbus_iommu_free, #endif - .map_page = sbus_iommu_map_page_gflush, - .unmap_page = sbus_iommu_unmap_page, + .map_phys = sbus_iommu_map_phys_gflush, + .unmap_phys = sbus_iommu_unmap_phys, .map_sg = sbus_iommu_map_sg_gflush, .unmap_sg = sbus_iommu_unmap_sg, }; @@ -437,8 +439,8 @@ static const struct dma_map_ops sbus_iommu_dma_pflush_ops = { .alloc = sbus_iommu_alloc, .free = sbus_iommu_free, #endif - .map_page = sbus_iommu_map_page_pflush, - .unmap_page = sbus_iommu_unmap_page, + .map_phys = sbus_iommu_map_phys_pflush, + .unmap_phys = sbus_iommu_unmap_phys, .map_sg = sbus_iommu_map_sg_pflush, .unmap_sg = sbus_iommu_unmap_sg, }; diff --git a/arch/um/include/asm/Kbuild b/arch/um/include/asm/Kbuild index b6810db24ca4..1b9b82bbe322 100644 --- a/arch/um/include/asm/Kbuild +++ b/arch/um/include/asm/Kbuild @@ -5,7 +5,6 @@ generic-y += device.h generic-y += dma-mapping.h generic-y += emergency-restart.h generic-y += exec.h -generic-y += extable.h generic-y += ftrace.h generic-y += hw_irq.h generic-y += irq_regs.h diff --git a/arch/um/kernel/asm-offsets.c b/arch/um/kernel/asm-offsets.c index d38447e39d5e..d620b6f6de9b 100644 --- a/arch/um/kernel/asm-offsets.c +++ b/arch/um/kernel/asm-offsets.c @@ -9,6 +9,7 @@ #include <linux/fs.h> #include <asm/mman.h> #include <asm/seccomp.h> +#include <asm/extable.h> /* workaround for a warning with -Wmissing-prototypes */ void foo(void); @@ -42,4 +43,7 @@ void foo(void) DEFINE(HOSTFS_ATTR_CTIME, ATTR_CTIME); DEFINE(HOSTFS_ATTR_ATIME_SET, ATTR_ATIME_SET); DEFINE(HOSTFS_ATTR_MTIME_SET, ATTR_MTIME_SET); + + DEFINE(ALT_INSTR_SIZE, sizeof(struct alt_instr)); + DEFINE(EXTABLE_SIZE, sizeof(struct exception_table_entry)); } diff --git a/arch/x86/include/asm/alternative.h b/arch/x86/include/asm/alternative.h index b14c045679e1..03364510d5fe 100644 --- a/arch/x86/include/asm/alternative.h +++ b/arch/x86/include/asm/alternative.h @@ -197,8 +197,8 @@ static inline int alternatives_text_reserved(void *start, void *end) "773:\n" #define ALTINSTR_ENTRY(ft_flags) \ - ".pushsection .altinstructions,\"a\"\n" \ - ANNOTATE_DATA_SPECIAL \ + ".pushsection .altinstructions, \"aM\", @progbits, " \ + __stringify(ALT_INSTR_SIZE) "\n" \ " .long 771b - .\n" /* label */ \ " .long 774f - .\n" /* new instruction */ \ " .4byte " __stringify(ft_flags) "\n" /* feature + flags */ \ @@ -208,7 +208,7 @@ static inline int alternatives_text_reserved(void *start, void *end) #define ALTINSTR_REPLACEMENT(newinstr) /* replacement */ \ ".pushsection .altinstr_replacement, \"ax\"\n" \ - ANNOTATE_DATA_SPECIAL \ + ANNOTATE_DATA_SPECIAL "\n" \ "# ALT: replacement\n" \ "774:\n\t" newinstr "\n775:\n" \ ".popsection\n" @@ -339,7 +339,6 @@ void nop_func(void); * instruction. See apply_alternatives(). */ .macro altinstr_entry orig alt ft_flags orig_len alt_len - ANNOTATE_DATA_SPECIAL .long \orig - . .long \alt - . .4byte \ft_flags @@ -363,7 +362,7 @@ void nop_func(void); 741: \ .skip -(((744f-743f)-(741b-740b)) > 0) * ((744f-743f)-(741b-740b)),0x90 ;\ 742: \ - .pushsection .altinstructions,"a" ; \ + .pushsection .altinstructions, "aM", @progbits, ALT_INSTR_SIZE ;\ altinstr_entry 740b,743f,flag,742b-740b,744f-743f ; \ .popsection ; \ .pushsection .altinstr_replacement,"ax" ; \ diff --git a/arch/x86/include/asm/asm.h b/arch/x86/include/asm/asm.h index bd62bd87a841..0e8c611bc9e2 100644 --- a/arch/x86/include/asm/asm.h +++ b/arch/x86/include/asm/asm.h @@ -126,18 +126,21 @@ static __always_inline __pure void *rip_rel_ptr(void *p) #ifdef __KERNEL__ +#ifndef COMPILE_OFFSETS +#include <asm/asm-offsets.h> +#endif + # include <asm/extable_fixup_types.h> /* Exception table entry */ #ifdef __ASSEMBLER__ -# define _ASM_EXTABLE_TYPE(from, to, type) \ - .pushsection "__ex_table","a" ; \ - .balign 4 ; \ - ANNOTATE_DATA_SPECIAL ; \ - .long (from) - . ; \ - .long (to) - . ; \ - .long type ; \ +# define _ASM_EXTABLE_TYPE(from, to, type) \ + .pushsection "__ex_table", "aM", @progbits, EXTABLE_SIZE ; \ + .balign 4 ; \ + .long (from) - . ; \ + .long (to) - . ; \ + .long type ; \ .popsection # ifdef CONFIG_KPROBES @@ -180,18 +183,18 @@ static __always_inline __pure void *rip_rel_ptr(void *p) ".purgem extable_type_reg\n" # define _ASM_EXTABLE_TYPE(from, to, type) \ - " .pushsection \"__ex_table\",\"a\"\n" \ + " .pushsection __ex_table, \"aM\", @progbits, " \ + __stringify(EXTABLE_SIZE) "\n" \ " .balign 4\n" \ - ANNOTATE_DATA_SPECIAL \ " .long (" #from ") - .\n" \ " .long (" #to ") - .\n" \ " .long " __stringify(type) " \n" \ " .popsection\n" # define _ASM_EXTABLE_TYPE_REG(from, to, type, reg) \ - " .pushsection \"__ex_table\",\"a\"\n" \ + " .pushsection __ex_table, \"aM\", @progbits, " \ + __stringify(EXTABLE_SIZE) "\n" \ " .balign 4\n" \ - ANNOTATE_DATA_SPECIAL \ " .long (" #from ") - .\n" \ " .long (" #to ") - .\n" \ DEFINE_EXTABLE_TYPE_REG \ diff --git a/arch/x86/include/asm/bug.h b/arch/x86/include/asm/bug.h index ab5bba6cf7f5..ee23b98353d7 100644 --- a/arch/x86/include/asm/bug.h +++ b/arch/x86/include/asm/bug.h @@ -70,7 +70,7 @@ extern void __WARN_trap(struct bug_entry *bug, ...); #define _BUG_FLAGS_ASM(format, file, line, flags, size, extra) \ ".pushsection __bug_table,\"aw\"\n\t" \ - ANNOTATE_DATA_SPECIAL \ + ANNOTATE_DATA_SPECIAL "\n\t" \ "2:\n\t" \ __BUG_ENTRY(format, file, line, flags) \ "\t.org 2b + " size "\n" \ diff --git a/arch/x86/include/asm/cpufeature.h b/arch/x86/include/asm/cpufeature.h index 4b1a6ade1700..3ddc1d33399b 100644 --- a/arch/x86/include/asm/cpufeature.h +++ b/arch/x86/include/asm/cpufeature.h @@ -101,7 +101,7 @@ static __always_inline bool _static_cpu_has(u16 bit) asm goto(ALTERNATIVE_TERNARY("jmp 6f", %c[feature], "", "jmp %l[t_no]") ".pushsection .altinstr_aux,\"ax\"\n" "6:\n" - ANNOTATE_DATA_SPECIAL + ANNOTATE_DATA_SPECIAL "\n" " testb %[bitnum], %a[cap_byte]\n" " jnz %l[t_yes]\n" " jmp %l[t_no]\n" diff --git a/arch/x86/include/asm/cpumask.h b/arch/x86/include/asm/cpumask.h index 70f6b60ad67b..9df9e9cde670 100644 --- a/arch/x86/include/asm/cpumask.h +++ b/arch/x86/include/asm/cpumask.h @@ -2,6 +2,8 @@ #ifndef _ASM_X86_CPUMASK_H #define _ASM_X86_CPUMASK_H #ifndef __ASSEMBLER__ + +#include <linux/compiler.h> #include <linux/cpumask.h> extern void setup_cpu_local_masks(void); diff --git a/arch/x86/include/asm/div64.h b/arch/x86/include/asm/div64.h index 9931e4c7d73f..30fd06ede751 100644 --- a/arch/x86/include/asm/div64.h +++ b/arch/x86/include/asm/div64.h @@ -60,6 +60,12 @@ static inline u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) } #define div_u64_rem div_u64_rem +/* + * gcc tends to zero extend 32bit values and do full 64bit maths. + * Define asm functions that avoid this. + * (clang generates better code for the C versions.) + */ +#ifndef __clang__ static inline u64 mul_u32_u32(u32 a, u32 b) { u32 high, low; @@ -71,6 +77,19 @@ static inline u64 mul_u32_u32(u32 a, u32 b) } #define mul_u32_u32 mul_u32_u32 +static inline u64 add_u64_u32(u64 a, u32 b) +{ + u32 high = a >> 32, low = a; + + asm ("addl %[b], %[low]; adcl $0, %[high]" + : [low] "+r" (low), [high] "+r" (high) + : [b] "rm" (b) ); + + return low | (u64)high << 32; +} +#define add_u64_u32 add_u64_u32 +#endif + /* * __div64_32() is never called on x86, so prevent the * generic definition from getting built. @@ -84,21 +103,25 @@ static inline u64 mul_u32_u32(u32 a, u32 b) * Will generate an #DE when the result doesn't fit u64, could fix with an * __ex_table[] entry when it becomes an issue. */ -static inline u64 mul_u64_u64_div_u64(u64 a, u64 mul, u64 div) +static inline u64 mul_u64_add_u64_div_u64(u64 rax, u64 mul, u64 add, u64 div) { - u64 q; + u64 rdx; + + asm ("mulq %[mul]" : "+a" (rax), "=d" (rdx) : [mul] "rm" (mul)); + + if (!statically_true(!add)) + asm ("addq %[add], %[lo]; adcq $0, %[hi]" : + [lo] "+r" (rax), [hi] "+r" (rdx) : [add] "irm" (add)); - asm ("mulq %2; divq %3" : "=a" (q) - : "a" (a), "rm" (mul), "rm" (div) - : "rdx"); + asm ("divq %[div]" : "+a" (rax), "+d" (rdx) : [div] "rm" (div)); - return q; + return rax; } -#define mul_u64_u64_div_u64 mul_u64_u64_div_u64 +#define mul_u64_add_u64_div_u64 mul_u64_add_u64_div_u64 static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 div) { - return mul_u64_u64_div_u64(a, mul, div); + return mul_u64_add_u64_div_u64(a, mul, 0, div); } #define mul_u64_u32_div mul_u64_u32_div diff --git a/arch/x86/include/asm/irq_stack.h b/arch/x86/include/asm/irq_stack.h index 735c3a491f60..8325b79f2ac6 100644 --- a/arch/x86/include/asm/irq_stack.h +++ b/arch/x86/include/asm/irq_stack.h @@ -101,7 +101,7 @@ #define ASM_CALL_ARG0 \ "1: call %c[__func] \n" \ - ANNOTATE_REACHABLE(1b) + ANNOTATE_REACHABLE(1b) " \n" #define ASM_CALL_ARG1 \ "movq %[arg1], %%rdi \n" \ diff --git a/arch/x86/include/asm/jump_label.h b/arch/x86/include/asm/jump_label.h index e0a6930a4029..05b16299588d 100644 --- a/arch/x86/include/asm/jump_label.h +++ b/arch/x86/include/asm/jump_label.h @@ -15,7 +15,7 @@ #define JUMP_TABLE_ENTRY(key, label) \ ".pushsection __jump_table, \"aw\" \n\t" \ _ASM_ALIGN "\n\t" \ - ANNOTATE_DATA_SPECIAL \ + ANNOTATE_DATA_SPECIAL "\n" \ ".long 1b - . \n\t" \ ".long " label " - . \n\t" \ _ASM_PTR " " key " - . \n\t" \ diff --git a/arch/x86/include/asm/nospec-branch.h b/arch/x86/include/asm/nospec-branch.h index a6526c5be5ca..4f4b5e8a1574 100644 --- a/arch/x86/include/asm/nospec-branch.h +++ b/arch/x86/include/asm/nospec-branch.h @@ -466,7 +466,7 @@ static inline void call_depth_return_thunk(void) {} */ # define CALL_NOSPEC \ ALTERNATIVE_2( \ - ANNOTATE_RETPOLINE_SAFE \ + ANNOTATE_RETPOLINE_SAFE "\n" \ "call *%[thunk_target]\n", \ " jmp 904f;\n" \ " .align 16\n" \ @@ -482,7 +482,7 @@ static inline void call_depth_return_thunk(void) {} "904: call 901b;\n", \ X86_FEATURE_RETPOLINE, \ "lfence;\n" \ - ANNOTATE_RETPOLINE_SAFE \ + ANNOTATE_RETPOLINE_SAFE "\n" \ "call *%[thunk_target]\n", \ X86_FEATURE_RETPOLINE_LFENCE) diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h index 37a8627d8277..3502939415ad 100644 --- a/arch/x86/include/asm/paravirt_types.h +++ b/arch/x86/include/asm/paravirt_types.h @@ -249,7 +249,7 @@ extern struct paravirt_patch_template pv_ops; * don't need to bother with CFI prefixes. */ #define PARAVIRT_CALL \ - ANNOTATE_RETPOLINE_SAFE \ + ANNOTATE_RETPOLINE_SAFE "\n\t" \ "call *%[paravirt_opptr];" /* diff --git a/arch/x86/include/asm/smap.h b/arch/x86/include/asm/smap.h index 20a3baae9568..977bef14a0ab 100644 --- a/arch/x86/include/asm/smap.h +++ b/arch/x86/include/asm/smap.h @@ -77,7 +77,7 @@ static __always_inline unsigned long smap_save(void) unsigned long flags; asm volatile ("# smap_save\n\t" - ALTERNATIVE(ANNOTATE_IGNORE_ALTERNATIVE + ALTERNATIVE(ANNOTATE_IGNORE_ALTERNATIVE "\n\t" "", "pushf; pop %0; clac", X86_FEATURE_SMAP) : "=rm" (flags) : : "memory", "cc"); @@ -88,7 +88,7 @@ static __always_inline unsigned long smap_save(void) static __always_inline void smap_restore(unsigned long flags) { asm volatile ("# smap_restore\n\t" - ALTERNATIVE(ANNOTATE_IGNORE_ALTERNATIVE + ALTERNATIVE(ANNOTATE_IGNORE_ALTERNATIVE "\n\t" "", "push %0; popf", X86_FEATURE_SMAP) : : "g" (flags) : "memory", "cc"); @@ -101,9 +101,9 @@ static __always_inline void smap_restore(unsigned long flags) ALTERNATIVE("", "stac", X86_FEATURE_SMAP) #define ASM_CLAC_UNSAFE \ - ALTERNATIVE("", ANNOTATE_IGNORE_ALTERNATIVE "clac", X86_FEATURE_SMAP) + ALTERNATIVE("", ANNOTATE_IGNORE_ALTERNATIVE "\n\t" "clac", X86_FEATURE_SMAP) #define ASM_STAC_UNSAFE \ - ALTERNATIVE("", ANNOTATE_IGNORE_ALTERNATIVE "stac", X86_FEATURE_SMAP) + ALTERNATIVE("", ANNOTATE_IGNORE_ALTERNATIVE "\n\t" "stac", X86_FEATURE_SMAP) #endif /* __ASSEMBLER__ */ diff --git a/arch/x86/include/asm/static_call.h b/arch/x86/include/asm/static_call.h index 41502bd2afd6..4cd725a8fe91 100644 --- a/arch/x86/include/asm/static_call.h +++ b/arch/x86/include/asm/static_call.h @@ -36,7 +36,7 @@ ".align 4 \n" \ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \ STATIC_CALL_TRAMP_STR(name) ": \n" \ - ANNOTATE_NOENDBR \ + ANNOTATE_NOENDBR " \n" \ insns " \n" \ ".byte 0x0f, 0xb9, 0xcc \n" \ ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \ diff --git a/arch/x86/include/asm/x86_init.h b/arch/x86/include/asm/x86_init.h index 36698cc9fb44..6c8a6ead84f6 100644 --- a/arch/x86/include/asm/x86_init.h +++ b/arch/x86/include/asm/x86_init.h @@ -79,7 +79,7 @@ struct x86_init_paging { /** * struct x86_init_timers - platform specific timer setup - * @setup_perpcu_clockev: set up the per cpu clock event device for the + * @setup_percpu_clockev: set up the per cpu clock event device for the * boot cpu * @timer_init: initialize the platform timer (default PIT/HPET) * @wallclock_init: init the wallclock device @@ -132,7 +132,7 @@ struct x86_hyper_init { /** * struct x86_init_acpi - x86 ACPI init functions - * @set_root_poitner: set RSDP address + * @set_root_pointer: set RSDP address * @get_root_pointer: get RSDP address * @reduced_hw_early_init: hardware reduced platform early init */ @@ -145,14 +145,14 @@ struct x86_init_acpi { /** * struct x86_guest - Functions used by misc guest incarnations like SEV, TDX, etc. * - * @enc_status_change_prepare Notify HV before the encryption status of a range is changed - * @enc_status_change_finish Notify HV after the encryption status of a range is changed - * @enc_tlb_flush_required Returns true if a TLB flush is needed before changing page encryption status - * @enc_cache_flush_required Returns true if a cache flush is needed before changing page encryption status - * @enc_kexec_begin Begin the two-step process of converting shared memory back + * @enc_status_change_prepare: Notify HV before the encryption status of a range is changed + * @enc_status_change_finish: Notify HV after the encryption status of a range is changed + * @enc_tlb_flush_required: Returns true if a TLB flush is needed before changing page encryption status + * @enc_cache_flush_required: Returns true if a cache flush is needed before changing page encryption status + * @enc_kexec_begin: Begin the two-step process of converting shared memory back * to private. It stops the new conversions from being started * and waits in-flight conversions to finish, if possible. - * @enc_kexec_finish Finish the two-step process of converting shared memory to + * @enc_kexec_finish: Finish the two-step process of converting shared memory to * private. All memory is private after the call when * the function returns. * It is called on only one CPU while the others are shut down @@ -229,7 +229,7 @@ struct x86_legacy_devices { * given platform/subarch. * @X86_LEGACY_I8042_FIRMWARE_ABSENT: firmware reports that the controller * is absent. - * @X86_LEGACY_i8042_EXPECTED_PRESENT: the controller is likely to be + * @X86_LEGACY_I8042_EXPECTED_PRESENT: the controller is likely to be * present, the i8042 driver should probe for controller existence. */ enum x86_legacy_i8042_state { @@ -244,6 +244,8 @@ enum x86_legacy_i8042_state { * @i8042: indicated if we expect the device to have i8042 controller * present. * @rtc: this device has a CMOS real-time clock present + * @warm_reset: 1 if platform allows warm reset, else 0 + * @no_vga: 1 if (FADT.boot_flags & ACPI_FADT_NO_VGA) is set, else 0 * @reserve_bios_regions: boot code will search for the EBDA address and the * start of the 640k - 1M BIOS region. If false, the platform must * ensure that its memory map correctly reserves sub-1MB regions as needed. @@ -290,9 +292,10 @@ struct x86_hyper_runtime { * @calibrate_tsc: calibrate TSC, if different from CPU * @get_wallclock: get time from HW clock like RTC etc. * @set_wallclock: set time back to HW clock - * @is_untracked_pat_range exclude from PAT logic - * @nmi_init enable NMI on cpus - * @get_nmi_reason get the reason an NMI was received + * @iommu_shutdown: set by an IOMMU driver for shutdown if necessary + * @is_untracked_pat_range: exclude from PAT logic + * @nmi_init: enable NMI on cpus + * @get_nmi_reason: get the reason an NMI was received * @save_sched_clock_state: save state for sched_clock() on suspend * @restore_sched_clock_state: restore state for sched_clock() on resume * @apic_post_init: adjust apic if needed @@ -307,6 +310,7 @@ struct x86_hyper_runtime { * @realmode_reserve: reserve memory for realmode trampoline * @realmode_init: initialize realmode trampoline * @hyper: x86 hypervisor specific runtime callbacks + * @guest: guest incarnations callbacks */ struct x86_platform_ops { unsigned long (*calibrate_cpu)(void); diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c index 74f4c659f9c9..28518371d8bf 100644 --- a/arch/x86/kernel/alternative.c +++ b/arch/x86/kernel/alternative.c @@ -2229,7 +2229,7 @@ asm ( " .pushsection .init.text, \"ax\", @progbits\n" " .type int3_selftest_asm, @function\n" "int3_selftest_asm:\n" - ANNOTATE_NOENDBR + ANNOTATE_NOENDBR "\n" /* * INT3 padded with NOP to CALL_INSN_SIZE. The INT3 triggers an * exception, then the int3_exception_nb notifier emulates a call to @@ -2247,7 +2247,7 @@ asm ( " .pushsection .init.text, \"ax\", @progbits\n" " .type int3_selftest_callee, @function\n" "int3_selftest_callee:\n" - ANNOTATE_NOENDBR + ANNOTATE_NOENDBR "\n" " movl $0x1234, (%" _ASM_ARG1 ")\n" ASM_RET " .size int3_selftest_callee, . - int3_selftest_callee\n" diff --git a/arch/x86/kernel/amd_gart_64.c b/arch/x86/kernel/amd_gart_64.c index e6e68a31634c..e8000a56732e 100644 --- a/arch/x86/kernel/amd_gart_64.c +++ b/arch/x86/kernel/amd_gart_64.c @@ -222,13 +222,14 @@ static dma_addr_t dma_map_area(struct device *dev, dma_addr_t phys_mem, } /* Map a single area into the IOMMU */ -static dma_addr_t gart_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, +static dma_addr_t gart_map_phys(struct device *dev, phys_addr_t paddr, + size_t size, enum dma_data_direction dir, unsigned long attrs) { unsigned long bus; - phys_addr_t paddr = page_to_phys(page) + offset; + + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; if (!need_iommu(dev, paddr, size)) return paddr; @@ -242,7 +243,7 @@ static dma_addr_t gart_map_page(struct device *dev, struct page *page, /* * Free a DMA mapping. */ -static void gart_unmap_page(struct device *dev, dma_addr_t dma_addr, +static void gart_unmap_phys(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { @@ -282,7 +283,7 @@ static void gart_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, for_each_sg(sg, s, nents, i) { if (!s->dma_length || !s->length) break; - gart_unmap_page(dev, s->dma_address, s->dma_length, dir, 0); + gart_unmap_phys(dev, s->dma_address, s->dma_length, dir, 0); } } @@ -487,7 +488,7 @@ static void gart_free_coherent(struct device *dev, size_t size, void *vaddr, dma_addr_t dma_addr, unsigned long attrs) { - gart_unmap_page(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); + gart_unmap_phys(dev, dma_addr, size, DMA_BIDIRECTIONAL, 0); dma_direct_free(dev, size, vaddr, dma_addr, attrs); } @@ -672,8 +673,8 @@ static __init int init_amd_gatt(struct agp_kern_info *info) static const struct dma_map_ops gart_dma_ops = { .map_sg = gart_map_sg, .unmap_sg = gart_unmap_sg, - .map_page = gart_map_page, - .unmap_page = gart_unmap_page, + .map_phys = gart_map_phys, + .unmap_phys = gart_unmap_phys, .alloc = gart_alloc_coherent, .free = gart_free_coherent, .mmap = dma_common_mmap, diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c index 32ba599a51f8..25fcde525c68 100644 --- a/arch/x86/kernel/asm-offsets.c +++ b/arch/x86/kernel/asm-offsets.c @@ -124,4 +124,7 @@ static void __used common(void) OFFSET(ARIA_CTX_rounds, aria_ctx, rounds); #endif + BLANK(); + DEFINE(ALT_INSTR_SIZE, sizeof(struct alt_instr)); + DEFINE(EXTABLE_SIZE, sizeof(struct exception_table_entry)); } diff --git a/arch/x86/kernel/cpu/mce/core.c b/arch/x86/kernel/cpu/mce/core.c index 6297416647ed..34440021e8cf 100644 --- a/arch/x86/kernel/cpu/mce/core.c +++ b/arch/x86/kernel/cpu/mce/core.c @@ -45,6 +45,7 @@ #include <linux/task_work.h> #include <linux/hardirq.h> #include <linux/kexec.h> +#include <linux/vmcore_info.h> #include <asm/fred.h> #include <asm/cpu_device_id.h> @@ -1729,6 +1730,9 @@ noinstr void do_machine_check(struct pt_regs *regs) } out: + /* Given it didn't panic, mark it as recoverable */ + hwerr_log_error_type(HWERR_RECOV_OTHERS); + instrumentation_end(); clear: diff --git a/arch/x86/kernel/rethook.c b/arch/x86/kernel/rethook.c index 8a1c0111ae79..85e2f2d16a90 100644 --- a/arch/x86/kernel/rethook.c +++ b/arch/x86/kernel/rethook.c @@ -25,7 +25,7 @@ asm( ".type arch_rethook_trampoline, @function\n" "arch_rethook_trampoline:\n" #ifdef CONFIG_X86_64 - ANNOTATE_NOENDBR /* This is only jumped from ret instruction */ + ANNOTATE_NOENDBR "\n" /* This is only jumped from ret instruction */ /* Push a fake return address to tell the unwinder it's a rethook. */ " pushq $arch_rethook_trampoline\n" UNWIND_HINT_FUNC diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c index 2892cdb14563..61592e41a6b1 100644 --- a/arch/x86/kernel/static_call.c +++ b/arch/x86/kernel/static_call.c @@ -50,8 +50,8 @@ asm (".global __static_call_return\n\t" ".type __static_call_return, @function\n\t" ASM_FUNC_ALIGN "\n\t" "__static_call_return:\n\t" - ANNOTATE_NOENDBR - ANNOTATE_RETPOLINE_SAFE + ANNOTATE_NOENDBR "\n\t" + ANNOTATE_RETPOLINE_SAFE "\n\t" "ret; int3\n\t" ".size __static_call_return, . - __static_call_return \n\t"); diff --git a/arch/x86/lib/error-inject.c b/arch/x86/lib/error-inject.c index b5a6d83106bc..512a2538596f 100644 --- a/arch/x86/lib/error-inject.c +++ b/arch/x86/lib/error-inject.c @@ -13,7 +13,7 @@ asm( ".globl just_return_func\n" ASM_FUNC_ALIGN "just_return_func:\n" - ANNOTATE_NOENDBR + ANNOTATE_NOENDBR "\n" ASM_RET ".size just_return_func, .-just_return_func\n" ); diff --git a/drivers/Makefile b/drivers/Makefile index 20eb17596b89..b9f70e01f269 100644 --- a/drivers/Makefile +++ b/drivers/Makefile @@ -160,7 +160,7 @@ obj-$(CONFIG_RPMSG) += rpmsg/ obj-$(CONFIG_SOUNDWIRE) += soundwire/ # Virtualization drivers -obj-$(CONFIG_VIRT_DRIVERS) += virt/ +obj-y += virt/ obj-$(CONFIG_HYPERV) += hv/ obj-$(CONFIG_PM_DEVFREQ) += devfreq/ diff --git a/drivers/acpi/apei/ghes.c b/drivers/acpi/apei/ghes.c index 626908491d8f..0dc767392a6c 100644 --- a/drivers/acpi/apei/ghes.c +++ b/drivers/acpi/apei/ghes.c @@ -44,6 +44,7 @@ #include <linux/uuid.h> #include <linux/ras.h> #include <linux/task_work.h> +#include <linux/vmcore_info.h> #include <acpi/actbl1.h> #include <acpi/ghes.h> @@ -864,6 +865,40 @@ int cxl_cper_kfifo_get(struct cxl_cper_work_data *wd) } EXPORT_SYMBOL_NS_GPL(cxl_cper_kfifo_get, "CXL"); +static void ghes_log_hwerr(int sev, guid_t *sec_type) +{ + if (sev != CPER_SEV_RECOVERABLE) + return; + + if (guid_equal(sec_type, &CPER_SEC_PROC_ARM) || + guid_equal(sec_type, &CPER_SEC_PROC_GENERIC) || + guid_equal(sec_type, &CPER_SEC_PROC_IA)) { + hwerr_log_error_type(HWERR_RECOV_CPU); + return; + } + + if (guid_equal(sec_type, &CPER_SEC_CXL_PROT_ERR) || + guid_equal(sec_type, &CPER_SEC_CXL_GEN_MEDIA_GUID) || + guid_equal(sec_type, &CPER_SEC_CXL_DRAM_GUID) || + guid_equal(sec_type, &CPER_SEC_CXL_MEM_MODULE_GUID)) { + hwerr_log_error_type(HWERR_RECOV_CXL); + return; + } + + if (guid_equal(sec_type, &CPER_SEC_PCIE) || + guid_equal(sec_type, &CPER_SEC_PCI_X_BUS)) { + hwerr_log_error_type(HWERR_RECOV_PCI); + return; + } + + if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) { + hwerr_log_error_type(HWERR_RECOV_MEMORY); + return; + } + + hwerr_log_error_type(HWERR_RECOV_OTHERS); +} + static void ghes_do_proc(struct ghes *ghes, const struct acpi_hest_generic_status *estatus) { @@ -885,6 +920,7 @@ static void ghes_do_proc(struct ghes *ghes, if (gdata->validation_bits & CPER_SEC_VALID_FRU_TEXT) fru_text = gdata->fru_text; + ghes_log_hwerr(sev, sec_type); if (guid_equal(sec_type, &CPER_SEC_PLATFORM_MEM)) { struct cper_sec_mem_err *mem_err = acpi_hest_get_payload(gdata); diff --git a/drivers/android/binder/process.rs b/drivers/android/binder/process.rs index e5237e9ec552..ac981614544e 100644 --- a/drivers/android/binder/process.rs +++ b/drivers/android/binder/process.rs @@ -19,6 +19,7 @@ use kernel::{ cred::Credential, error::Error, fs::file::{self, File}, + id_pool::IdPool, list::{List, ListArc, ListArcField, ListLinks}, mm, prelude::*, @@ -394,6 +395,8 @@ kernel::list::impl_list_item! { struct ProcessNodeRefs { /// Used to look up nodes using the 32-bit id that this process knows it by. by_handle: RBTree<u32, ListArc<NodeRefInfo, { NodeRefInfo::LIST_PROC }>>, + /// Used to quickly find unused ids in `by_handle`. + handle_is_present: IdPool, /// Used to look up nodes without knowing their local 32-bit id. The usize is the address of /// the underlying `Node` struct as returned by `Node::global_id`. by_node: RBTree<usize, u32>, @@ -408,6 +411,7 @@ impl ProcessNodeRefs { fn new() -> Self { Self { by_handle: RBTree::new(), + handle_is_present: IdPool::new(), by_node: RBTree::new(), freeze_listeners: RBTree::new(), } @@ -802,7 +806,7 @@ impl Process { pub(crate) fn insert_or_update_handle( self: ArcBorrow<'_, Process>, node_ref: NodeRef, - is_mananger: bool, + is_manager: bool, ) -> Result<u32> { { let mut refs = self.node_refs.lock(); @@ -821,7 +825,33 @@ impl Process { let reserve2 = RBTreeNodeReservation::new(GFP_KERNEL)?; let info = UniqueArc::new_uninit(GFP_KERNEL)?; - let mut refs = self.node_refs.lock(); + let mut refs_lock = self.node_refs.lock(); + let mut refs = &mut *refs_lock; + + let (unused_id, by_handle_slot) = loop { + // ID 0 may only be used by the manager. + let start = if is_manager { 0 } else { 1 }; + + if let Some(res) = refs.handle_is_present.find_unused_id(start) { + match refs.by_handle.entry(res.as_u32()) { + rbtree::Entry::Vacant(entry) => break (res, entry), + rbtree::Entry::Occupied(_) => { + pr_err!("Detected mismatch between handle_is_present and by_handle"); + res.acquire(); + kernel::warn_on!(true); + return Err(EINVAL); + } + } + } + + let grow_request = refs.handle_is_present.grow_request().ok_or(ENOMEM)?; + drop(refs_lock); + let resizer = grow_request.realloc(GFP_KERNEL)?; + refs_lock = self.node_refs.lock(); + refs = &mut *refs_lock; + refs.handle_is_present.grow(resizer); + }; + let handle = unused_id.as_u32(); // Do a lookup again as node may have been inserted before the lock was reacquired. if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) { @@ -831,20 +861,9 @@ impl Process { return Ok(handle); } - // Find id. - let mut target: u32 = if is_mananger { 0 } else { 1 }; - for handle in refs.by_handle.keys() { - if *handle > target { - break; - } - if *handle == target { - target = target.checked_add(1).ok_or(ENOMEM)?; - } - } - let gid = node_ref.node.global_id(); let (info_proc, info_node) = { - let info_init = NodeRefInfo::new(node_ref, target, self.into()); + let info_init = NodeRefInfo::new(node_ref, handle, self.into()); match info.pin_init_with(info_init) { Ok(info) => ListArc::pair_from_pin_unique(info), // error is infallible @@ -865,9 +884,10 @@ impl Process { // `info_node` into the right node's `refs` list. unsafe { info_proc.node_ref2().node.insert_node_info(info_node) }; - refs.by_node.insert(reserve1.into_node(gid, target)); - refs.by_handle.insert(reserve2.into_node(target, info_proc)); - Ok(target) + refs.by_node.insert(reserve1.into_node(gid, handle)); + by_handle_slot.insert(info_proc, reserve2); + unused_id.acquire(); + Ok(handle) } pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> { @@ -932,6 +952,16 @@ impl Process { let id = info.node_ref().node.global_id(); refs.by_handle.remove(&handle); refs.by_node.remove(&id); + refs.handle_is_present.release_id(handle as usize); + + if let Some(shrink) = refs.handle_is_present.shrink_request() { + drop(refs); + // This intentionally ignores allocation failures. + if let Ok(new_bitmap) = shrink.realloc(GFP_KERNEL) { + refs = self.node_refs.lock(); + refs.handle_is_present.shrink(new_bitmap); + } + } } } else { // All refs are cleared in process exit, so this warning is expected in that case. diff --git a/drivers/base/bus.c b/drivers/base/bus.c index 320e155c6be7..9eb7771706f0 100644 --- a/drivers/base/bus.c +++ b/drivers/base/bus.c @@ -334,6 +334,19 @@ static struct device *next_device(struct klist_iter *i) return dev; } +static struct device *prev_device(struct klist_iter *i) +{ + struct klist_node *n = klist_prev(i); + struct device *dev = NULL; + struct device_private *dev_prv; + + if (n) { + dev_prv = to_device_private_bus(n); + dev = dev_prv->device; + } + return dev; +} + /** * bus_for_each_dev - device iterator. * @bus: bus type. @@ -414,6 +427,31 @@ struct device *bus_find_device(const struct bus_type *bus, } EXPORT_SYMBOL_GPL(bus_find_device); +struct device *bus_find_device_reverse(const struct bus_type *bus, + struct device *start, const void *data, + device_match_t match) +{ + struct subsys_private *sp = bus_to_subsys(bus); + struct klist_iter i; + struct device *dev; + + if (!sp) + return NULL; + + klist_iter_init_node(&sp->klist_devices, &i, + (start ? &start->p->knode_bus : NULL)); + while ((dev = prev_device(&i))) { + if (match(dev, data)) { + get_device(dev); + break; + } + } + klist_iter_exit(&i); + subsys_put(sp); + return dev; +} +EXPORT_SYMBOL_GPL(bus_find_device_reverse); + static struct device_driver *next_driver(struct klist_iter *i) { struct klist_node *n = klist_next(i); diff --git a/drivers/clk/at91/clk-peripheral.c b/drivers/clk/at91/clk-peripheral.c index e700f40fd87f..e7208c47268b 100644 --- a/drivers/clk/at91/clk-peripheral.c +++ b/drivers/clk/at91/clk-peripheral.c @@ -3,6 +3,7 @@ * Copyright (C) 2013 Boris BREZILLON <b.brezillon@overkiz.com> */ +#include <linux/bitfield.h> #include <linux/bitops.h> #include <linux/clk-provider.h> #include <linux/clkdev.h> diff --git a/drivers/clk/at91/pmc.h b/drivers/clk/at91/pmc.h index 5daa32c4cf25..543d7aee8d24 100644 --- a/drivers/clk/at91/pmc.h +++ b/drivers/clk/at91/pmc.h @@ -117,9 +117,6 @@ struct at91_clk_pms { unsigned int parent; }; -#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) -#define field_prep(_mask, _val) (((_val) << (ffs(_mask) - 1)) & (_mask)) - #define ndck(a, s) (a[s - 1].id + 1) #define nck(a) (a[ARRAY_SIZE(a) - 1].id + 1) diff --git a/drivers/clk/renesas/clk-div6.c b/drivers/clk/renesas/clk-div6.c index 3abd6e5400ad..f7b827b5e9b2 100644 --- a/drivers/clk/renesas/clk-div6.c +++ b/drivers/clk/renesas/clk-div6.c @@ -7,6 +7,7 @@ * Contact: Laurent Pinchart <laurent.pinchart@ideasonboard.com> */ +#include <linux/bitfield.h> #include <linux/clk-provider.h> #include <linux/init.h> #include <linux/io.h> @@ -171,8 +172,7 @@ static u8 cpg_div6_clock_get_parent(struct clk_hw *hw) if (clock->src_mask == 0) return 0; - hw_index = (readl(clock->reg) & clock->src_mask) >> - __ffs(clock->src_mask); + hw_index = field_get(clock->src_mask, readl(clock->reg)); for (i = 0; i < clk_hw_get_num_parents(hw); i++) { if (clock->parents[i] == hw_index) return i; @@ -191,7 +191,7 @@ static int cpg_div6_clock_set_parent(struct clk_hw *hw, u8 index) if (index >= clk_hw_get_num_parents(hw)) return -EINVAL; - src = clock->parents[index] << __ffs(clock->src_mask); + src = field_prep(clock->src_mask, clock->parents[index]); writel((readl(clock->reg) & ~clock->src_mask) | src, clock->reg); return 0; } diff --git a/drivers/clk/renesas/rcar-gen3-cpg.c b/drivers/clk/renesas/rcar-gen3-cpg.c index 10ae20489df9..b954278ddd9d 100644 --- a/drivers/clk/renesas/rcar-gen3-cpg.c +++ b/drivers/clk/renesas/rcar-gen3-cpg.c @@ -54,10 +54,8 @@ static unsigned long cpg_pll_clk_recalc_rate(struct clk_hw *hw, { struct cpg_pll_clk *pll_clk = to_pll_clk(hw); unsigned int mult; - u32 val; - val = readl(pll_clk->pllcr_reg) & CPG_PLLnCR_STC_MASK; - mult = (val >> __ffs(CPG_PLLnCR_STC_MASK)) + 1; + mult = FIELD_GET(CPG_PLLnCR_STC_MASK, readl(pll_clk->pllcr_reg)) + 1; return parent_rate * mult * pll_clk->fixed_mult; } @@ -94,7 +92,7 @@ static int cpg_pll_clk_set_rate(struct clk_hw *hw, unsigned long rate, val = readl(pll_clk->pllcr_reg); val &= ~CPG_PLLnCR_STC_MASK; - val |= (mult - 1) << __ffs(CPG_PLLnCR_STC_MASK); + val |= FIELD_PREP(CPG_PLLnCR_STC_MASK, mult - 1); writel(val, pll_clk->pllcr_reg); for (i = 1000; i; i--) { @@ -176,11 +174,7 @@ static unsigned long cpg_z_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { struct cpg_z_clk *zclk = to_z_clk(hw); - unsigned int mult; - u32 val; - - val = readl(zclk->reg) & zclk->mask; - mult = 32 - (val >> __ffs(zclk->mask)); + unsigned int mult = 32 - field_get(zclk->mask, readl(zclk->reg)); return DIV_ROUND_CLOSEST_ULL((u64)parent_rate * mult, 32 * zclk->fixed_div); @@ -231,7 +225,8 @@ static int cpg_z_clk_set_rate(struct clk_hw *hw, unsigned long rate, if (readl(zclk->kick_reg) & CPG_FRQCRB_KICK) return -EBUSY; - cpg_reg_modify(zclk->reg, zclk->mask, (32 - mult) << __ffs(zclk->mask)); + cpg_reg_modify(zclk->reg, zclk->mask, + field_prep(zclk->mask, 32 - mult)); /* * Set KICK bit in FRQCRB to update hardware setting and wait for diff --git a/drivers/clk/renesas/rcar-gen4-cpg.c b/drivers/clk/renesas/rcar-gen4-cpg.c index fb9a876aaba5..db3a0b8ef2b9 100644 --- a/drivers/clk/renesas/rcar-gen4-cpg.c +++ b/drivers/clk/renesas/rcar-gen4-cpg.c @@ -279,11 +279,7 @@ static unsigned long cpg_z_clk_recalc_rate(struct clk_hw *hw, unsigned long parent_rate) { struct cpg_z_clk *zclk = to_z_clk(hw); - unsigned int mult; - u32 val; - - val = readl(zclk->reg) & zclk->mask; - mult = 32 - (val >> __ffs(zclk->mask)); + unsigned int mult = 32 - field_get(zclk->mask, readl(zclk->reg)); return DIV_ROUND_CLOSEST_ULL((u64)parent_rate * mult, 32 * zclk->fixed_div); @@ -334,7 +330,8 @@ static int cpg_z_clk_set_rate(struct clk_hw *hw, unsigned long rate, if (readl(zclk->kick_reg) & CPG_FRQCRB_KICK) return -EBUSY; - cpg_reg_modify(zclk->reg, zclk->mask, (32 - mult) << __ffs(zclk->mask)); + cpg_reg_modify(zclk->reg, zclk->mask, + field_prep(zclk->mask, 32 - mult)); /* * Set KICK bit in FRQCRB to update hardware setting and wait for diff --git a/drivers/crypto/ccp/Kconfig b/drivers/crypto/ccp/Kconfig index f394e45e11ab..f16a0f611317 100644 --- a/drivers/crypto/ccp/Kconfig +++ b/drivers/crypto/ccp/Kconfig @@ -39,6 +39,7 @@ config CRYPTO_DEV_SP_PSP bool "Platform Security Processor (PSP) device" default y depends on CRYPTO_DEV_CCP_DD && X86_64 && AMD_IOMMU + select PCI_TSM if PCI help Provide support for the AMD Platform Security Processor (PSP). The PSP is a dedicated processor that provides support for key diff --git a/drivers/crypto/ccp/Makefile b/drivers/crypto/ccp/Makefile index a9626b30044a..0424e08561ef 100644 --- a/drivers/crypto/ccp/Makefile +++ b/drivers/crypto/ccp/Makefile @@ -16,6 +16,10 @@ ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += psp-dev.o \ hsti.o \ sfs.o +ifeq ($(CONFIG_PCI_TSM),y) +ccp-$(CONFIG_CRYPTO_DEV_SP_PSP) += sev-dev-tsm.o sev-dev-tio.o +endif + obj-$(CONFIG_CRYPTO_DEV_CCP_CRYPTO) += ccp-crypto.o ccp-crypto-objs := ccp-crypto-main.o \ ccp-crypto-aes.o \ diff --git a/drivers/crypto/ccp/sev-dev-tio.c b/drivers/crypto/ccp/sev-dev-tio.c new file mode 100644 index 000000000000..9a98f98c20a7 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev-tio.c @@ -0,0 +1,864 @@ +// SPDX-License-Identifier: GPL-2.0-only + +// Interface to PSP for CCP/SEV-TIO/SNP-VM + +#include <linux/pci.h> +#include <linux/tsm.h> +#include <linux/psp.h> +#include <linux/vmalloc.h> +#include <linux/bitfield.h> +#include <linux/pci-doe.h> +#include <asm/sev-common.h> +#include <asm/sev.h> +#include <asm/page.h> +#include "sev-dev.h" +#include "sev-dev-tio.h" + +#define to_tio_status(dev_data) \ + (container_of((dev_data), struct tio_dsm, data)->sev->tio_status) + +#define SLA_PAGE_TYPE_DATA 0 +#define SLA_PAGE_TYPE_SCATTER 1 +#define SLA_PAGE_SIZE_4K 0 +#define SLA_PAGE_SIZE_2M 1 +#define SLA_SZ(s) ((s).page_size == SLA_PAGE_SIZE_2M ? SZ_2M : SZ_4K) +#define SLA_SCATTER_LEN(s) (SLA_SZ(s) / sizeof(struct sla_addr_t)) +#define SLA_EOL ((struct sla_addr_t) { .pfn = ((1UL << 40) - 1) }) +#define SLA_NULL ((struct sla_addr_t) { 0 }) +#define IS_SLA_NULL(s) ((s).sla == SLA_NULL.sla) +#define IS_SLA_EOL(s) ((s).sla == SLA_EOL.sla) + +static phys_addr_t sla_to_pa(struct sla_addr_t sla) +{ + u64 pfn = sla.pfn; + u64 pa = pfn << PAGE_SHIFT; + + return pa; +} + +static void *sla_to_va(struct sla_addr_t sla) +{ + void *va = __va(__sme_clr(sla_to_pa(sla))); + + return va; +} + +#define sla_to_pfn(sla) (__pa(sla_to_va(sla)) >> PAGE_SHIFT) +#define sla_to_page(sla) virt_to_page(sla_to_va(sla)) + +static struct sla_addr_t make_sla(struct page *pg, bool stp) +{ + u64 pa = __sme_set(page_to_phys(pg)); + struct sla_addr_t ret = { + .pfn = pa >> PAGE_SHIFT, + .page_size = SLA_PAGE_SIZE_4K, /* Do not do SLA_PAGE_SIZE_2M ATM */ + .page_type = stp ? SLA_PAGE_TYPE_SCATTER : SLA_PAGE_TYPE_DATA + }; + + return ret; +} + +/* the BUFFER Structure */ +#define SLA_BUFFER_FLAG_ENCRYPTION BIT(0) + +/* + * struct sla_buffer_hdr - Scatter list address buffer header + * + * @capacity_sz: Total capacity of the buffer in bytes + * @payload_sz: Size of buffer payload in bytes, must be multiple of 32B + * @flags: Buffer flags (SLA_BUFFER_FLAG_ENCRYPTION: buffer is encrypted) + * @iv: Initialization vector used for encryption + * @authtag: Authentication tag for encrypted buffer + */ +struct sla_buffer_hdr { + u32 capacity_sz; + u32 payload_sz; /* The size of BUFFER_PAYLOAD in bytes. Must be multiple of 32B */ + u32 flags; + u8 reserved1[4]; + u8 iv[16]; /* IV used for the encryption of this buffer */ + u8 authtag[16]; /* Authentication tag for this buffer */ + u8 reserved2[16]; +} __packed; + +enum spdm_data_type_t { + DOBJ_DATA_TYPE_SPDM = 0x1, + DOBJ_DATA_TYPE_SECURE_SPDM = 0x2, +}; + +struct spdm_dobj_hdr_req { + struct spdm_dobj_hdr hdr; /* hdr.id == SPDM_DOBJ_ID_REQ */ + u8 data_type; /* spdm_data_type_t */ + u8 reserved2[5]; +} __packed; + +struct spdm_dobj_hdr_resp { + struct spdm_dobj_hdr hdr; /* hdr.id == SPDM_DOBJ_ID_RESP */ + u8 data_type; /* spdm_data_type_t */ + u8 reserved2[5]; +} __packed; + +/* Defined in sev-dev-tio.h so sev-dev-tsm.c can read types of blobs */ +struct spdm_dobj_hdr_cert; +struct spdm_dobj_hdr_meas; +struct spdm_dobj_hdr_report; + +/* Used in all SPDM-aware TIO commands */ +struct spdm_ctrl { + struct sla_addr_t req; + struct sla_addr_t resp; + struct sla_addr_t scratch; + struct sla_addr_t output; +} __packed; + +static size_t sla_dobj_id_to_size(u8 id) +{ + size_t n; + + BUILD_BUG_ON(sizeof(struct spdm_dobj_hdr_resp) != 0x10); + switch (id) { + case SPDM_DOBJ_ID_REQ: + n = sizeof(struct spdm_dobj_hdr_req); + break; + case SPDM_DOBJ_ID_RESP: + n = sizeof(struct spdm_dobj_hdr_resp); + break; + default: + WARN_ON(1); + n = 0; + break; + } + + return n; +} + +#define SPDM_DOBJ_HDR_SIZE(hdr) sla_dobj_id_to_size((hdr)->id) +#define SPDM_DOBJ_DATA(hdr) ((u8 *)(hdr) + SPDM_DOBJ_HDR_SIZE(hdr)) +#define SPDM_DOBJ_LEN(hdr) ((hdr)->length - SPDM_DOBJ_HDR_SIZE(hdr)) + +#define sla_to_dobj_resp_hdr(buf) ((struct spdm_dobj_hdr_resp *) \ + sla_to_dobj_hdr_check((buf), SPDM_DOBJ_ID_RESP)) +#define sla_to_dobj_req_hdr(buf) ((struct spdm_dobj_hdr_req *) \ + sla_to_dobj_hdr_check((buf), SPDM_DOBJ_ID_REQ)) + +static struct spdm_dobj_hdr *sla_to_dobj_hdr(struct sla_buffer_hdr *buf) +{ + if (!buf) + return NULL; + + return (struct spdm_dobj_hdr *) &buf[1]; +} + +static struct spdm_dobj_hdr *sla_to_dobj_hdr_check(struct sla_buffer_hdr *buf, u32 check_dobjid) +{ + struct spdm_dobj_hdr *hdr = sla_to_dobj_hdr(buf); + + if (WARN_ON_ONCE(!hdr)) + return NULL; + + if (hdr->id != check_dobjid) { + pr_err("! ERROR: expected %d, found %d\n", check_dobjid, hdr->id); + return NULL; + } + + return hdr; +} + +static void *sla_to_data(struct sla_buffer_hdr *buf, u32 dobjid) +{ + struct spdm_dobj_hdr *hdr = sla_to_dobj_hdr(buf); + + if (WARN_ON_ONCE(dobjid != SPDM_DOBJ_ID_REQ && dobjid != SPDM_DOBJ_ID_RESP)) + return NULL; + + if (!hdr) + return NULL; + + return (u8 *) hdr + sla_dobj_id_to_size(dobjid); +} + +/* + * struct sev_data_tio_status - SEV_CMD_TIO_STATUS command + * + * @length: Length of this command buffer in bytes + * @status_paddr: System physical address of the TIO_STATUS structure + */ +struct sev_data_tio_status { + u32 length; + u8 reserved[4]; + u64 status_paddr; +} __packed; + +/* TIO_INIT */ +struct sev_data_tio_init { + u32 length; + u8 reserved[12]; +} __packed; + +/* + * struct sev_data_tio_dev_create - TIO_DEV_CREATE command + * + * @length: Length in bytes of this command buffer + * @dev_ctx_sla: Scatter list address pointing to a buffer to be used as a device context buffer + * @device_id: PCIe Routing Identifier of the device to connect to + * @root_port_id: PCIe Routing Identifier of the root port of the device + * @segment_id: PCIe Segment Identifier of the device to connect to + */ +struct sev_data_tio_dev_create { + u32 length; + u8 reserved1[4]; + struct sla_addr_t dev_ctx_sla; + u16 device_id; + u16 root_port_id; + u8 segment_id; + u8 reserved2[11]; +} __packed; + +/* + * struct sev_data_tio_dev_connect - TIO_DEV_CONNECT command + * + * @length: Length in bytes of this command buffer + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + * @tc_mask: Bitmask of the traffic classes to initialize for SEV-TIO usage. + * Setting the kth bit of the TC_MASK to 1 indicates that the traffic + * class k will be initialized + * @cert_slot: Slot number of the certificate requested for constructing the SPDM session + * @ide_stream_id: IDE stream IDs to be associated with this device. + * Valid only if corresponding bit in TC_MASK is set + */ +struct sev_data_tio_dev_connect { + u32 length; + u8 reserved1[4]; + struct spdm_ctrl spdm_ctrl; + u8 reserved2[8]; + struct sla_addr_t dev_ctx_sla; + u8 tc_mask; + u8 cert_slot; + u8 reserved3[6]; + u8 ide_stream_id[8]; + u8 reserved4[8]; +} __packed; + +/* + * struct sev_data_tio_dev_disconnect - TIO_DEV_DISCONNECT command + * + * @length: Length in bytes of this command buffer + * @flags: Command flags (TIO_DEV_DISCONNECT_FLAG_FORCE: force disconnect) + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + */ +#define TIO_DEV_DISCONNECT_FLAG_FORCE BIT(0) + +struct sev_data_tio_dev_disconnect { + u32 length; + u32 flags; + struct spdm_ctrl spdm_ctrl; + struct sla_addr_t dev_ctx_sla; +} __packed; + +/* + * struct sev_data_tio_dev_meas - TIO_DEV_MEASUREMENTS command + * + * @length: Length in bytes of this command buffer + * @flags: Command flags (TIO_DEV_MEAS_FLAG_RAW_BITSTREAM: request raw measurements) + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + * @meas_nonce: Nonce for measurement freshness verification + */ +#define TIO_DEV_MEAS_FLAG_RAW_BITSTREAM BIT(0) + +struct sev_data_tio_dev_meas { + u32 length; + u32 flags; + struct spdm_ctrl spdm_ctrl; + struct sla_addr_t dev_ctx_sla; + u8 meas_nonce[32]; +} __packed; + +/* + * struct sev_data_tio_dev_certs - TIO_DEV_CERTIFICATES command + * + * @length: Length in bytes of this command buffer + * @spdm_ctrl: SPDM control structure defined in Section 5.1 + * @dev_ctx_sla: Scatter list address of the device context buffer + */ +struct sev_data_tio_dev_certs { + u32 length; + u8 reserved[4]; + struct spdm_ctrl spdm_ctrl; + struct sla_addr_t dev_ctx_sla; +} __packed; + +/* + * struct sev_data_tio_dev_reclaim - TIO_DEV_RECLAIM command + * + * @length: Length in bytes of this command buffer + * @dev_ctx_sla: Scatter list address of the device context buffer + * + * This command reclaims resources associated with a device context. + */ +struct sev_data_tio_dev_reclaim { + u32 length; + u8 reserved[4]; + struct sla_addr_t dev_ctx_sla; +} __packed; + +static struct sla_buffer_hdr *sla_buffer_map(struct sla_addr_t sla) +{ + struct sla_buffer_hdr *buf; + + BUILD_BUG_ON(sizeof(struct sla_buffer_hdr) != 0x40); + if (IS_SLA_NULL(sla)) + return NULL; + + if (sla.page_type == SLA_PAGE_TYPE_SCATTER) { + struct sla_addr_t *scatter = sla_to_va(sla); + unsigned int i, npages = 0; + + for (i = 0; i < SLA_SCATTER_LEN(sla); ++i) { + if (WARN_ON_ONCE(SLA_SZ(scatter[i]) > SZ_4K)) + return NULL; + + if (WARN_ON_ONCE(scatter[i].page_type == SLA_PAGE_TYPE_SCATTER)) + return NULL; + + if (IS_SLA_EOL(scatter[i])) { + npages = i; + break; + } + } + if (WARN_ON_ONCE(!npages)) + return NULL; + + struct page **pp = kmalloc_array(npages, sizeof(pp[0]), GFP_KERNEL); + + if (!pp) + return NULL; + + for (i = 0; i < npages; ++i) + pp[i] = sla_to_page(scatter[i]); + + buf = vm_map_ram(pp, npages, 0); + kfree(pp); + } else { + struct page *pg = sla_to_page(sla); + + buf = vm_map_ram(&pg, 1, 0); + } + + return buf; +} + +static void sla_buffer_unmap(struct sla_addr_t sla, struct sla_buffer_hdr *buf) +{ + if (!buf) + return; + + if (sla.page_type == SLA_PAGE_TYPE_SCATTER) { + struct sla_addr_t *scatter = sla_to_va(sla); + unsigned int i, npages = 0; + + for (i = 0; i < SLA_SCATTER_LEN(sla); ++i) { + if (IS_SLA_EOL(scatter[i])) { + npages = i; + break; + } + } + if (!npages) + return; + + vm_unmap_ram(buf, npages); + } else { + vm_unmap_ram(buf, 1); + } +} + +static void dobj_response_init(struct sla_buffer_hdr *buf) +{ + struct spdm_dobj_hdr *dobj = sla_to_dobj_hdr(buf); + + dobj->id = SPDM_DOBJ_ID_RESP; + dobj->version.major = 0x1; + dobj->version.minor = 0; + dobj->length = 0; + buf->payload_sz = sla_dobj_id_to_size(dobj->id) + dobj->length; +} + +static void sla_free(struct sla_addr_t sla, size_t len, bool firmware_state) +{ + unsigned int npages = PAGE_ALIGN(len) >> PAGE_SHIFT; + struct sla_addr_t *scatter = NULL; + int ret = 0, i; + + if (IS_SLA_NULL(sla)) + return; + + if (firmware_state) { + if (sla.page_type == SLA_PAGE_TYPE_SCATTER) { + scatter = sla_to_va(sla); + + for (i = 0; i < npages; ++i) { + if (IS_SLA_EOL(scatter[i])) + break; + + ret = snp_reclaim_pages(sla_to_pa(scatter[i]), 1, false); + if (ret) + break; + } + } else { + ret = snp_reclaim_pages(sla_to_pa(sla), 1, false); + } + } + + if (WARN_ON(ret)) + return; + + if (scatter) { + for (i = 0; i < npages; ++i) { + if (IS_SLA_EOL(scatter[i])) + break; + free_page((unsigned long)sla_to_va(scatter[i])); + } + } + + free_page((unsigned long)sla_to_va(sla)); +} + +static struct sla_addr_t sla_alloc(size_t len, bool firmware_state) +{ + unsigned long i, npages = PAGE_ALIGN(len) >> PAGE_SHIFT; + struct sla_addr_t *scatter = NULL; + struct sla_addr_t ret = SLA_NULL; + struct sla_buffer_hdr *buf; + struct page *pg; + + if (npages == 0) + return ret; + + if (WARN_ON_ONCE(npages > ((PAGE_SIZE / sizeof(struct sla_addr_t)) + 1))) + return ret; + + BUILD_BUG_ON(PAGE_SIZE < SZ_4K); + + if (npages > 1) { + pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pg) + return SLA_NULL; + + ret = make_sla(pg, true); + scatter = page_to_virt(pg); + for (i = 0; i < npages; ++i) { + pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pg) + goto no_reclaim_exit; + + scatter[i] = make_sla(pg, false); + } + scatter[i] = SLA_EOL; + } else { + pg = alloc_page(GFP_KERNEL | __GFP_ZERO); + if (!pg) + return SLA_NULL; + + ret = make_sla(pg, false); + } + + buf = sla_buffer_map(ret); + if (!buf) + goto no_reclaim_exit; + + buf->capacity_sz = (npages << PAGE_SHIFT); + sla_buffer_unmap(ret, buf); + + if (firmware_state) { + if (scatter) { + for (i = 0; i < npages; ++i) { + if (rmp_make_private(sla_to_pfn(scatter[i]), 0, + PG_LEVEL_4K, 0, true)) + goto free_exit; + } + } else { + if (rmp_make_private(sla_to_pfn(ret), 0, PG_LEVEL_4K, 0, true)) + goto no_reclaim_exit; + } + } + + return ret; + +no_reclaim_exit: + firmware_state = false; +free_exit: + sla_free(ret, len, firmware_state); + return SLA_NULL; +} + +/* Expands a buffer, only firmware owned buffers allowed for now */ +static int sla_expand(struct sla_addr_t *sla, size_t *len) +{ + struct sla_buffer_hdr *oldbuf = sla_buffer_map(*sla), *newbuf; + struct sla_addr_t oldsla = *sla, newsla; + size_t oldlen = *len, newlen; + + if (!oldbuf) + return -EFAULT; + + newlen = oldbuf->capacity_sz; + if (oldbuf->capacity_sz == oldlen) { + /* This buffer does not require expansion, must be another buffer */ + sla_buffer_unmap(oldsla, oldbuf); + return 1; + } + + pr_notice("Expanding BUFFER from %ld to %ld bytes\n", oldlen, newlen); + + newsla = sla_alloc(newlen, true); + if (IS_SLA_NULL(newsla)) + return -ENOMEM; + + newbuf = sla_buffer_map(newsla); + if (!newbuf) { + sla_free(newsla, newlen, true); + return -EFAULT; + } + + memcpy(newbuf, oldbuf, oldlen); + + sla_buffer_unmap(newsla, newbuf); + sla_free(oldsla, oldlen, true); + *sla = newsla; + *len = newlen; + + return 0; +} + +static int sev_tio_do_cmd(int cmd, void *data, size_t data_len, int *psp_ret, + struct tsm_dsm_tio *dev_data) +{ + int rc; + + *psp_ret = 0; + rc = sev_do_cmd(cmd, data, psp_ret); + + if (WARN_ON(!rc && *psp_ret == SEV_RET_SPDM_REQUEST)) + return -EIO; + + if (rc == 0 && *psp_ret == SEV_RET_EXPAND_BUFFER_LENGTH_REQUEST) { + int rc1, rc2; + + rc1 = sla_expand(&dev_data->output, &dev_data->output_len); + if (rc1 < 0) + return rc1; + + rc2 = sla_expand(&dev_data->scratch, &dev_data->scratch_len); + if (rc2 < 0) + return rc2; + + if (!rc1 && !rc2) + /* Neither buffer requires expansion, this is wrong */ + return -EFAULT; + + *psp_ret = 0; + rc = sev_do_cmd(cmd, data, psp_ret); + } + + if ((rc == 0 || rc == -EIO) && *psp_ret == SEV_RET_SPDM_REQUEST) { + struct spdm_dobj_hdr_resp *resp_hdr; + struct spdm_dobj_hdr_req *req_hdr; + struct sev_tio_status *tio_status = to_tio_status(dev_data); + size_t resp_len = tio_status->spdm_req_size_max - + (sla_dobj_id_to_size(SPDM_DOBJ_ID_RESP) + sizeof(struct sla_buffer_hdr)); + + if (!dev_data->cmd) { + if (WARN_ON_ONCE(!data_len || (data_len != *(u32 *) data))) + return -EINVAL; + if (WARN_ON(data_len > sizeof(dev_data->cmd_data))) + return -EFAULT; + memcpy(dev_data->cmd_data, data, data_len); + memset(&dev_data->cmd_data[data_len], 0xFF, + sizeof(dev_data->cmd_data) - data_len); + dev_data->cmd = cmd; + } + + req_hdr = sla_to_dobj_req_hdr(dev_data->reqbuf); + resp_hdr = sla_to_dobj_resp_hdr(dev_data->respbuf); + switch (req_hdr->data_type) { + case DOBJ_DATA_TYPE_SPDM: + rc = PCI_DOE_FEATURE_CMA; + break; + case DOBJ_DATA_TYPE_SECURE_SPDM: + rc = PCI_DOE_FEATURE_SSESSION; + break; + default: + return -EINVAL; + } + resp_hdr->data_type = req_hdr->data_type; + dev_data->spdm.req_len = req_hdr->hdr.length - + sla_dobj_id_to_size(SPDM_DOBJ_ID_REQ); + dev_data->spdm.rsp_len = resp_len; + } else if (dev_data && dev_data->cmd) { + /* For either error or success just stop the bouncing */ + memset(dev_data->cmd_data, 0, sizeof(dev_data->cmd_data)); + dev_data->cmd = 0; + } + + return rc; +} + +int sev_tio_continue(struct tsm_dsm_tio *dev_data) +{ + struct spdm_dobj_hdr_resp *resp_hdr; + int ret; + + if (!dev_data || !dev_data->cmd) + return -EINVAL; + + resp_hdr = sla_to_dobj_resp_hdr(dev_data->respbuf); + resp_hdr->hdr.length = ALIGN(sla_dobj_id_to_size(SPDM_DOBJ_ID_RESP) + + dev_data->spdm.rsp_len, 32); + dev_data->respbuf->payload_sz = resp_hdr->hdr.length; + + ret = sev_tio_do_cmd(dev_data->cmd, dev_data->cmd_data, 0, + &dev_data->psp_ret, dev_data); + if (ret) + return ret; + + if (dev_data->psp_ret != SEV_RET_SUCCESS) + return -EINVAL; + + return 0; +} + +static void spdm_ctrl_init(struct spdm_ctrl *ctrl, struct tsm_dsm_tio *dev_data) +{ + ctrl->req = dev_data->req; + ctrl->resp = dev_data->resp; + ctrl->scratch = dev_data->scratch; + ctrl->output = dev_data->output; +} + +static void spdm_ctrl_free(struct tsm_dsm_tio *dev_data) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + size_t len = tio_status->spdm_req_size_max - + (sla_dobj_id_to_size(SPDM_DOBJ_ID_RESP) + + sizeof(struct sla_buffer_hdr)); + struct tsm_spdm *spdm = &dev_data->spdm; + + sla_buffer_unmap(dev_data->resp, dev_data->respbuf); + sla_buffer_unmap(dev_data->req, dev_data->reqbuf); + spdm->rsp = NULL; + spdm->req = NULL; + sla_free(dev_data->req, len, true); + sla_free(dev_data->resp, len, false); + sla_free(dev_data->scratch, tio_status->spdm_scratch_size_max, true); + + dev_data->req.sla = 0; + dev_data->resp.sla = 0; + dev_data->scratch.sla = 0; + dev_data->respbuf = NULL; + dev_data->reqbuf = NULL; + sla_free(dev_data->output, tio_status->spdm_out_size_max, true); +} + +static int spdm_ctrl_alloc(struct tsm_dsm_tio *dev_data) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + struct tsm_spdm *spdm = &dev_data->spdm; + int ret; + + dev_data->req = sla_alloc(tio_status->spdm_req_size_max, true); + dev_data->resp = sla_alloc(tio_status->spdm_req_size_max, false); + dev_data->scratch_len = tio_status->spdm_scratch_size_max; + dev_data->scratch = sla_alloc(dev_data->scratch_len, true); + dev_data->output_len = tio_status->spdm_out_size_max; + dev_data->output = sla_alloc(dev_data->output_len, true); + + if (IS_SLA_NULL(dev_data->req) || IS_SLA_NULL(dev_data->resp) || + IS_SLA_NULL(dev_data->scratch) || IS_SLA_NULL(dev_data->dev_ctx)) { + ret = -ENOMEM; + goto free_spdm_exit; + } + + dev_data->reqbuf = sla_buffer_map(dev_data->req); + dev_data->respbuf = sla_buffer_map(dev_data->resp); + if (!dev_data->reqbuf || !dev_data->respbuf) { + ret = -EFAULT; + goto free_spdm_exit; + } + + spdm->req = sla_to_data(dev_data->reqbuf, SPDM_DOBJ_ID_REQ); + spdm->rsp = sla_to_data(dev_data->respbuf, SPDM_DOBJ_ID_RESP); + if (!spdm->req || !spdm->rsp) { + ret = -EFAULT; + goto free_spdm_exit; + } + + dobj_response_init(dev_data->respbuf); + + return 0; + +free_spdm_exit: + spdm_ctrl_free(dev_data); + return ret; +} + +int sev_tio_init_locked(void *tio_status_page) +{ + struct sev_tio_status *tio_status = tio_status_page; + struct sev_data_tio_status data_status = { + .length = sizeof(data_status), + }; + int ret, psp_ret; + + data_status.status_paddr = __psp_pa(tio_status_page); + ret = __sev_do_cmd_locked(SEV_CMD_TIO_STATUS, &data_status, &psp_ret); + if (ret) + return ret; + + if (tio_status->length < offsetofend(struct sev_tio_status, tdictx_size) || + tio_status->reserved) + return -EFAULT; + + if (!tio_status->tio_en && !tio_status->tio_init_done) + return -ENOENT; + + if (tio_status->tio_init_done) + return -EBUSY; + + struct sev_data_tio_init ti = { .length = sizeof(ti) }; + + ret = __sev_do_cmd_locked(SEV_CMD_TIO_INIT, &ti, &psp_ret); + if (ret) + return ret; + + ret = __sev_do_cmd_locked(SEV_CMD_TIO_STATUS, &data_status, &psp_ret); + if (ret) + return ret; + + return 0; +} + +int sev_tio_dev_create(struct tsm_dsm_tio *dev_data, u16 device_id, + u16 root_port_id, u8 segment_id) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + struct sev_data_tio_dev_create create = { + .length = sizeof(create), + .device_id = device_id, + .root_port_id = root_port_id, + .segment_id = segment_id, + }; + void *data_pg; + int ret; + + dev_data->dev_ctx = sla_alloc(tio_status->devctx_size, true); + if (IS_SLA_NULL(dev_data->dev_ctx)) + return -ENOMEM; + + data_pg = snp_alloc_firmware_page(GFP_KERNEL_ACCOUNT); + if (!data_pg) { + ret = -ENOMEM; + goto free_ctx_exit; + } + + create.dev_ctx_sla = dev_data->dev_ctx; + ret = sev_do_cmd(SEV_CMD_TIO_DEV_CREATE, &create, &dev_data->psp_ret); + if (ret) + goto free_data_pg_exit; + + dev_data->data_pg = data_pg; + + return 0; + +free_data_pg_exit: + snp_free_firmware_page(data_pg); +free_ctx_exit: + sla_free(create.dev_ctx_sla, tio_status->devctx_size, true); + return ret; +} + +int sev_tio_dev_reclaim(struct tsm_dsm_tio *dev_data) +{ + struct sev_tio_status *tio_status = to_tio_status(dev_data); + struct sev_data_tio_dev_reclaim r = { + .length = sizeof(r), + .dev_ctx_sla = dev_data->dev_ctx, + }; + int ret; + + if (dev_data->data_pg) { + snp_free_firmware_page(dev_data->data_pg); + dev_data->data_pg = NULL; + } + + if (IS_SLA_NULL(dev_data->dev_ctx)) + return 0; + + ret = sev_do_cmd(SEV_CMD_TIO_DEV_RECLAIM, &r, &dev_data->psp_ret); + + sla_free(dev_data->dev_ctx, tio_status->devctx_size, true); + dev_data->dev_ctx = SLA_NULL; + + spdm_ctrl_free(dev_data); + + return ret; +} + +int sev_tio_dev_connect(struct tsm_dsm_tio *dev_data, u8 tc_mask, u8 ids[8], u8 cert_slot) +{ + struct sev_data_tio_dev_connect connect = { + .length = sizeof(connect), + .tc_mask = tc_mask, + .cert_slot = cert_slot, + .dev_ctx_sla = dev_data->dev_ctx, + .ide_stream_id = { + ids[0], ids[1], ids[2], ids[3], + ids[4], ids[5], ids[6], ids[7] + }, + }; + int ret; + + if (WARN_ON(IS_SLA_NULL(dev_data->dev_ctx))) + return -EFAULT; + if (!(tc_mask & 1)) + return -EINVAL; + + ret = spdm_ctrl_alloc(dev_data); + if (ret) + return ret; + + spdm_ctrl_init(&connect.spdm_ctrl, dev_data); + + return sev_tio_do_cmd(SEV_CMD_TIO_DEV_CONNECT, &connect, sizeof(connect), + &dev_data->psp_ret, dev_data); +} + +int sev_tio_dev_disconnect(struct tsm_dsm_tio *dev_data, bool force) +{ + struct sev_data_tio_dev_disconnect dc = { + .length = sizeof(dc), + .dev_ctx_sla = dev_data->dev_ctx, + .flags = force ? TIO_DEV_DISCONNECT_FLAG_FORCE : 0, + }; + + if (WARN_ON_ONCE(IS_SLA_NULL(dev_data->dev_ctx))) + return -EFAULT; + + spdm_ctrl_init(&dc.spdm_ctrl, dev_data); + + return sev_tio_do_cmd(SEV_CMD_TIO_DEV_DISCONNECT, &dc, sizeof(dc), + &dev_data->psp_ret, dev_data); +} + +int sev_tio_cmd_buffer_len(int cmd) +{ + switch (cmd) { + case SEV_CMD_TIO_STATUS: return sizeof(struct sev_data_tio_status); + case SEV_CMD_TIO_INIT: return sizeof(struct sev_data_tio_init); + case SEV_CMD_TIO_DEV_CREATE: return sizeof(struct sev_data_tio_dev_create); + case SEV_CMD_TIO_DEV_RECLAIM: return sizeof(struct sev_data_tio_dev_reclaim); + case SEV_CMD_TIO_DEV_CONNECT: return sizeof(struct sev_data_tio_dev_connect); + case SEV_CMD_TIO_DEV_DISCONNECT: return sizeof(struct sev_data_tio_dev_disconnect); + default: return 0; + } +} diff --git a/drivers/crypto/ccp/sev-dev-tio.h b/drivers/crypto/ccp/sev-dev-tio.h new file mode 100644 index 000000000000..67512b3dbc53 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev-tio.h @@ -0,0 +1,123 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +#ifndef __PSP_SEV_TIO_H__ +#define __PSP_SEV_TIO_H__ + +#include <linux/pci-tsm.h> +#include <linux/pci-ide.h> +#include <linux/tsm.h> +#include <uapi/linux/psp-sev.h> + +struct sla_addr_t { + union { + u64 sla; + struct { + u64 page_type :1, + page_size :1, + reserved1 :10, + pfn :40, + reserved2 :12; + }; + }; +} __packed; + +#define SEV_TIO_MAX_COMMAND_LENGTH 128 + +/* SPDM control structure for DOE */ +struct tsm_spdm { + unsigned long req_len; + void *req; + unsigned long rsp_len; + void *rsp; +}; + +/* Describes TIO device */ +struct tsm_dsm_tio { + u8 cert_slot; + struct sla_addr_t dev_ctx; + struct sla_addr_t req; + struct sla_addr_t resp; + struct sla_addr_t scratch; + struct sla_addr_t output; + size_t output_len; + size_t scratch_len; + struct tsm_spdm spdm; + struct sla_buffer_hdr *reqbuf; /* vmap'ed @req for DOE */ + struct sla_buffer_hdr *respbuf; /* vmap'ed @resp for DOE */ + + int cmd; + int psp_ret; + u8 cmd_data[SEV_TIO_MAX_COMMAND_LENGTH]; + void *data_pg; /* Data page for DEV_STATUS/TDI_STATUS/TDI_INFO/ASID_FENCE */ + +#define TIO_IDE_MAX_TC 8 + struct pci_ide *ide[TIO_IDE_MAX_TC]; +}; + +/* Describes TSM structure for PF0 pointed by pci_dev->tsm */ +struct tio_dsm { + struct pci_tsm_pf0 tsm; + struct tsm_dsm_tio data; + struct sev_device *sev; +}; + +/* Data object IDs */ +#define SPDM_DOBJ_ID_NONE 0 +#define SPDM_DOBJ_ID_REQ 1 +#define SPDM_DOBJ_ID_RESP 2 + +struct spdm_dobj_hdr { + u32 id; /* Data object type identifier */ + u32 length; /* Length of the data object, INCLUDING THIS HEADER */ + struct { /* Version of the data object structure */ + u8 minor; + u8 major; + } version; +} __packed; + +/** + * struct sev_tio_status - TIO_STATUS command's info_paddr buffer + * + * @length: Length of this structure in bytes + * @tio_en: Indicates that SNP_INIT_EX initialized the RMP for SEV-TIO + * @tio_init_done: Indicates TIO_INIT has been invoked + * @spdm_req_size_min: Minimum SPDM request buffer size in bytes + * @spdm_req_size_max: Maximum SPDM request buffer size in bytes + * @spdm_scratch_size_min: Minimum SPDM scratch buffer size in bytes + * @spdm_scratch_size_max: Maximum SPDM scratch buffer size in bytes + * @spdm_out_size_min: Minimum SPDM output buffer size in bytes + * @spdm_out_size_max: Maximum for the SPDM output buffer size in bytes + * @spdm_rsp_size_min: Minimum SPDM response buffer size in bytes + * @spdm_rsp_size_max: Maximum SPDM response buffer size in bytes + * @devctx_size: Size of a device context buffer in bytes + * @tdictx_size: Size of a TDI context buffer in bytes + * @tio_crypto_alg: TIO crypto algorithms supported + */ +struct sev_tio_status { + u32 length; + u32 tio_en :1, + tio_init_done :1, + reserved :30; + u32 spdm_req_size_min; + u32 spdm_req_size_max; + u32 spdm_scratch_size_min; + u32 spdm_scratch_size_max; + u32 spdm_out_size_min; + u32 spdm_out_size_max; + u32 spdm_rsp_size_min; + u32 spdm_rsp_size_max; + u32 devctx_size; + u32 tdictx_size; + u32 tio_crypto_alg; + u8 reserved2[12]; +} __packed; + +int sev_tio_init_locked(void *tio_status_page); +int sev_tio_continue(struct tsm_dsm_tio *dev_data); + +int sev_tio_dev_create(struct tsm_dsm_tio *dev_data, u16 device_id, u16 root_port_id, + u8 segment_id); +int sev_tio_dev_connect(struct tsm_dsm_tio *dev_data, u8 tc_mask, u8 ids[8], u8 cert_slot); +int sev_tio_dev_disconnect(struct tsm_dsm_tio *dev_data, bool force); +int sev_tio_dev_reclaim(struct tsm_dsm_tio *dev_data); + +#endif /* __PSP_SEV_TIO_H__ */ diff --git a/drivers/crypto/ccp/sev-dev-tsm.c b/drivers/crypto/ccp/sev-dev-tsm.c new file mode 100644 index 000000000000..ea29cd5d0ff9 --- /dev/null +++ b/drivers/crypto/ccp/sev-dev-tsm.c @@ -0,0 +1,405 @@ +// SPDX-License-Identifier: GPL-2.0-only + +// Interface to CCP/SEV-TIO for generic PCIe TDISP module + +#include <linux/pci.h> +#include <linux/device.h> +#include <linux/tsm.h> +#include <linux/iommu.h> +#include <linux/pci-doe.h> +#include <linux/bitfield.h> +#include <linux/module.h> + +#include <asm/sev-common.h> +#include <asm/sev.h> + +#include "psp-dev.h" +#include "sev-dev.h" +#include "sev-dev-tio.h" + +MODULE_IMPORT_NS("PCI_IDE"); + +#define TIO_DEFAULT_NR_IDE_STREAMS 1 + +static uint nr_ide_streams = TIO_DEFAULT_NR_IDE_STREAMS; +module_param_named(ide_nr, nr_ide_streams, uint, 0644); +MODULE_PARM_DESC(ide_nr, "Set the maximum number of IDE streams per PHB"); + +#define dev_to_sp(dev) ((struct sp_device *)dev_get_drvdata(dev)) +#define dev_to_psp(dev) ((struct psp_device *)(dev_to_sp(dev)->psp_data)) +#define dev_to_sev(dev) ((struct sev_device *)(dev_to_psp(dev)->sev_data)) +#define tsm_dev_to_sev(tsmdev) dev_to_sev((tsmdev)->dev.parent) + +#define pdev_to_tio_dsm(pdev) (container_of((pdev)->tsm, struct tio_dsm, tsm.base_tsm)) + +static int sev_tio_spdm_cmd(struct tio_dsm *dsm, int ret) +{ + struct tsm_dsm_tio *dev_data = &dsm->data; + struct tsm_spdm *spdm = &dev_data->spdm; + + /* Check the main command handler response before entering the loop */ + if (ret == 0 && dev_data->psp_ret != SEV_RET_SUCCESS) + return -EINVAL; + + if (ret <= 0) + return ret; + + /* ret > 0 means "SPDM requested" */ + while (ret == PCI_DOE_FEATURE_CMA || ret == PCI_DOE_FEATURE_SSESSION) { + ret = pci_doe(dsm->tsm.doe_mb, PCI_VENDOR_ID_PCI_SIG, ret, + spdm->req, spdm->req_len, spdm->rsp, spdm->rsp_len); + if (ret < 0) + break; + + WARN_ON_ONCE(ret == 0); /* The response should never be empty */ + spdm->rsp_len = ret; + ret = sev_tio_continue(dev_data); + } + + return ret; +} + +static int stream_enable(struct pci_ide *ide) +{ + struct pci_dev *rp = pcie_find_root_port(ide->pdev); + int ret; + + ret = pci_ide_stream_enable(rp, ide); + if (ret) + return ret; + + ret = pci_ide_stream_enable(ide->pdev, ide); + if (ret) + pci_ide_stream_disable(rp, ide); + + return ret; +} + +static int streams_enable(struct pci_ide **ide) +{ + int ret = 0; + + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (ide[i]) { + ret = stream_enable(ide[i]); + if (ret) + break; + } + } + + return ret; +} + +static void stream_disable(struct pci_ide *ide) +{ + pci_ide_stream_disable(ide->pdev, ide); + pci_ide_stream_disable(pcie_find_root_port(ide->pdev), ide); +} + +static void streams_disable(struct pci_ide **ide) +{ + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) + if (ide[i]) + stream_disable(ide[i]); +} + +static void stream_setup(struct pci_ide *ide) +{ + struct pci_dev *rp = pcie_find_root_port(ide->pdev); + + ide->partner[PCI_IDE_EP].rid_start = 0; + ide->partner[PCI_IDE_EP].rid_end = 0xffff; + ide->partner[PCI_IDE_RP].rid_start = 0; + ide->partner[PCI_IDE_RP].rid_end = 0xffff; + + ide->pdev->ide_cfg = 0; + ide->pdev->ide_tee_limit = 1; + rp->ide_cfg = 1; + rp->ide_tee_limit = 0; + + pci_warn(ide->pdev, "Forcing CFG/TEE for %s", pci_name(rp)); + pci_ide_stream_setup(ide->pdev, ide); + pci_ide_stream_setup(rp, ide); +} + +static u8 streams_setup(struct pci_ide **ide, u8 *ids) +{ + bool def = false; + u8 tc_mask = 0; + int i; + + for (i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (!ide[i]) { + ids[i] = 0xFF; + continue; + } + + tc_mask |= BIT(i); + ids[i] = ide[i]->stream_id; + + if (!def) { + struct pci_ide_partner *settings; + + settings = pci_ide_to_settings(ide[i]->pdev, ide[i]); + settings->default_stream = 1; + def = true; + } + + stream_setup(ide[i]); + } + + return tc_mask; +} + +static int streams_register(struct pci_ide **ide) +{ + int ret = 0, i; + + for (i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (ide[i]) { + ret = pci_ide_stream_register(ide[i]); + if (ret) + break; + } + } + + return ret; +} + +static void streams_unregister(struct pci_ide **ide) +{ + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) + if (ide[i]) + pci_ide_stream_unregister(ide[i]); +} + +static void stream_teardown(struct pci_ide *ide) +{ + pci_ide_stream_teardown(ide->pdev, ide); + pci_ide_stream_teardown(pcie_find_root_port(ide->pdev), ide); +} + +static void streams_teardown(struct pci_ide **ide) +{ + for (int i = 0; i < TIO_IDE_MAX_TC; ++i) { + if (ide[i]) { + stream_teardown(ide[i]); + pci_ide_stream_free(ide[i]); + ide[i] = NULL; + } + } +} + +static int stream_alloc(struct pci_dev *pdev, struct pci_ide **ide, + unsigned int tc) +{ + struct pci_dev *rp = pcie_find_root_port(pdev); + struct pci_ide *ide1; + + if (ide[tc]) { + pci_err(pdev, "Stream for class=%d already registered", tc); + return -EBUSY; + } + + /* FIXME: find a better way */ + if (nr_ide_streams != TIO_DEFAULT_NR_IDE_STREAMS) + pci_notice(pdev, "Enable non-default %d streams", nr_ide_streams); + pci_ide_set_nr_streams(to_pci_host_bridge(rp->bus->bridge), nr_ide_streams); + + ide1 = pci_ide_stream_alloc(pdev); + if (!ide1) + return -EFAULT; + + /* Blindly assign streamid=0 to TC=0, and so on */ + ide1->stream_id = tc; + + ide[tc] = ide1; + + return 0; +} + +static struct pci_tsm *tio_pf0_probe(struct pci_dev *pdev, struct sev_device *sev) +{ + struct tio_dsm *dsm __free(kfree) = kzalloc(sizeof(*dsm), GFP_KERNEL); + int rc; + + if (!dsm) + return NULL; + + rc = pci_tsm_pf0_constructor(pdev, &dsm->tsm, sev->tsmdev); + if (rc) + return NULL; + + pci_dbg(pdev, "TSM enabled\n"); + dsm->sev = sev; + return &no_free_ptr(dsm)->tsm.base_tsm; +} + +static struct pci_tsm *dsm_probe(struct tsm_dev *tsmdev, struct pci_dev *pdev) +{ + struct sev_device *sev = tsm_dev_to_sev(tsmdev); + + if (is_pci_tsm_pf0(pdev)) + return tio_pf0_probe(pdev, sev); + return 0; +} + +static void dsm_remove(struct pci_tsm *tsm) +{ + struct pci_dev *pdev = tsm->pdev; + + pci_dbg(pdev, "TSM disabled\n"); + + if (is_pci_tsm_pf0(pdev)) { + struct tio_dsm *dsm = container_of(tsm, struct tio_dsm, tsm.base_tsm); + + pci_tsm_pf0_destructor(&dsm->tsm); + kfree(dsm); + } +} + +static int dsm_create(struct tio_dsm *dsm) +{ + struct pci_dev *pdev = dsm->tsm.base_tsm.pdev; + u8 segment_id = pdev->bus ? pci_domain_nr(pdev->bus) : 0; + struct pci_dev *rootport = pcie_find_root_port(pdev); + u16 device_id = pci_dev_id(pdev); + u16 root_port_id; + u32 lnkcap = 0; + + if (pci_read_config_dword(rootport, pci_pcie_cap(rootport) + PCI_EXP_LNKCAP, + &lnkcap)) + return -ENODEV; + + root_port_id = FIELD_GET(PCI_EXP_LNKCAP_PN, lnkcap); + + return sev_tio_dev_create(&dsm->data, device_id, root_port_id, segment_id); +} + +static int dsm_connect(struct pci_dev *pdev) +{ + struct tio_dsm *dsm = pdev_to_tio_dsm(pdev); + struct tsm_dsm_tio *dev_data = &dsm->data; + u8 ids[TIO_IDE_MAX_TC]; + u8 tc_mask; + int ret; + + if (pci_find_doe_mailbox(pdev, PCI_VENDOR_ID_PCI_SIG, + PCI_DOE_FEATURE_SSESSION) != dsm->tsm.doe_mb) { + pci_err(pdev, "CMA DOE MB must support SSESSION\n"); + return -EFAULT; + } + + ret = stream_alloc(pdev, dev_data->ide, 0); + if (ret) + return ret; + + ret = dsm_create(dsm); + if (ret) + goto ide_free_exit; + + tc_mask = streams_setup(dev_data->ide, ids); + + ret = sev_tio_dev_connect(dev_data, tc_mask, ids, dev_data->cert_slot); + ret = sev_tio_spdm_cmd(dsm, ret); + if (ret) + goto free_exit; + + streams_enable(dev_data->ide); + + ret = streams_register(dev_data->ide); + if (ret) + goto free_exit; + + return 0; + +free_exit: + sev_tio_dev_reclaim(dev_data); + + streams_disable(dev_data->ide); +ide_free_exit: + + streams_teardown(dev_data->ide); + + return ret; +} + +static void dsm_disconnect(struct pci_dev *pdev) +{ + bool force = SYSTEM_HALT <= system_state && system_state <= SYSTEM_RESTART; + struct tio_dsm *dsm = pdev_to_tio_dsm(pdev); + struct tsm_dsm_tio *dev_data = &dsm->data; + int ret; + + ret = sev_tio_dev_disconnect(dev_data, force); + ret = sev_tio_spdm_cmd(dsm, ret); + if (ret && !force) { + ret = sev_tio_dev_disconnect(dev_data, true); + sev_tio_spdm_cmd(dsm, ret); + } + + sev_tio_dev_reclaim(dev_data); + + streams_disable(dev_data->ide); + streams_unregister(dev_data->ide); + streams_teardown(dev_data->ide); +} + +static struct pci_tsm_ops sev_tsm_ops = { + .probe = dsm_probe, + .remove = dsm_remove, + .connect = dsm_connect, + .disconnect = dsm_disconnect, +}; + +void sev_tsm_init_locked(struct sev_device *sev, void *tio_status_page) +{ + struct sev_tio_status *t = kzalloc(sizeof(*t), GFP_KERNEL); + struct tsm_dev *tsmdev; + int ret; + + WARN_ON(sev->tio_status); + + if (!t) + return; + + ret = sev_tio_init_locked(tio_status_page); + if (ret) { + pr_warn("SEV-TIO STATUS failed with %d\n", ret); + goto error_exit; + } + + tsmdev = tsm_register(sev->dev, &sev_tsm_ops); + if (IS_ERR(tsmdev)) + goto error_exit; + + memcpy(t, tio_status_page, sizeof(*t)); + + pr_notice("SEV-TIO status: EN=%d INIT_DONE=%d rq=%d..%d rs=%d..%d " + "scr=%d..%d out=%d..%d dev=%d tdi=%d algos=%x\n", + t->tio_en, t->tio_init_done, + t->spdm_req_size_min, t->spdm_req_size_max, + t->spdm_rsp_size_min, t->spdm_rsp_size_max, + t->spdm_scratch_size_min, t->spdm_scratch_size_max, + t->spdm_out_size_min, t->spdm_out_size_max, + t->devctx_size, t->tdictx_size, + t->tio_crypto_alg); + + sev->tsmdev = tsmdev; + sev->tio_status = t; + + return; + +error_exit: + kfree(t); + pr_err("Failed to enable SEV-TIO: ret=%d en=%d initdone=%d SEV=%d\n", + ret, t->tio_en, t->tio_init_done, boot_cpu_has(X86_FEATURE_SEV)); +} + +void sev_tsm_uninit(struct sev_device *sev) +{ + if (sev->tsmdev) + tsm_unregister(sev->tsmdev); + + sev->tsmdev = NULL; +} diff --git a/drivers/crypto/ccp/sev-dev.c b/drivers/crypto/ccp/sev-dev.c index 52ba892444a8..956ea609d0cc 100644 --- a/drivers/crypto/ccp/sev-dev.c +++ b/drivers/crypto/ccp/sev-dev.c @@ -75,6 +75,14 @@ static bool psp_init_on_probe = true; module_param(psp_init_on_probe, bool, 0444); MODULE_PARM_DESC(psp_init_on_probe, " if true, the PSP will be initialized on module init. Else the PSP will be initialized on the first command requiring it"); +#if IS_ENABLED(CONFIG_PCI_TSM) +static bool sev_tio_enabled = true; +module_param_named(tio, sev_tio_enabled, bool, 0444); +MODULE_PARM_DESC(tio, "Enables TIO in SNP_INIT_EX"); +#else +static const bool sev_tio_enabled = false; +#endif + MODULE_FIRMWARE("amd/amd_sev_fam17h_model0xh.sbin"); /* 1st gen EPYC */ MODULE_FIRMWARE("amd/amd_sev_fam17h_model3xh.sbin"); /* 2nd gen EPYC */ MODULE_FIRMWARE("amd/amd_sev_fam19h_model0xh.sbin"); /* 3rd gen EPYC */ @@ -251,7 +259,7 @@ static int sev_cmd_buffer_len(int cmd) case SEV_CMD_SNP_COMMIT: return sizeof(struct sev_data_snp_commit); case SEV_CMD_SNP_FEATURE_INFO: return sizeof(struct sev_data_snp_feature_info); case SEV_CMD_SNP_VLEK_LOAD: return sizeof(struct sev_user_data_snp_vlek_load); - default: return 0; + default: return sev_tio_cmd_buffer_len(cmd); } return 0; @@ -380,13 +388,7 @@ static int sev_write_init_ex_file_if_required(int cmd_id) return sev_write_init_ex_file(); } -/* - * snp_reclaim_pages() needs __sev_do_cmd_locked(), and __sev_do_cmd_locked() - * needs snp_reclaim_pages(), so a forward declaration is needed. - */ -static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret); - -static int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked) +int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked) { int ret, err, i; @@ -420,6 +422,7 @@ cleanup: snp_leak_pages(__phys_to_pfn(paddr), npages - i); return ret; } +EXPORT_SYMBOL_GPL(snp_reclaim_pages); static int rmp_mark_pages_firmware(unsigned long paddr, unsigned int npages, bool locked) { @@ -850,7 +853,7 @@ static int snp_reclaim_cmd_buf(int cmd, void *cmd_buf) return 0; } -static int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) +int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret) { struct cmd_buf_desc desc_list[CMD_BUF_DESC_MAX] = {0}; struct psp_device *psp = psp_master; @@ -1392,6 +1395,8 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) * */ if (sev_version_greater_or_equal(SNP_MIN_API_MAJOR, 52)) { + bool tio_supp = !!(sev->snp_feat_info_0.ebx & SNP_SEV_TIO_SUPPORTED); + /* * Firmware checks that the pages containing the ranges enumerated * in the RANGES structure are either in the default page state or in the @@ -1432,6 +1437,17 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) data.init_rmp = 1; data.list_paddr_en = 1; data.list_paddr = __psp_pa(snp_range_list); + + data.tio_en = tio_supp && sev_tio_enabled && amd_iommu_sev_tio_supported(); + + /* + * When psp_init_on_probe is disabled, the userspace calling + * SEV ioctl can inadvertently shut down SNP and SEV-TIO causing + * unexpected state loss. + */ + if (data.tio_en && !psp_init_on_probe) + dev_warn(sev->dev, "SEV-TIO as incompatible with psp_init_on_probe=0\n"); + cmd = SEV_CMD_SNP_INIT_EX; } else { cmd = SEV_CMD_SNP_INIT; @@ -1469,7 +1485,8 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) snp_hv_fixed_pages_state_update(sev, HV_FIXED); sev->snp_initialized = true; - dev_dbg(sev->dev, "SEV-SNP firmware initialized\n"); + dev_dbg(sev->dev, "SEV-SNP firmware initialized, SEV-TIO is %s\n", + data.tio_en ? "enabled" : "disabled"); dev_info(sev->dev, "SEV-SNP API:%d.%d build:%d\n", sev->api_major, sev->api_minor, sev->build); @@ -1477,6 +1494,23 @@ static int __sev_snp_init_locked(int *error, unsigned int max_snp_asid) atomic_notifier_chain_register(&panic_notifier_list, &snp_panic_notifier); + if (data.tio_en) { + /* + * This executes with the sev_cmd_mutex held so down the stack + * snp_reclaim_pages(locked=false) might be needed (which is extremely + * unlikely) but will cause a deadlock. + * Instead of exporting __snp_alloc_firmware_pages(), allocate a page + * for this one call here. + */ + void *tio_status = page_address(__snp_alloc_firmware_pages( + GFP_KERNEL_ACCOUNT | __GFP_ZERO, 0, true)); + + if (tio_status) { + sev_tsm_init_locked(sev, tio_status); + __snp_free_firmware_pages(virt_to_page(tio_status), 0, true); + } + } + sev_es_tmr_size = SNP_TMR_SIZE; return 0; @@ -2756,8 +2790,20 @@ static void __sev_firmware_shutdown(struct sev_device *sev, bool panic) static void sev_firmware_shutdown(struct sev_device *sev) { + /* + * Calling without sev_cmd_mutex held as TSM will likely try disconnecting + * IDE and this ends up calling sev_do_cmd() which locks sev_cmd_mutex. + */ + if (sev->tio_status) + sev_tsm_uninit(sev); + mutex_lock(&sev_cmd_mutex); + __sev_firmware_shutdown(sev, false); + + kfree(sev->tio_status); + sev->tio_status = NULL; + mutex_unlock(&sev_cmd_mutex); } diff --git a/drivers/crypto/ccp/sev-dev.h b/drivers/crypto/ccp/sev-dev.h index ac03bd0848f7..b1cd556bbbf6 100644 --- a/drivers/crypto/ccp/sev-dev.h +++ b/drivers/crypto/ccp/sev-dev.h @@ -34,6 +34,8 @@ struct sev_misc_dev { struct miscdevice misc; }; +struct sev_tio_status; + struct sev_device { struct device *dev; struct psp_device *psp; @@ -61,15 +63,24 @@ struct sev_device { struct sev_user_data_snp_status snp_plat_status; struct snp_feature_info snp_feat_info_0; + + struct tsm_dev *tsmdev; + struct sev_tio_status *tio_status; }; int sev_dev_init(struct psp_device *psp); void sev_dev_destroy(struct psp_device *psp); +int __sev_do_cmd_locked(int cmd, void *data, int *psp_ret); + void sev_pci_init(void); void sev_pci_exit(void); struct page *snp_alloc_hv_fixed_pages(unsigned int num_2mb_pages); void snp_free_hv_fixed_pages(struct page *page); +void sev_tsm_init_locked(struct sev_device *sev, void *tio_status_page); +void sev_tsm_uninit(struct sev_device *sev); +int sev_tio_cmd_buffer_len(int cmd); + #endif /* __SEV_DEV_H */ diff --git a/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c index 69295a9ddf0a..4ccc94ed9493 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c +++ b/drivers/crypto/intel/qat/qat_common/adf_pm_dbgfs_utils.c @@ -1,18 +1,12 @@ // SPDX-License-Identifier: GPL-2.0-only /* Copyright(c) 2025 Intel Corporation */ +#include <linux/bitfield.h> #include <linux/bitops.h> #include <linux/sprintf.h> #include <linux/string_helpers.h> #include "adf_pm_dbgfs_utils.h" -/* - * This is needed because a variable is used to index the mask at - * pm_scnprint_table(), making it not compile time constant, so the compile - * asserts from FIELD_GET() or u32_get_bits() won't be fulfilled. - */ -#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) - #define PM_INFO_MAX_KEY_LEN 21 static int pm_scnprint_table(char *buff, const struct pm_status_row *table, diff --git a/drivers/edac/ie31200_edac.c b/drivers/edac/ie31200_edac.c index 8d4ddaa85ae8..eaab6af143e1 100644 --- a/drivers/edac/ie31200_edac.c +++ b/drivers/edac/ie31200_edac.c @@ -44,6 +44,7 @@ * but lo_hi_readq() ensures that we are safe across all e3-1200 processors. */ +#include <linux/bitfield.h> #include <linux/module.h> #include <linux/init.h> #include <linux/pci.h> @@ -139,9 +140,6 @@ #define IE31200_CAPID0_DDPCD BIT(6) #define IE31200_CAPID0_ECC BIT(1) -/* Non-constant mask variant of FIELD_GET() */ -#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) - static int nr_channels; static struct pci_dev *mci_pdev; static int ie31200_registered = 1; diff --git a/drivers/gpio/gpio-aspeed.c b/drivers/gpio/gpio-aspeed.c index 2e0ae953dd99..cbdf781994dc 100644 --- a/drivers/gpio/gpio-aspeed.c +++ b/drivers/gpio/gpio-aspeed.c @@ -5,6 +5,7 @@ * Joel Stanley <joel@jms.id.au> */ +#include <linux/bitfield.h> #include <linux/cleanup.h> #include <linux/clk.h> #include <linux/gpio/aspeed.h> @@ -30,10 +31,6 @@ */ #include <linux/gpio/consumer.h> -/* Non-constant mask variant of FIELD_GET() and FIELD_PREP() */ -#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) -#define field_prep(_mask, _val) (((_val) << (ffs(_mask) - 1)) & (_mask)) - #define GPIO_G7_IRQ_STS_BASE 0x100 #define GPIO_G7_IRQ_STS_OFFSET(x) (GPIO_G7_IRQ_STS_BASE + (x) * 0x4) #define GPIO_G7_CTRL_REG_BASE 0x180 diff --git a/drivers/iio/dac/ad3530r.c b/drivers/iio/dac/ad3530r.c index 6134613777b8..b97b46090d80 100644 --- a/drivers/iio/dac/ad3530r.c +++ b/drivers/iio/dac/ad3530r.c @@ -53,9 +53,6 @@ #define AD3530R_MAX_CHANNELS 8 #define AD3531R_MAX_CHANNELS 4 -/* Non-constant mask variant of FIELD_PREP() */ -#define field_prep(_mask, _val) (((_val) << (ffs(_mask) - 1)) & (_mask)) - enum ad3530r_mode { AD3530R_NORMAL_OP, AD3530R_POWERDOWN_1K, diff --git a/drivers/iio/temperature/mlx90614.c b/drivers/iio/temperature/mlx90614.c index 8a44a00bfd5e..1ad21b73e1b4 100644 --- a/drivers/iio/temperature/mlx90614.c +++ b/drivers/iio/temperature/mlx90614.c @@ -22,6 +22,7 @@ * the "wakeup" GPIO is not given, power management will be disabled. */ +#include <linux/bitfield.h> #include <linux/delay.h> #include <linux/err.h> #include <linux/gpio/consumer.h> @@ -68,10 +69,6 @@ #define MLX90614_CONST_SCALE 20 /* Scale in milliKelvin (0.02 * 1000) */ #define MLX90614_CONST_FIR 0x7 /* Fixed value for FIR part of low pass filter */ -/* Non-constant mask variant of FIELD_GET() and FIELD_PREP() */ -#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) -#define field_prep(_mask, _val) (((_val) << (ffs(_mask) - 1)) & (_mask)) - struct mlx_chip_info { /* EEPROM offsets with 16-bit data, MSB first */ /* emissivity correction coefficient */ diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index 78b1c44bd6b5..320733e7d8b4 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -107,6 +107,7 @@ /* Extended Feature 2 Bits */ +#define FEATURE_SEVSNPIO_SUP BIT_ULL(1) #define FEATURE_SNPAVICSUP GENMASK_ULL(7, 5) #define FEATURE_SNPAVICSUP_GAM(x) \ (FIELD_GET(FEATURE_SNPAVICSUP, x) == 0x1) diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index fdd6328bca89..4b2953418977 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -2261,6 +2261,9 @@ static void print_iommu_info(void) if (check_feature(FEATURE_SNP)) pr_cont(" SNP"); + if (check_feature2(FEATURE_SEVSNPIO_SUP)) + pr_cont(" SEV-TIO"); + pr_cont("\n"); } @@ -4028,4 +4031,10 @@ int amd_iommu_snp_disable(void) return 0; } EXPORT_SYMBOL_GPL(amd_iommu_snp_disable); + +bool amd_iommu_sev_tio_supported(void) +{ + return check_feature2(FEATURE_SEVSNPIO_SUP); +} +EXPORT_SYMBOL_GPL(amd_iommu_sev_tio_supported); #endif diff --git a/drivers/nvdimm/Kconfig b/drivers/nvdimm/Kconfig index fde3e17c836c..44ab929a1ad5 100644 --- a/drivers/nvdimm/Kconfig +++ b/drivers/nvdimm/Kconfig @@ -97,6 +97,25 @@ config OF_PMEM Select Y if unsure. +config RAMDAX + tristate "Support persistent memory interfaces on RAM carveouts" + depends on X86_PMEM_LEGACY || OF || COMPILE_TEST + default LIBNVDIMM + help + Allows creation of DAX devices on RAM carveouts. + + Memory ranges that are manually specified by the + 'memmap=nn[KMG]!ss[KMG]' kernel command line or defined by dummy + pmem-region device tree nodes would be managed by this driver as DIMM + devices with support for dynamic layout of namespaces. + The driver steals 128K in the end of the memmap range for the + namespace management. This allows supporting up to 509 namespaces + (see 'ndctl create-namespace --help'). + The driver should be force bound to e820_pmem or pmem-region platform + devices using 'driver_override' device attribute. + + Select N if unsure. + config NVDIMM_KEYS def_bool y depends on ENCRYPTED_KEYS diff --git a/drivers/nvdimm/Makefile b/drivers/nvdimm/Makefile index ba0296dca9db..8c268814936c 100644 --- a/drivers/nvdimm/Makefile +++ b/drivers/nvdimm/Makefile @@ -5,6 +5,7 @@ obj-$(CONFIG_ND_BTT) += nd_btt.o obj-$(CONFIG_X86_PMEM_LEGACY) += nd_e820.o obj-$(CONFIG_OF_PMEM) += of_pmem.o obj-$(CONFIG_VIRTIO_PMEM) += virtio_pmem.o nd_virtio.o +obj-$(CONFIG_RAMDAX) += ramdax.o nd_pmem-y := pmem.o diff --git a/drivers/nvdimm/ramdax.c b/drivers/nvdimm/ramdax.c new file mode 100644 index 000000000000..954cb7919807 --- /dev/null +++ b/drivers/nvdimm/ramdax.c @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025, Mike Rapoport, Microsoft + * + * Based on e820 pmem driver: + * Copyright (c) 2015, Christoph Hellwig. + * Copyright (c) 2015, Intel Corporation. + */ +#include <linux/platform_device.h> +#include <linux/memory_hotplug.h> +#include <linux/libnvdimm.h> +#include <linux/module.h> +#include <linux/numa.h> +#include <linux/slab.h> +#include <linux/io.h> +#include <linux/of.h> + +#include <uapi/linux/ndctl.h> + +#define LABEL_AREA_SIZE SZ_128K + +struct ramdax_dimm { + struct nvdimm *nvdimm; + void *label_area; +}; + +static void ramdax_remove(struct platform_device *pdev) +{ + struct nvdimm_bus *nvdimm_bus = platform_get_drvdata(pdev); + + nvdimm_bus_unregister(nvdimm_bus); +} + +static int ramdax_register_region(struct resource *res, + struct nvdimm *nvdimm, + struct nvdimm_bus *nvdimm_bus) +{ + struct nd_mapping_desc mapping; + struct nd_region_desc ndr_desc; + struct nd_interleave_set *nd_set; + int nid = phys_to_target_node(res->start); + + nd_set = kzalloc(sizeof(*nd_set), GFP_KERNEL); + if (!nd_set) + return -ENOMEM; + + nd_set->cookie1 = 0xcafebeefcafebeef; + nd_set->cookie2 = nd_set->cookie1; + nd_set->altcookie = nd_set->cookie1; + + memset(&mapping, 0, sizeof(mapping)); + mapping.nvdimm = nvdimm; + mapping.start = 0; + mapping.size = resource_size(res) - LABEL_AREA_SIZE; + + memset(&ndr_desc, 0, sizeof(ndr_desc)); + ndr_desc.res = res; + ndr_desc.numa_node = numa_map_to_online_node(nid); + ndr_desc.target_node = nid; + ndr_desc.num_mappings = 1; + ndr_desc.mapping = &mapping; + ndr_desc.nd_set = nd_set; + + if (!nvdimm_pmem_region_create(nvdimm_bus, &ndr_desc)) + goto err_free_nd_set; + + return 0; + +err_free_nd_set: + kfree(nd_set); + return -ENXIO; +} + +static int ramdax_register_dimm(struct resource *res, void *data) +{ + resource_size_t start = res->start; + resource_size_t size = resource_size(res); + unsigned long flags = 0, cmd_mask = 0; + struct nvdimm_bus *nvdimm_bus = data; + struct ramdax_dimm *dimm; + int err; + + dimm = kzalloc(sizeof(*dimm), GFP_KERNEL); + if (!dimm) + return -ENOMEM; + + dimm->label_area = memremap(start + size - LABEL_AREA_SIZE, + LABEL_AREA_SIZE, MEMREMAP_WB); + if (!dimm->label_area) { + err = -ENOMEM; + goto err_free_dimm; + } + + set_bit(NDD_LABELING, &flags); + set_bit(NDD_REGISTER_SYNC, &flags); + set_bit(ND_CMD_GET_CONFIG_SIZE, &cmd_mask); + set_bit(ND_CMD_GET_CONFIG_DATA, &cmd_mask); + set_bit(ND_CMD_SET_CONFIG_DATA, &cmd_mask); + dimm->nvdimm = nvdimm_create(nvdimm_bus, dimm, + /* dimm_attribute_groups */ NULL, + flags, cmd_mask, 0, NULL); + if (!dimm->nvdimm) { + err = -ENOMEM; + goto err_unmap_label; + } + + err = ramdax_register_region(res, dimm->nvdimm, nvdimm_bus); + if (err) + goto err_remove_nvdimm; + + return 0; + +err_remove_nvdimm: + nvdimm_delete(dimm->nvdimm); +err_unmap_label: + memunmap(dimm->label_area); +err_free_dimm: + kfree(dimm); + return err; +} + +static int ramdax_get_config_size(struct nvdimm *nvdimm, int buf_len, + struct nd_cmd_get_config_size *cmd) +{ + if (sizeof(*cmd) > buf_len) + return -EINVAL; + + *cmd = (struct nd_cmd_get_config_size){ + .status = 0, + .config_size = LABEL_AREA_SIZE, + .max_xfer = 8, + }; + + return 0; +} + +static int ramdax_get_config_data(struct nvdimm *nvdimm, int buf_len, + struct nd_cmd_get_config_data_hdr *cmd) +{ + struct ramdax_dimm *dimm = nvdimm_provider_data(nvdimm); + + if (sizeof(*cmd) > buf_len) + return -EINVAL; + if (struct_size(cmd, out_buf, cmd->in_length) > buf_len) + return -EINVAL; + if (size_add(cmd->in_offset, cmd->in_length) > LABEL_AREA_SIZE) + return -EINVAL; + + memcpy(cmd->out_buf, dimm->label_area + cmd->in_offset, cmd->in_length); + + return 0; +} + +static int ramdax_set_config_data(struct nvdimm *nvdimm, int buf_len, + struct nd_cmd_set_config_hdr *cmd) +{ + struct ramdax_dimm *dimm = nvdimm_provider_data(nvdimm); + + if (sizeof(*cmd) > buf_len) + return -EINVAL; + if (struct_size(cmd, in_buf, cmd->in_length) > buf_len) + return -EINVAL; + if (size_add(cmd->in_offset, cmd->in_length) > LABEL_AREA_SIZE) + return -EINVAL; + + memcpy(dimm->label_area + cmd->in_offset, cmd->in_buf, cmd->in_length); + + return 0; +} + +static int ramdax_nvdimm_ctl(struct nvdimm *nvdimm, unsigned int cmd, + void *buf, unsigned int buf_len) +{ + unsigned long cmd_mask = nvdimm_cmd_mask(nvdimm); + + if (!test_bit(cmd, &cmd_mask)) + return -ENOTTY; + + switch (cmd) { + case ND_CMD_GET_CONFIG_SIZE: + return ramdax_get_config_size(nvdimm, buf_len, buf); + case ND_CMD_GET_CONFIG_DATA: + return ramdax_get_config_data(nvdimm, buf_len, buf); + case ND_CMD_SET_CONFIG_DATA: + return ramdax_set_config_data(nvdimm, buf_len, buf); + default: + return -ENOTTY; + } +} + +static int ramdax_ctl(struct nvdimm_bus_descriptor *nd_desc, + struct nvdimm *nvdimm, unsigned int cmd, void *buf, + unsigned int buf_len, int *cmd_rc) +{ + /* + * No firmware response to translate, let the transport error + * code take precedence. + */ + *cmd_rc = 0; + + if (!nvdimm) + return -ENOTTY; + return ramdax_nvdimm_ctl(nvdimm, cmd, buf, buf_len); +} + +#ifdef CONFIG_OF +static const struct of_device_id ramdax_of_matches[] = { + { .compatible = "pmem-region", }, + { }, +}; +#endif + +static int ramdax_probe_of(struct platform_device *pdev, + struct nvdimm_bus *bus, struct device_node *np) +{ + int err; + + if (!of_match_node(ramdax_of_matches, np)) + return -ENODEV; + + for (int i = 0; i < pdev->num_resources; i++) { + err = ramdax_register_dimm(&pdev->resource[i], bus); + if (err) + goto err_unregister; + } + + return 0; + +err_unregister: + /* + * FIXME: should we unregister the dimms that were registered + * successfully + */ + return err; +} + +static int ramdax_probe(struct platform_device *pdev) +{ + static struct nvdimm_bus_descriptor nd_desc; + struct device *dev = &pdev->dev; + struct nvdimm_bus *nvdimm_bus; + struct device_node *np; + int rc = -ENXIO; + + nd_desc.provider_name = "ramdax"; + nd_desc.module = THIS_MODULE; + nd_desc.ndctl = ramdax_ctl; + nvdimm_bus = nvdimm_bus_register(dev, &nd_desc); + if (!nvdimm_bus) + goto err; + + np = dev_of_node(&pdev->dev); + if (np) + rc = ramdax_probe_of(pdev, nvdimm_bus, np); + else + rc = walk_iomem_res_desc(IORES_DESC_PERSISTENT_MEMORY_LEGACY, + IORESOURCE_MEM, 0, -1, nvdimm_bus, + ramdax_register_dimm); + if (rc) + goto err; + + platform_set_drvdata(pdev, nvdimm_bus); + + return 0; +err: + nvdimm_bus_unregister(nvdimm_bus); + return rc; +} + +static struct platform_driver ramdax_driver = { + .probe = ramdax_probe, + .remove = ramdax_remove, + .driver = { + .name = "ramdax", + }, +}; + +module_platform_driver(ramdax_driver); + +MODULE_DESCRIPTION("NVDIMM support for e820 type-12 memory and OF pmem-region"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Microsoft Corporation"); diff --git a/drivers/nvdimm/security.c b/drivers/nvdimm/security.c index 4adce8c38870..e41f6951ca0f 100644 --- a/drivers/nvdimm/security.c +++ b/drivers/nvdimm/security.c @@ -424,7 +424,7 @@ static int security_overwrite(struct nvdimm *nvdimm, unsigned int keyid) * query. */ get_device(dev); - queue_delayed_work(system_wq, &nvdimm->dwork, 0); + queue_delayed_work(system_percpu_wq, &nvdimm->dwork, 0); } return rc; @@ -457,7 +457,7 @@ static void __nvdimm_security_overwrite_query(struct nvdimm *nvdimm) /* setup delayed work again */ tmo += 10; - queue_delayed_work(system_wq, &nvdimm->dwork, tmo * HZ); + queue_delayed_work(system_percpu_wq, &nvdimm->dwork, tmo * HZ); nvdimm->sec.overwrite_tmo = min(15U * 60U, tmo); return; } diff --git a/drivers/nvme/common/auth.c b/drivers/nvme/common/auth.c index 1f51fbebd9fa..e07e7d4bf8b6 100644 --- a/drivers/nvme/common/auth.c +++ b/drivers/nvme/common/auth.c @@ -178,7 +178,7 @@ struct nvme_dhchap_key *nvme_auth_extract_key(unsigned char *secret, if (!key) return ERR_PTR(-ENOMEM); - key_len = base64_decode(secret, allocated_len, key->key); + key_len = base64_decode(secret, allocated_len, key->key, true, BASE64_STD); if (key_len < 0) { pr_debug("base64 key decoding error %d\n", key_len); @@ -663,7 +663,7 @@ int nvme_auth_generate_digest(u8 hmac_id, u8 *psk, size_t psk_len, if (ret) goto out_free_digest; - ret = base64_encode(digest, digest_len, enc); + ret = base64_encode(digest, digest_len, enc, true, BASE64_STD); if (ret < hmac_len) { ret = -ENOKEY; goto out_free_digest; diff --git a/drivers/parisc/ccio-dma.c b/drivers/parisc/ccio-dma.c index feef537257d0..4e7071714356 100644 --- a/drivers/parisc/ccio-dma.c +++ b/drivers/parisc/ccio-dma.c @@ -517,10 +517,10 @@ static u32 hint_lookup[] = { * ccio_io_pdir_entry - Initialize an I/O Pdir. * @pdir_ptr: A pointer into I/O Pdir. * @sid: The Space Identifier. - * @vba: The virtual address. + * @pba: The physical address. * @hints: The DMA Hint. * - * Given a virtual address (vba, arg2) and space id, (sid, arg1), + * Given a physical address (pba, arg2) and space id, (sid, arg1), * load the I/O PDIR entry pointed to by pdir_ptr (arg0). Each IO Pdir * entry consists of 8 bytes as shown below (MSB == bit 0): * @@ -543,7 +543,7 @@ static u32 hint_lookup[] = { * index are bits 12:19 of the value returned by LCI. */ static void -ccio_io_pdir_entry(__le64 *pdir_ptr, space_t sid, unsigned long vba, +ccio_io_pdir_entry(__le64 *pdir_ptr, space_t sid, phys_addr_t pba, unsigned long hints) { register unsigned long pa; @@ -557,7 +557,7 @@ ccio_io_pdir_entry(__le64 *pdir_ptr, space_t sid, unsigned long vba, ** "hints" parm includes the VALID bit! ** "dep" clobbers the physical address offset bits as well. */ - pa = lpa(vba); + pa = pba; asm volatile("depw %1,31,12,%0" : "+r" (pa) : "r" (hints)); ((u32 *)pdir_ptr)[1] = (u32) pa; @@ -582,7 +582,7 @@ ccio_io_pdir_entry(__le64 *pdir_ptr, space_t sid, unsigned long vba, ** Grab virtual index [0:11] ** Deposit virt_idx bits into I/O PDIR word */ - asm volatile ("lci %%r0(%1), %0" : "=r" (ci) : "r" (vba)); + asm volatile ("lci %%r0(%1), %0" : "=r" (ci) : "r" (phys_to_virt(pba))); asm volatile ("extru %1,19,12,%0" : "+r" (ci) : "r" (ci)); asm volatile ("depw %1,15,12,%0" : "+r" (pa) : "r" (ci)); @@ -704,14 +704,14 @@ ccio_dma_supported(struct device *dev, u64 mask) /** * ccio_map_single - Map an address range into the IOMMU. * @dev: The PCI device. - * @addr: The start address of the DMA region. + * @addr: The physical address of the DMA region. * @size: The length of the DMA region. * @direction: The direction of the DMA transaction (to/from device). * * This function implements the pci_map_single function. */ static dma_addr_t -ccio_map_single(struct device *dev, void *addr, size_t size, +ccio_map_single(struct device *dev, phys_addr_t addr, size_t size, enum dma_data_direction direction) { int idx; @@ -730,7 +730,7 @@ ccio_map_single(struct device *dev, void *addr, size_t size, BUG_ON(size <= 0); /* save offset bits */ - offset = ((unsigned long) addr) & ~IOVP_MASK; + offset = offset_in_page(addr); /* round up to nearest IOVP_SIZE */ size = ALIGN(size + offset, IOVP_SIZE); @@ -746,15 +746,15 @@ ccio_map_single(struct device *dev, void *addr, size_t size, pdir_start = &(ioc->pdir_base[idx]); - DBG_RUN("%s() %px -> %#lx size: %zu\n", - __func__, addr, (long)(iovp | offset), size); + DBG_RUN("%s() %pa -> %#lx size: %zu\n", + __func__, &addr, (long)(iovp | offset), size); /* If not cacheline aligned, force SAFE_DMA on the whole mess */ - if((size % L1_CACHE_BYTES) || ((unsigned long)addr % L1_CACHE_BYTES)) + if ((size % L1_CACHE_BYTES) || (addr % L1_CACHE_BYTES)) hint |= HINT_SAFE_DMA; while(size > 0) { - ccio_io_pdir_entry(pdir_start, KERNEL_SPACE, (unsigned long)addr, hint); + ccio_io_pdir_entry(pdir_start, KERNEL_SPACE, addr, hint); DBG_RUN(" pdir %p %08x%08x\n", pdir_start, @@ -773,17 +773,18 @@ ccio_map_single(struct device *dev, void *addr, size_t size, static dma_addr_t -ccio_map_page(struct device *dev, struct page *page, unsigned long offset, - size_t size, enum dma_data_direction direction, - unsigned long attrs) +ccio_map_phys(struct device *dev, phys_addr_t phys, size_t size, + enum dma_data_direction direction, unsigned long attrs) { - return ccio_map_single(dev, page_address(page) + offset, size, - direction); + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; + + return ccio_map_single(dev, phys, size, direction); } /** - * ccio_unmap_page - Unmap an address range from the IOMMU. + * ccio_unmap_phys - Unmap an address range from the IOMMU. * @dev: The PCI device. * @iova: The start address of the DMA region. * @size: The length of the DMA region. @@ -791,7 +792,7 @@ ccio_map_page(struct device *dev, struct page *page, unsigned long offset, * @attrs: attributes */ static void -ccio_unmap_page(struct device *dev, dma_addr_t iova, size_t size, +ccio_unmap_phys(struct device *dev, dma_addr_t iova, size_t size, enum dma_data_direction direction, unsigned long attrs) { struct ioc *ioc; @@ -853,7 +854,8 @@ ccio_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, gfp_t flag, if (ret) { memset(ret, 0, size); - *dma_handle = ccio_map_single(dev, ret, size, DMA_BIDIRECTIONAL); + *dma_handle = ccio_map_single(dev, virt_to_phys(ret), size, + DMA_BIDIRECTIONAL); } return ret; @@ -873,7 +875,7 @@ static void ccio_free(struct device *dev, size_t size, void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) { - ccio_unmap_page(dev, dma_handle, size, 0, 0); + ccio_unmap_phys(dev, dma_handle, size, 0, 0); free_pages((unsigned long)cpu_addr, get_order(size)); } @@ -920,7 +922,7 @@ ccio_map_sg(struct device *dev, struct scatterlist *sglist, int nents, /* Fast path single entry scatterlists. */ if (nents == 1) { sg_dma_address(sglist) = ccio_map_single(dev, - sg_virt(sglist), sglist->length, + sg_phys(sglist), sglist->length, direction); sg_dma_len(sglist) = sglist->length; return 1; @@ -1004,7 +1006,7 @@ ccio_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, #ifdef CCIO_COLLECT_STATS ioc->usg_pages += sg_dma_len(sglist) >> PAGE_SHIFT; #endif - ccio_unmap_page(dev, sg_dma_address(sglist), + ccio_unmap_phys(dev, sg_dma_address(sglist), sg_dma_len(sglist), direction, 0); ++sglist; nents--; @@ -1017,8 +1019,8 @@ static const struct dma_map_ops ccio_ops = { .dma_supported = ccio_dma_supported, .alloc = ccio_alloc, .free = ccio_free, - .map_page = ccio_map_page, - .unmap_page = ccio_unmap_page, + .map_phys = ccio_map_phys, + .unmap_phys = ccio_unmap_phys, .map_sg = ccio_map_sg, .unmap_sg = ccio_unmap_sg, .get_sgtable = dma_common_get_sgtable, @@ -1072,7 +1074,7 @@ static int ccio_proc_info(struct seq_file *m, void *p) ioc->msingle_calls, ioc->msingle_pages, (int)((ioc->msingle_pages * 1000)/ioc->msingle_calls)); - /* KLUGE - unmap_sg calls unmap_page for each mapped page */ + /* KLUGE - unmap_sg calls unmap_phys for each mapped page */ min = ioc->usingle_calls - ioc->usg_calls; max = ioc->usingle_pages - ioc->usg_pages; seq_printf(m, "pci_unmap_single: %8ld calls %8ld pages (avg %d/1000)\n", diff --git a/drivers/parisc/gsc.c b/drivers/parisc/gsc.c index a0daaa548bc3..8ba778170447 100644 --- a/drivers/parisc/gsc.c +++ b/drivers/parisc/gsc.c @@ -154,7 +154,9 @@ static int gsc_set_affinity_irq(struct irq_data *d, const struct cpumask *dest, gsc_dev->eim = ((u32) gsc_dev->gsc_irq.txn_addr) | gsc_dev->gsc_irq.txn_data; /* switch IRQ's for devices below LASI/WAX to other CPU */ - gsc_writel(gsc_dev->eim, gsc_dev->hpa + OFFSET_IAR); + /* ASP chip (svers 0x70) does not support reprogramming */ + if (gsc_dev->gsc->id.sversion != 0x70) + gsc_writel(gsc_dev->eim, gsc_dev->hpa + OFFSET_IAR); irq_data_update_effective_affinity(d, &tmask); diff --git a/drivers/parisc/iommu-helpers.h b/drivers/parisc/iommu-helpers.h index c43f1a212a5c..0691884f5095 100644 --- a/drivers/parisc/iommu-helpers.h +++ b/drivers/parisc/iommu-helpers.h @@ -14,7 +14,7 @@ static inline unsigned int iommu_fill_pdir(struct ioc *ioc, struct scatterlist *startsg, int nents, unsigned long hint, - void (*iommu_io_pdir_entry)(__le64 *, space_t, unsigned long, + void (*iommu_io_pdir_entry)(__le64 *, space_t, phys_addr_t, unsigned long)) { struct scatterlist *dma_sg = startsg; /* pointer to current DMA */ @@ -28,7 +28,7 @@ iommu_fill_pdir(struct ioc *ioc, struct scatterlist *startsg, int nents, dma_sg--; while (nents-- > 0) { - unsigned long vaddr; + phys_addr_t paddr; long size; DBG_RUN_SG(" %d : %08lx %p/%05x\n", nents, @@ -67,7 +67,7 @@ iommu_fill_pdir(struct ioc *ioc, struct scatterlist *startsg, int nents, BUG_ON(pdirp == NULL); - vaddr = (unsigned long)sg_virt(startsg); + paddr = sg_phys(startsg); sg_dma_len(dma_sg) += startsg->length; size = startsg->length + dma_offset; dma_offset = 0; @@ -76,8 +76,8 @@ iommu_fill_pdir(struct ioc *ioc, struct scatterlist *startsg, int nents, #endif do { iommu_io_pdir_entry(pdirp, KERNEL_SPACE, - vaddr, hint); - vaddr += IOVP_SIZE; + paddr, hint); + paddr += IOVP_SIZE; size -= IOVP_SIZE; pdirp++; } while(unlikely(size > 0)); diff --git a/drivers/parisc/sba_iommu.c b/drivers/parisc/sba_iommu.c index fc3863c09f83..a6eb6bffa5ea 100644 --- a/drivers/parisc/sba_iommu.c +++ b/drivers/parisc/sba_iommu.c @@ -532,7 +532,7 @@ typedef unsigned long space_t; * sba_io_pdir_entry - fill in one IO PDIR entry * @pdir_ptr: pointer to IO PDIR entry * @sid: process Space ID - currently only support KERNEL_SPACE - * @vba: Virtual CPU address of buffer to map + * @pba: Physical address of buffer to map * @hint: DMA hint set to use for this mapping * * SBA Mapping Routine @@ -569,20 +569,17 @@ typedef unsigned long space_t; */ static void -sba_io_pdir_entry(__le64 *pdir_ptr, space_t sid, unsigned long vba, +sba_io_pdir_entry(__le64 *pdir_ptr, space_t sid, phys_addr_t pba, unsigned long hint) { - u64 pa; /* physical address */ register unsigned ci; /* coherent index */ - pa = lpa(vba); - pa &= IOVP_MASK; + asm("lci 0(%1), %0" : "=r" (ci) : "r" (phys_to_virt(pba))); + pba &= IOVP_MASK; + pba |= (ci >> PAGE_SHIFT) & 0xff; /* move CI (8 bits) into lowest byte */ - asm("lci 0(%1), %0" : "=r" (ci) : "r" (vba)); - pa |= (ci >> PAGE_SHIFT) & 0xff; /* move CI (8 bits) into lowest byte */ - - pa |= SBA_PDIR_VALID_BIT; /* set "valid" bit */ - *pdir_ptr = cpu_to_le64(pa); /* swap and store into I/O Pdir */ + pba |= SBA_PDIR_VALID_BIT; /* set "valid" bit */ + *pdir_ptr = cpu_to_le64(pba); /* swap and store into I/O Pdir */ /* * If the PDC_MODEL capabilities has Non-coherent IO-PDIR bit set @@ -707,7 +704,7 @@ static int sba_dma_supported( struct device *dev, u64 mask) * See Documentation/core-api/dma-api-howto.rst */ static dma_addr_t -sba_map_single(struct device *dev, void *addr, size_t size, +sba_map_single(struct device *dev, phys_addr_t addr, size_t size, enum dma_data_direction direction) { struct ioc *ioc; @@ -722,7 +719,7 @@ sba_map_single(struct device *dev, void *addr, size_t size, return DMA_MAPPING_ERROR; /* save offset bits */ - offset = ((dma_addr_t) (long) addr) & ~IOVP_MASK; + offset = offset_in_page(addr); /* round up to nearest IOVP_SIZE */ size = (size + offset + ~IOVP_MASK) & IOVP_MASK; @@ -739,13 +736,13 @@ sba_map_single(struct device *dev, void *addr, size_t size, pide = sba_alloc_range(ioc, dev, size); iovp = (dma_addr_t) pide << IOVP_SHIFT; - DBG_RUN("%s() 0x%p -> 0x%lx\n", - __func__, addr, (long) iovp | offset); + DBG_RUN("%s() 0x%pa -> 0x%lx\n", + __func__, &addr, (long) iovp | offset); pdir_start = &(ioc->pdir_base[pide]); while (size > 0) { - sba_io_pdir_entry(pdir_start, KERNEL_SPACE, (unsigned long) addr, 0); + sba_io_pdir_entry(pdir_start, KERNEL_SPACE, addr, 0); DBG_RUN(" pdir 0x%p %02x%02x%02x%02x%02x%02x%02x%02x\n", pdir_start, @@ -778,17 +775,18 @@ sba_map_single(struct device *dev, void *addr, size_t size, static dma_addr_t -sba_map_page(struct device *dev, struct page *page, unsigned long offset, - size_t size, enum dma_data_direction direction, - unsigned long attrs) +sba_map_phys(struct device *dev, phys_addr_t phys, size_t size, + enum dma_data_direction direction, unsigned long attrs) { - return sba_map_single(dev, page_address(page) + offset, size, - direction); + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; + + return sba_map_single(dev, phys, size, direction); } /** - * sba_unmap_page - unmap one IOVA and free resources + * sba_unmap_phys - unmap one IOVA and free resources * @dev: instance of PCI owned by the driver that's asking. * @iova: IOVA of driver buffer previously mapped. * @size: number of bytes mapped in driver buffer. @@ -798,7 +796,7 @@ sba_map_page(struct device *dev, struct page *page, unsigned long offset, * See Documentation/core-api/dma-api-howto.rst */ static void -sba_unmap_page(struct device *dev, dma_addr_t iova, size_t size, +sba_unmap_phys(struct device *dev, dma_addr_t iova, size_t size, enum dma_data_direction direction, unsigned long attrs) { struct ioc *ioc; @@ -893,7 +891,7 @@ static void *sba_alloc(struct device *hwdev, size_t size, dma_addr_t *dma_handle if (ret) { memset(ret, 0, size); - *dma_handle = sba_map_single(hwdev, ret, size, 0); + *dma_handle = sba_map_single(hwdev, virt_to_phys(ret), size, 0); } return ret; @@ -914,7 +912,7 @@ static void sba_free(struct device *hwdev, size_t size, void *vaddr, dma_addr_t dma_handle, unsigned long attrs) { - sba_unmap_page(hwdev, dma_handle, size, 0, 0); + sba_unmap_phys(hwdev, dma_handle, size, 0, 0); free_pages((unsigned long) vaddr, get_order(size)); } @@ -962,7 +960,7 @@ sba_map_sg(struct device *dev, struct scatterlist *sglist, int nents, /* Fast path single entry scatterlists. */ if (nents == 1) { - sg_dma_address(sglist) = sba_map_single(dev, sg_virt(sglist), + sg_dma_address(sglist) = sba_map_single(dev, sg_phys(sglist), sglist->length, direction); sg_dma_len(sglist) = sglist->length; return 1; @@ -1061,7 +1059,7 @@ sba_unmap_sg(struct device *dev, struct scatterlist *sglist, int nents, while (nents && sg_dma_len(sglist)) { - sba_unmap_page(dev, sg_dma_address(sglist), sg_dma_len(sglist), + sba_unmap_phys(dev, sg_dma_address(sglist), sg_dma_len(sglist), direction, 0); #ifdef SBA_COLLECT_STATS ioc->usg_pages += ((sg_dma_address(sglist) & ~IOVP_MASK) + sg_dma_len(sglist) + IOVP_SIZE - 1) >> PAGE_SHIFT; @@ -1085,8 +1083,8 @@ static const struct dma_map_ops sba_ops = { .dma_supported = sba_dma_supported, .alloc = sba_alloc, .free = sba_free, - .map_page = sba_map_page, - .unmap_page = sba_unmap_page, + .map_phys = sba_map_phys, + .unmap_phys = sba_unmap_phys, .map_sg = sba_map_sg, .unmap_sg = sba_unmap_sg, .get_sgtable = dma_common_get_sgtable, diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig index f94f5d384362..00b0210e1f1d 100644 --- a/drivers/pci/Kconfig +++ b/drivers/pci/Kconfig @@ -122,6 +122,24 @@ config XEN_PCIDEV_FRONTEND config PCI_ATS bool +config PCI_IDE + bool + +config PCI_TSM + bool "PCI TSM: Device security protocol support" + select PCI_IDE + select PCI_DOE + select TSM + help + The TEE (Trusted Execution Environment) Device Interface + Security Protocol (TDISP) defines a "TSM" as a platform agent + that manages device authentication, link encryption, link + integrity protection, and assignment of PCI device functions + (virtual or physical) to confidential computing VMs that can + access (DMA) guest private memory. + + Enable a platform TSM driver to use this capability. + config PCI_DOE bool "Enable PCI Data Object Exchange (DOE) support" help diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile index f3c81c892786..e10cfe5a280b 100644 --- a/drivers/pci/Makefile +++ b/drivers/pci/Makefile @@ -34,6 +34,8 @@ obj-$(CONFIG_PCI_P2PDMA) += p2pdma.o obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o obj-$(CONFIG_VGA_ARB) += vgaarb.o obj-$(CONFIG_PCI_DOE) += doe.o +obj-$(CONFIG_PCI_IDE) += ide.o +obj-$(CONFIG_PCI_TSM) += tsm.o obj-$(CONFIG_PCI_DYNAMIC_OF_NODES) += of_property.o obj-$(CONFIG_PCI_NPEM) += npem.o obj-$(CONFIG_PCIE_TPH) += tph.o diff --git a/drivers/pci/bus.c b/drivers/pci/bus.c index 9daf13ed3714..4383a36fd6ca 100644 --- a/drivers/pci/bus.c +++ b/drivers/pci/bus.c @@ -8,6 +8,7 @@ */ #include <linux/module.h> #include <linux/kernel.h> +#include <linux/cleanup.h> #include <linux/pci.h> #include <linux/errno.h> #include <linux/ioport.h> @@ -435,6 +436,27 @@ static int __pci_walk_bus(struct pci_bus *top, int (*cb)(struct pci_dev *, void return ret; } +static int __pci_walk_bus_reverse(struct pci_bus *top, + int (*cb)(struct pci_dev *, void *), + void *userdata) +{ + struct pci_dev *dev; + int ret = 0; + + list_for_each_entry_reverse(dev, &top->devices, bus_list) { + if (dev->subordinate) { + ret = __pci_walk_bus_reverse(dev->subordinate, cb, + userdata); + if (ret) + break; + } + ret = cb(dev, userdata); + if (ret) + break; + } + return ret; +} + /** * pci_walk_bus - walk devices on/under bus, calling callback. * @top: bus whose devices should be walked @@ -456,6 +478,23 @@ void pci_walk_bus(struct pci_bus *top, int (*cb)(struct pci_dev *, void *), void } EXPORT_SYMBOL_GPL(pci_walk_bus); +/** + * pci_walk_bus_reverse - walk devices on/under bus, calling callback. + * @top: bus whose devices should be walked + * @cb: callback to be called for each device found + * @userdata: arbitrary pointer to be passed to callback + * + * Same semantics as pci_walk_bus(), but walks the bus in reverse order. + */ +void pci_walk_bus_reverse(struct pci_bus *top, + int (*cb)(struct pci_dev *, void *), void *userdata) +{ + down_read(&pci_bus_sem); + __pci_walk_bus_reverse(top, cb, userdata); + up_read(&pci_bus_sem); +} +EXPORT_SYMBOL_GPL(pci_walk_bus_reverse); + void pci_walk_bus_locked(struct pci_bus *top, int (*cb)(struct pci_dev *, void *), void *userdata) { lockdep_assert_held(&pci_bus_sem); diff --git a/drivers/pci/doe.c b/drivers/pci/doe.c index aae9a8a00406..62be9c8dbc52 100644 --- a/drivers/pci/doe.c +++ b/drivers/pci/doe.c @@ -24,8 +24,6 @@ #include "pci.h" -#define PCI_DOE_FEATURE_DISCOVERY 0 - /* Timeout of 1 second from 6.30.2 Operation, PCI Spec r6.0 */ #define PCI_DOE_TIMEOUT HZ #define PCI_DOE_POLL_INTERVAL (PCI_DOE_TIMEOUT / 128) diff --git a/drivers/pci/ide.c b/drivers/pci/ide.c new file mode 100644 index 000000000000..f0ef474e1a0d --- /dev/null +++ b/drivers/pci/ide.c @@ -0,0 +1,815 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright(c) 2024-2025 Intel Corporation. All rights reserved. */ + +/* PCIe r7.0 section 6.33 Integrity & Data Encryption (IDE) */ + +#define dev_fmt(fmt) "PCI/IDE: " fmt +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/pci.h> +#include <linux/pci-ide.h> +#include <linux/pci_regs.h> +#include <linux/slab.h> +#include <linux/sysfs.h> +#include <linux/tsm.h> + +#include "pci.h" + +static int __sel_ide_offset(u16 ide_cap, u8 nr_link_ide, u8 stream_index, + u8 nr_ide_mem) +{ + u32 offset = ide_cap + PCI_IDE_LINK_STREAM_0 + + nr_link_ide * PCI_IDE_LINK_BLOCK_SIZE; + + /* + * Assume a constant number of address association resources per stream + * index + */ + return offset + stream_index * PCI_IDE_SEL_BLOCK_SIZE(nr_ide_mem); +} + +static int sel_ide_offset(struct pci_dev *pdev, + struct pci_ide_partner *settings) +{ + return __sel_ide_offset(pdev->ide_cap, pdev->nr_link_ide, + settings->stream_index, pdev->nr_ide_mem); +} + +static bool reserve_stream_index(struct pci_dev *pdev, u8 idx) +{ + int ret; + + ret = ida_alloc_range(&pdev->ide_stream_ida, idx, idx, GFP_KERNEL); + return ret >= 0; +} + +static bool reserve_stream_id(struct pci_host_bridge *hb, u8 id) +{ + int ret; + + ret = ida_alloc_range(&hb->ide_stream_ids_ida, id, id, GFP_KERNEL); + return ret >= 0; +} + +static bool claim_stream(struct pci_host_bridge *hb, u8 stream_id, + struct pci_dev *pdev, u8 stream_idx) +{ + dev_info(&hb->dev, "Stream ID %d active at init\n", stream_id); + if (!reserve_stream_id(hb, stream_id)) { + dev_info(&hb->dev, "Failed to claim %s Stream ID %d\n", + stream_id == PCI_IDE_RESERVED_STREAM_ID ? "reserved" : + "active", + stream_id); + return false; + } + + /* No stream index to reserve in the Link IDE case */ + if (!pdev) + return true; + + if (!reserve_stream_index(pdev, stream_idx)) { + pci_info(pdev, "Failed to claim active Selective Stream %d\n", + stream_idx); + return false; + } + + return true; +} + +void pci_ide_init(struct pci_dev *pdev) +{ + struct pci_host_bridge *hb = pci_find_host_bridge(pdev->bus); + u16 nr_link_ide, nr_ide_mem, nr_streams; + u16 ide_cap; + u32 val; + + /* + * Unconditionally init so that ida idle state is consistent with + * pdev->ide_cap. + */ + ida_init(&pdev->ide_stream_ida); + + if (!pci_is_pcie(pdev)) + return; + + ide_cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_IDE); + if (!ide_cap) + return; + + pci_read_config_dword(pdev, ide_cap + PCI_IDE_CAP, &val); + if ((val & PCI_IDE_CAP_SELECTIVE) == 0) + return; + + /* + * Require endpoint IDE capability to be paired with IDE Root Port IDE + * capability. + */ + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_ENDPOINT) { + struct pci_dev *rp = pcie_find_root_port(pdev); + + if (!rp->ide_cap) + return; + } + + pdev->ide_cfg = FIELD_GET(PCI_IDE_CAP_SEL_CFG, val); + pdev->ide_tee_limit = FIELD_GET(PCI_IDE_CAP_TEE_LIMITED, val); + + if (val & PCI_IDE_CAP_LINK) + nr_link_ide = 1 + FIELD_GET(PCI_IDE_CAP_LINK_TC_NUM, val); + else + nr_link_ide = 0; + + nr_ide_mem = 0; + nr_streams = 1 + FIELD_GET(PCI_IDE_CAP_SEL_NUM, val); + for (u16 i = 0; i < nr_streams; i++) { + int pos = __sel_ide_offset(ide_cap, nr_link_ide, i, nr_ide_mem); + int nr_assoc; + u32 val; + u8 id; + + pci_read_config_dword(pdev, pos + PCI_IDE_SEL_CAP, &val); + + /* + * Let's not entertain streams that do not have a constant + * number of address association blocks + */ + nr_assoc = FIELD_GET(PCI_IDE_SEL_CAP_ASSOC_NUM, val); + if (i && (nr_assoc != nr_ide_mem)) { + pci_info(pdev, "Unsupported Selective Stream %d capability, SKIP the rest\n", i); + nr_streams = i; + break; + } + + nr_ide_mem = nr_assoc; + + /* + * Claim Stream IDs and Selective Stream blocks that are already + * active on the device + */ + pci_read_config_dword(pdev, pos + PCI_IDE_SEL_CTL, &val); + id = FIELD_GET(PCI_IDE_SEL_CTL_ID, val); + if ((val & PCI_IDE_SEL_CTL_EN) && + !claim_stream(hb, id, pdev, i)) + return; + } + + /* Reserve link stream-ids that are already active on the device */ + for (u16 i = 0; i < nr_link_ide; ++i) { + int pos = ide_cap + PCI_IDE_LINK_STREAM_0 + i * PCI_IDE_LINK_BLOCK_SIZE; + u8 id; + + pci_read_config_dword(pdev, pos + PCI_IDE_LINK_CTL_0, &val); + id = FIELD_GET(PCI_IDE_LINK_CTL_ID, val); + if ((val & PCI_IDE_LINK_CTL_EN) && + !claim_stream(hb, id, NULL, -1)) + return; + } + + for (u16 i = 0; i < nr_streams; i++) { + int pos = __sel_ide_offset(ide_cap, nr_link_ide, i, nr_ide_mem); + + pci_read_config_dword(pdev, pos + PCI_IDE_SEL_CAP, &val); + if (val & PCI_IDE_SEL_CTL_EN) + continue; + val &= ~PCI_IDE_SEL_CTL_ID; + val |= FIELD_PREP(PCI_IDE_SEL_CTL_ID, PCI_IDE_RESERVED_STREAM_ID); + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_CTL, val); + } + + for (u16 i = 0; i < nr_link_ide; ++i) { + int pos = ide_cap + PCI_IDE_LINK_STREAM_0 + + i * PCI_IDE_LINK_BLOCK_SIZE; + + pci_read_config_dword(pdev, pos, &val); + if (val & PCI_IDE_LINK_CTL_EN) + continue; + val &= ~PCI_IDE_LINK_CTL_ID; + val |= FIELD_PREP(PCI_IDE_LINK_CTL_ID, PCI_IDE_RESERVED_STREAM_ID); + pci_write_config_dword(pdev, pos, val); + } + + pdev->ide_cap = ide_cap; + pdev->nr_link_ide = nr_link_ide; + pdev->nr_sel_ide = nr_streams; + pdev->nr_ide_mem = nr_ide_mem; +} + +struct stream_index { + struct ida *ida; + u8 stream_index; +}; + +static void free_stream_index(struct stream_index *stream) +{ + ida_free(stream->ida, stream->stream_index); +} + +DEFINE_FREE(free_stream, struct stream_index *, if (_T) free_stream_index(_T)) +static struct stream_index *alloc_stream_index(struct ida *ida, u16 max, + struct stream_index *stream) +{ + int id; + + if (!max) + return NULL; + + id = ida_alloc_max(ida, max - 1, GFP_KERNEL); + if (id < 0) + return NULL; + + *stream = (struct stream_index) { + .ida = ida, + .stream_index = id, + }; + return stream; +} + +/** + * pci_ide_stream_alloc() - Reserve stream indices and probe for settings + * @pdev: IDE capable PCIe Endpoint Physical Function + * + * Retrieve the Requester ID range of @pdev for programming its Root + * Port IDE RID Association registers, and conversely retrieve the + * Requester ID of the Root Port for programming @pdev's IDE RID + * Association registers. + * + * Allocate a Selective IDE Stream Register Block instance per port. + * + * Allocate a platform stream resource from the associated host bridge. + * Retrieve stream association parameters for Requester ID range and + * address range restrictions for the stream. + */ +struct pci_ide *pci_ide_stream_alloc(struct pci_dev *pdev) +{ + /* EP, RP, + HB Stream allocation */ + struct stream_index __stream[PCI_IDE_HB + 1]; + struct pci_bus_region pref_assoc = { 0, -1 }; + struct pci_bus_region mem_assoc = { 0, -1 }; + struct resource *mem, *pref; + struct pci_host_bridge *hb; + struct pci_dev *rp, *br; + int num_vf, rid_end; + + if (!pci_is_pcie(pdev)) + return NULL; + + if (pci_pcie_type(pdev) != PCI_EXP_TYPE_ENDPOINT) + return NULL; + + if (!pdev->ide_cap) + return NULL; + + struct pci_ide *ide __free(kfree) = kzalloc(sizeof(*ide), GFP_KERNEL); + if (!ide) + return NULL; + + hb = pci_find_host_bridge(pdev->bus); + struct stream_index *hb_stream __free(free_stream) = alloc_stream_index( + &hb->ide_stream_ida, hb->nr_ide_streams, &__stream[PCI_IDE_HB]); + if (!hb_stream) + return NULL; + + rp = pcie_find_root_port(pdev); + struct stream_index *rp_stream __free(free_stream) = alloc_stream_index( + &rp->ide_stream_ida, rp->nr_sel_ide, &__stream[PCI_IDE_RP]); + if (!rp_stream) + return NULL; + + struct stream_index *ep_stream __free(free_stream) = alloc_stream_index( + &pdev->ide_stream_ida, pdev->nr_sel_ide, &__stream[PCI_IDE_EP]); + if (!ep_stream) + return NULL; + + /* for SR-IOV case, cover all VFs */ + num_vf = pci_num_vf(pdev); + if (num_vf) + rid_end = PCI_DEVID(pci_iov_virtfn_bus(pdev, num_vf), + pci_iov_virtfn_devfn(pdev, num_vf)); + else + rid_end = pci_dev_id(pdev); + + br = pci_upstream_bridge(pdev); + if (!br) + return NULL; + + /* + * Check if the device consumes memory and/or prefetch-memory. Setup + * downstream address association ranges for each. + */ + mem = pci_resource_n(br, PCI_BRIDGE_MEM_WINDOW); + pref = pci_resource_n(br, PCI_BRIDGE_PREF_MEM_WINDOW); + if (resource_assigned(mem)) + pcibios_resource_to_bus(br->bus, &mem_assoc, mem); + if (resource_assigned(pref)) + pcibios_resource_to_bus(br->bus, &pref_assoc, pref); + + *ide = (struct pci_ide) { + .pdev = pdev, + .partner = { + [PCI_IDE_EP] = { + .rid_start = pci_dev_id(rp), + .rid_end = pci_dev_id(rp), + .stream_index = no_free_ptr(ep_stream)->stream_index, + /* Disable upstream address association */ + .mem_assoc = { 0, -1 }, + .pref_assoc = { 0, -1 }, + }, + [PCI_IDE_RP] = { + .rid_start = pci_dev_id(pdev), + .rid_end = rid_end, + .stream_index = no_free_ptr(rp_stream)->stream_index, + .mem_assoc = mem_assoc, + .pref_assoc = pref_assoc, + }, + }, + .host_bridge_stream = no_free_ptr(hb_stream)->stream_index, + .stream_id = -1, + }; + + return_ptr(ide); +} +EXPORT_SYMBOL_GPL(pci_ide_stream_alloc); + +/** + * pci_ide_stream_free() - unwind pci_ide_stream_alloc() + * @ide: idle IDE settings descriptor + * + * Free all of the stream index (register block) allocations acquired by + * pci_ide_stream_alloc(). The stream represented by @ide is assumed to + * be unregistered and not instantiated in any device. + */ +void pci_ide_stream_free(struct pci_ide *ide) +{ + struct pci_dev *pdev = ide->pdev; + struct pci_dev *rp = pcie_find_root_port(pdev); + struct pci_host_bridge *hb = pci_find_host_bridge(pdev->bus); + + ida_free(&pdev->ide_stream_ida, ide->partner[PCI_IDE_EP].stream_index); + ida_free(&rp->ide_stream_ida, ide->partner[PCI_IDE_RP].stream_index); + ida_free(&hb->ide_stream_ida, ide->host_bridge_stream); + kfree(ide); +} +EXPORT_SYMBOL_GPL(pci_ide_stream_free); + +/** + * pci_ide_stream_release() - unwind and release an @ide context + * @ide: partially or fully registered IDE settings descriptor + * + * In support of automatic cleanup of IDE setup routines perform IDE + * teardown in expected reverse order of setup and with respect to which + * aspects of IDE setup have successfully completed. + * + * Be careful that setup order mirrors this shutdown order. Otherwise, + * open code releasing the IDE context. + */ +void pci_ide_stream_release(struct pci_ide *ide) +{ + struct pci_dev *pdev = ide->pdev; + struct pci_dev *rp = pcie_find_root_port(pdev); + + if (ide->partner[PCI_IDE_RP].enable) + pci_ide_stream_disable(rp, ide); + + if (ide->partner[PCI_IDE_EP].enable) + pci_ide_stream_disable(pdev, ide); + + if (ide->tsm_dev) + tsm_ide_stream_unregister(ide); + + if (ide->partner[PCI_IDE_RP].setup) + pci_ide_stream_teardown(rp, ide); + + if (ide->partner[PCI_IDE_EP].setup) + pci_ide_stream_teardown(pdev, ide); + + if (ide->name) + pci_ide_stream_unregister(ide); + + pci_ide_stream_free(ide); +} +EXPORT_SYMBOL_GPL(pci_ide_stream_release); + +struct pci_ide_stream_id { + struct pci_host_bridge *hb; + u8 stream_id; +}; + +static struct pci_ide_stream_id * +request_stream_id(struct pci_host_bridge *hb, u8 stream_id, + struct pci_ide_stream_id *sid) +{ + if (!reserve_stream_id(hb, stream_id)) + return NULL; + + *sid = (struct pci_ide_stream_id) { + .hb = hb, + .stream_id = stream_id, + }; + + return sid; +} +DEFINE_FREE(free_stream_id, struct pci_ide_stream_id *, + if (_T) ida_free(&_T->hb->ide_stream_ids_ida, _T->stream_id)) + +/** + * pci_ide_stream_register() - Prepare to activate an IDE Stream + * @ide: IDE settings descriptor + * + * After a Stream ID has been acquired for @ide, record the presence of + * the stream in sysfs. The expectation is that @ide is immutable while + * registered. + */ +int pci_ide_stream_register(struct pci_ide *ide) +{ + struct pci_dev *pdev = ide->pdev; + struct pci_host_bridge *hb = pci_find_host_bridge(pdev->bus); + struct pci_ide_stream_id __sid; + u8 ep_stream, rp_stream; + int rc; + + if (ide->stream_id < 0 || ide->stream_id > U8_MAX) { + pci_err(pdev, "Setup fail: Invalid Stream ID: %d\n", ide->stream_id); + return -ENXIO; + } + + struct pci_ide_stream_id *sid __free(free_stream_id) = + request_stream_id(hb, ide->stream_id, &__sid); + if (!sid) { + pci_err(pdev, "Setup fail: Stream ID %d in use\n", ide->stream_id); + return -EBUSY; + } + + ep_stream = ide->partner[PCI_IDE_EP].stream_index; + rp_stream = ide->partner[PCI_IDE_RP].stream_index; + const char *name __free(kfree) = kasprintf(GFP_KERNEL, "stream%d.%d.%d", + ide->host_bridge_stream, + rp_stream, ep_stream); + if (!name) + return -ENOMEM; + + rc = sysfs_create_link(&hb->dev.kobj, &pdev->dev.kobj, name); + if (rc) + return rc; + + ide->name = no_free_ptr(name); + + /* Stream ID reservation recorded in @ide is now successfully registered */ + retain_and_null_ptr(sid); + + return 0; +} +EXPORT_SYMBOL_GPL(pci_ide_stream_register); + +/** + * pci_ide_stream_unregister() - unwind pci_ide_stream_register() + * @ide: idle IDE settings descriptor + * + * In preparation for freeing @ide, remove sysfs enumeration for the + * stream. + */ +void pci_ide_stream_unregister(struct pci_ide *ide) +{ + struct pci_dev *pdev = ide->pdev; + struct pci_host_bridge *hb = pci_find_host_bridge(pdev->bus); + + sysfs_remove_link(&hb->dev.kobj, ide->name); + kfree(ide->name); + ida_free(&hb->ide_stream_ids_ida, ide->stream_id); + ide->name = NULL; +} +EXPORT_SYMBOL_GPL(pci_ide_stream_unregister); + +static int pci_ide_domain(struct pci_dev *pdev) +{ + if (pdev->fm_enabled) + return pci_domain_nr(pdev->bus); + return 0; +} + +struct pci_ide_partner *pci_ide_to_settings(struct pci_dev *pdev, struct pci_ide *ide) +{ + if (!pci_is_pcie(pdev)) { + pci_warn_once(pdev, "not a PCIe device\n"); + return NULL; + } + + switch (pci_pcie_type(pdev)) { + case PCI_EXP_TYPE_ENDPOINT: + if (pdev != ide->pdev) { + pci_warn_once(pdev, "setup expected Endpoint: %s\n", pci_name(ide->pdev)); + return NULL; + } + return &ide->partner[PCI_IDE_EP]; + case PCI_EXP_TYPE_ROOT_PORT: { + struct pci_dev *rp = pcie_find_root_port(ide->pdev); + + if (pdev != rp) { + pci_warn_once(pdev, "setup expected Root Port: %s\n", + pci_name(rp)); + return NULL; + } + return &ide->partner[PCI_IDE_RP]; + } + default: + pci_warn_once(pdev, "invalid device type\n"); + return NULL; + } +} +EXPORT_SYMBOL_GPL(pci_ide_to_settings); + +static void set_ide_sel_ctl(struct pci_dev *pdev, struct pci_ide *ide, + struct pci_ide_partner *settings, int pos, + bool enable) +{ + u32 val = FIELD_PREP(PCI_IDE_SEL_CTL_ID, ide->stream_id) | + FIELD_PREP(PCI_IDE_SEL_CTL_DEFAULT, settings->default_stream) | + FIELD_PREP(PCI_IDE_SEL_CTL_CFG_EN, pdev->ide_cfg) | + FIELD_PREP(PCI_IDE_SEL_CTL_TEE_LIMITED, pdev->ide_tee_limit) | + FIELD_PREP(PCI_IDE_SEL_CTL_EN, enable); + + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_CTL, val); +} + +#define SEL_ADDR1_LOWER GENMASK(31, 20) +#define SEL_ADDR_UPPER GENMASK_ULL(63, 32) +#define PREP_PCI_IDE_SEL_ADDR1(base, limit) \ + (FIELD_PREP(PCI_IDE_SEL_ADDR_1_VALID, 1) | \ + FIELD_PREP(PCI_IDE_SEL_ADDR_1_BASE_LOW, \ + FIELD_GET(SEL_ADDR1_LOWER, (base))) | \ + FIELD_PREP(PCI_IDE_SEL_ADDR_1_LIMIT_LOW, \ + FIELD_GET(SEL_ADDR1_LOWER, (limit)))) + +static void mem_assoc_to_regs(struct pci_bus_region *region, + struct pci_ide_regs *regs, int idx) +{ + /* convert to u64 range for bitfield size checks */ + struct range r = { region->start, region->end }; + + regs->addr[idx].assoc1 = PREP_PCI_IDE_SEL_ADDR1(r.start, r.end); + regs->addr[idx].assoc2 = FIELD_GET(SEL_ADDR_UPPER, r.end); + regs->addr[idx].assoc3 = FIELD_GET(SEL_ADDR_UPPER, r.start); +} + +/** + * pci_ide_stream_to_regs() - convert IDE settings to association register values + * @pdev: PCIe device object for either a Root Port or Endpoint Partner Port + * @ide: registered IDE settings descriptor + * @regs: output register values + */ +static void pci_ide_stream_to_regs(struct pci_dev *pdev, struct pci_ide *ide, + struct pci_ide_regs *regs) +{ + struct pci_ide_partner *settings = pci_ide_to_settings(pdev, ide); + int assoc_idx = 0; + + memset(regs, 0, sizeof(*regs)); + + if (!settings) + return; + + regs->rid1 = FIELD_PREP(PCI_IDE_SEL_RID_1_LIMIT, settings->rid_end); + + regs->rid2 = FIELD_PREP(PCI_IDE_SEL_RID_2_VALID, 1) | + FIELD_PREP(PCI_IDE_SEL_RID_2_BASE, settings->rid_start) | + FIELD_PREP(PCI_IDE_SEL_RID_2_SEG, pci_ide_domain(pdev)); + + if (pdev->nr_ide_mem && pci_bus_region_size(&settings->mem_assoc)) { + mem_assoc_to_regs(&settings->mem_assoc, regs, assoc_idx); + assoc_idx++; + } + + if (pdev->nr_ide_mem > assoc_idx && + pci_bus_region_size(&settings->pref_assoc)) { + mem_assoc_to_regs(&settings->pref_assoc, regs, assoc_idx); + assoc_idx++; + } + + regs->nr_addr = assoc_idx; +} + +/** + * pci_ide_stream_setup() - program settings to Selective IDE Stream registers + * @pdev: PCIe device object for either a Root Port or Endpoint Partner Port + * @ide: registered IDE settings descriptor + * + * When @pdev is a PCI_EXP_TYPE_ENDPOINT then the PCI_IDE_EP partner + * settings are written to @pdev's Selective IDE Stream register block, + * and when @pdev is a PCI_EXP_TYPE_ROOT_PORT, the PCI_IDE_RP settings + * are selected. + */ +void pci_ide_stream_setup(struct pci_dev *pdev, struct pci_ide *ide) +{ + struct pci_ide_partner *settings = pci_ide_to_settings(pdev, ide); + struct pci_ide_regs regs; + int pos; + + if (!settings) + return; + + pci_ide_stream_to_regs(pdev, ide, ®s); + + pos = sel_ide_offset(pdev, settings); + + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_RID_1, regs.rid1); + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_RID_2, regs.rid2); + + for (int i = 0; i < regs.nr_addr; i++) { + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_ADDR_1(i), + regs.addr[i].assoc1); + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_ADDR_2(i), + regs.addr[i].assoc2); + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_ADDR_3(i), + regs.addr[i].assoc3); + } + + /* clear extra unused address association blocks */ + for (int i = regs.nr_addr; i < pdev->nr_ide_mem; i++) { + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_ADDR_1(i), 0); + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_ADDR_2(i), 0); + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_ADDR_3(i), 0); + } + + /* + * Setup control register early for devices that expect + * stream_id is set during key programming. + */ + set_ide_sel_ctl(pdev, ide, settings, pos, false); + settings->setup = 1; +} +EXPORT_SYMBOL_GPL(pci_ide_stream_setup); + +/** + * pci_ide_stream_teardown() - disable the stream and clear all settings + * @pdev: PCIe device object for either a Root Port or Endpoint Partner Port + * @ide: registered IDE settings descriptor + * + * For stream destruction, zero all registers that may have been written + * by pci_ide_stream_setup(). Consider pci_ide_stream_disable() to leave + * settings in place while temporarily disabling the stream. + */ +void pci_ide_stream_teardown(struct pci_dev *pdev, struct pci_ide *ide) +{ + struct pci_ide_partner *settings = pci_ide_to_settings(pdev, ide); + int pos, i; + + if (!settings) + return; + + pos = sel_ide_offset(pdev, settings); + + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_CTL, 0); + + for (i = 0; i < pdev->nr_ide_mem; i++) { + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_ADDR_1(i), 0); + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_ADDR_2(i), 0); + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_ADDR_3(i), 0); + } + + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_RID_2, 0); + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_RID_1, 0); + settings->setup = 0; +} +EXPORT_SYMBOL_GPL(pci_ide_stream_teardown); + +/** + * pci_ide_stream_enable() - enable a Selective IDE Stream + * @pdev: PCIe device object for either a Root Port or Endpoint Partner Port + * @ide: registered and setup IDE settings descriptor + * + * Activate the stream by writing to the Selective IDE Stream Control + * Register. + * + * Return: 0 if the stream successfully entered the "secure" state, and -EINVAL + * if @ide is invalid, and -ENXIO if the stream fails to enter the secure state. + * + * Note that the state may go "insecure" at any point after returning 0, but + * those events are equivalent to a "link down" event and handled via + * asynchronous error reporting. + * + * Caller is responsible to clear the enable bit in the -ENXIO case. + */ +int pci_ide_stream_enable(struct pci_dev *pdev, struct pci_ide *ide) +{ + struct pci_ide_partner *settings = pci_ide_to_settings(pdev, ide); + int pos; + u32 val; + + if (!settings) + return -EINVAL; + + pos = sel_ide_offset(pdev, settings); + + set_ide_sel_ctl(pdev, ide, settings, pos, true); + settings->enable = 1; + + pci_read_config_dword(pdev, pos + PCI_IDE_SEL_STS, &val); + if (FIELD_GET(PCI_IDE_SEL_STS_STATE, val) != + PCI_IDE_SEL_STS_STATE_SECURE) + return -ENXIO; + + return 0; +} +EXPORT_SYMBOL_GPL(pci_ide_stream_enable); + +/** + * pci_ide_stream_disable() - disable a Selective IDE Stream + * @pdev: PCIe device object for either a Root Port or Endpoint Partner Port + * @ide: registered and setup IDE settings descriptor + * + * Clear the Selective IDE Stream Control Register, but leave all other + * registers untouched. + */ +void pci_ide_stream_disable(struct pci_dev *pdev, struct pci_ide *ide) +{ + struct pci_ide_partner *settings = pci_ide_to_settings(pdev, ide); + int pos; + + if (!settings) + return; + + pos = sel_ide_offset(pdev, settings); + + pci_write_config_dword(pdev, pos + PCI_IDE_SEL_CTL, 0); + settings->enable = 0; +} +EXPORT_SYMBOL_GPL(pci_ide_stream_disable); + +void pci_ide_init_host_bridge(struct pci_host_bridge *hb) +{ + hb->nr_ide_streams = 256; + ida_init(&hb->ide_stream_ida); + ida_init(&hb->ide_stream_ids_ida); + reserve_stream_id(hb, PCI_IDE_RESERVED_STREAM_ID); +} + +static ssize_t available_secure_streams_show(struct device *dev, + struct device_attribute *attr, + char *buf) +{ + struct pci_host_bridge *hb = to_pci_host_bridge(dev); + int nr = READ_ONCE(hb->nr_ide_streams); + int avail = nr; + + if (!nr) + return -ENXIO; + + /* + * Yes, this is inefficient and racy, but it is only for occasional + * platform resource surveys. Worst case is bounded to 256 streams. + */ + for (int i = 0; i < nr; i++) + if (ida_exists(&hb->ide_stream_ida, i)) + avail--; + return sysfs_emit(buf, "%d\n", avail); +} +static DEVICE_ATTR_RO(available_secure_streams); + +static struct attribute *pci_ide_attrs[] = { + &dev_attr_available_secure_streams.attr, + NULL +}; + +static umode_t pci_ide_attr_visible(struct kobject *kobj, struct attribute *a, int n) +{ + struct device *dev = kobj_to_dev(kobj); + struct pci_host_bridge *hb = to_pci_host_bridge(dev); + + if (a == &dev_attr_available_secure_streams.attr) + if (!hb->nr_ide_streams) + return 0; + + return a->mode; +} + +const struct attribute_group pci_ide_attr_group = { + .attrs = pci_ide_attrs, + .is_visible = pci_ide_attr_visible, +}; + +/** + * pci_ide_set_nr_streams() - sets size of the pool of IDE Stream resources + * @hb: host bridge boundary for the stream pool + * @nr: number of streams + * + * Platform PCI init and/or expert test module use only. Limit IDE + * Stream establishment by setting the number of stream resources + * available at the host bridge. Platform init code must set this before + * the first pci_ide_stream_alloc() call if the platform has less than the + * default of 256 streams per host-bridge. + * + * The "PCI_IDE" symbol namespace is required because this is typically + * a detail that is settled in early PCI init. I.e. this export is not + * for endpoint drivers. + */ +void pci_ide_set_nr_streams(struct pci_host_bridge *hb, u16 nr) +{ + hb->nr_ide_streams = min(nr, 256); + WARN_ON_ONCE(!ida_is_empty(&hb->ide_stream_ida)); + sysfs_update_group(&hb->dev.kobj, &pci_ide_attr_group); +} +EXPORT_SYMBOL_NS_GPL(pci_ide_set_nr_streams, "PCI_IDE"); + +void pci_ide_destroy(struct pci_dev *pdev) +{ + ida_destroy(&pdev->ide_stream_ida); +} diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c index 80a7c4fe6b03..c2df915ad2d2 100644 --- a/drivers/pci/pci-sysfs.c +++ b/drivers/pci/pci-sysfs.c @@ -1856,5 +1856,9 @@ const struct attribute_group *pci_dev_attr_groups[] = { #ifdef CONFIG_PCI_DOE &pci_doe_sysfs_group, #endif +#ifdef CONFIG_PCI_TSM + &pci_tsm_auth_attr_group, + &pci_tsm_attr_group, +#endif NULL, }; diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h index a33bc4e0bf34..0e67014aa001 100644 --- a/drivers/pci/pci.h +++ b/drivers/pci/pci.h @@ -615,6 +615,27 @@ static inline void pci_doe_sysfs_init(struct pci_dev *pdev) { } static inline void pci_doe_sysfs_teardown(struct pci_dev *pdev) { } #endif +#ifdef CONFIG_PCI_IDE +void pci_ide_init(struct pci_dev *dev); +void pci_ide_init_host_bridge(struct pci_host_bridge *hb); +void pci_ide_destroy(struct pci_dev *dev); +extern const struct attribute_group pci_ide_attr_group; +#else +static inline void pci_ide_init(struct pci_dev *dev) { } +static inline void pci_ide_init_host_bridge(struct pci_host_bridge *hb) { } +static inline void pci_ide_destroy(struct pci_dev *dev) { } +#endif + +#ifdef CONFIG_PCI_TSM +void pci_tsm_init(struct pci_dev *pdev); +void pci_tsm_destroy(struct pci_dev *pdev); +extern const struct attribute_group pci_tsm_attr_group; +extern const struct attribute_group pci_tsm_auth_attr_group; +#else +static inline void pci_tsm_init(struct pci_dev *pdev) { } +static inline void pci_tsm_destroy(struct pci_dev *pdev) { } +#endif + /** * pci_dev_set_io_state - Set the new error state if possible. * diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c index 0b5ed4722ac3..e0bcaa896803 100644 --- a/drivers/pci/pcie/aer.c +++ b/drivers/pci/pcie/aer.c @@ -30,6 +30,7 @@ #include <linux/kfifo.h> #include <linux/ratelimit.h> #include <linux/slab.h> +#include <linux/vmcore_info.h> #include <acpi/apei.h> #include <acpi/ghes.h> #include <ras/ras_event.h> @@ -765,6 +766,7 @@ static void pci_dev_aer_stats_incr(struct pci_dev *pdev, break; case AER_NONFATAL: aer_info->dev_total_nonfatal_errs++; + hwerr_log_error_type(HWERR_RECOV_PCI); counter = &aer_info->dev_nonfatal_errs[0]; max = AER_MAX_TYPEOF_UNCOR_ERRS; break; diff --git a/drivers/pci/probe.c b/drivers/pci/probe.c index 124d2d309c58..41183aed8f5d 100644 --- a/drivers/pci/probe.c +++ b/drivers/pci/probe.c @@ -658,6 +658,18 @@ static void pci_release_host_bridge_dev(struct device *dev) kfree(bridge); } +static const struct attribute_group *pci_host_bridge_groups[] = { +#ifdef CONFIG_PCI_IDE + &pci_ide_attr_group, +#endif + NULL +}; + +static const struct device_type pci_host_bridge_type = { + .groups = pci_host_bridge_groups, + .release = pci_release_host_bridge_dev, +}; + static void pci_init_host_bridge(struct pci_host_bridge *bridge) { INIT_LIST_HEAD(&bridge->windows); @@ -677,6 +689,8 @@ static void pci_init_host_bridge(struct pci_host_bridge *bridge) bridge->native_dpc = 1; bridge->domain_nr = PCI_DOMAIN_NR_NOT_SET; bridge->native_cxl_error = 1; + bridge->dev.type = &pci_host_bridge_type; + pci_ide_init_host_bridge(bridge); device_initialize(&bridge->dev); } @@ -690,7 +704,6 @@ struct pci_host_bridge *pci_alloc_host_bridge(size_t priv) return NULL; pci_init_host_bridge(bridge); - bridge->dev.release = pci_release_host_bridge_dev; return bridge; } @@ -2296,6 +2309,17 @@ int pci_configure_extended_tags(struct pci_dev *dev, void *ign) return 0; } +static void pci_dev3_init(struct pci_dev *pdev) +{ + u16 cap = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DEV3); + u32 val = 0; + + if (!cap) + return; + pci_read_config_dword(pdev, cap + PCI_DEV3_STA, &val); + pdev->fm_enabled = !!(val & PCI_DEV3_STA_SEGMENT); +} + /** * pcie_relaxed_ordering_enabled - Probe for PCIe relaxed ordering enable * @dev: PCI device to query @@ -2680,6 +2704,8 @@ static void pci_init_capabilities(struct pci_dev *dev) pci_doe_init(dev); /* Data Object Exchange */ pci_tph_init(dev); /* TLP Processing Hints */ pci_rebar_init(dev); /* Resizable BAR */ + pci_dev3_init(dev); /* Device 3 capabilities */ + pci_ide_init(dev); /* Link Integrity and Data Encryption */ pcie_report_downtraining(dev); pci_init_reset_methods(dev); @@ -2773,6 +2799,9 @@ void pci_device_add(struct pci_dev *dev, struct pci_bus *bus) ret = device_add(&dev->dev); WARN_ON(ret < 0); + /* Establish pdev->tsm for newly added (e.g. new SR-IOV VFs) */ + pci_tsm_init(dev); + pci_npem_create(dev); pci_doe_sysfs_init(dev); diff --git a/drivers/pci/remove.c b/drivers/pci/remove.c index ce5c25adef55..417a9ea59117 100644 --- a/drivers/pci/remove.c +++ b/drivers/pci/remove.c @@ -57,6 +57,12 @@ static void pci_destroy_dev(struct pci_dev *dev) pci_doe_sysfs_teardown(dev); pci_npem_remove(dev); + /* + * While device is in D0 drop the device from TSM link operations + * including unbind and disconnect (IDE + SPDM teardown). + */ + pci_tsm_destroy(dev); + device_del(&dev->dev); down_write(&pci_bus_sem); @@ -64,6 +70,7 @@ static void pci_destroy_dev(struct pci_dev *dev) up_write(&pci_bus_sem); pci_doe_destroy(dev); + pci_ide_destroy(dev); pcie_aspm_exit_link_state(dev); pci_bridge_d3_update(dev); pci_pwrctrl_unregister(&dev->dev); diff --git a/drivers/pci/search.c b/drivers/pci/search.c index 53840634fbfc..e6e84dc62e82 100644 --- a/drivers/pci/search.c +++ b/drivers/pci/search.c @@ -282,6 +282,45 @@ static struct pci_dev *pci_get_dev_by_id(const struct pci_device_id *id, return pdev; } +static struct pci_dev *pci_get_dev_by_id_reverse(const struct pci_device_id *id, + struct pci_dev *from) +{ + struct device *dev; + struct device *dev_start = NULL; + struct pci_dev *pdev = NULL; + + if (from) + dev_start = &from->dev; + dev = bus_find_device_reverse(&pci_bus_type, dev_start, (void *)id, + match_pci_dev_by_id); + if (dev) + pdev = to_pci_dev(dev); + pci_dev_put(from); + return pdev; +} + +enum pci_search_direction { + PCI_SEARCH_FORWARD, + PCI_SEARCH_REVERSE, +}; + +static struct pci_dev *__pci_get_subsys(unsigned int vendor, unsigned int device, + unsigned int ss_vendor, unsigned int ss_device, + struct pci_dev *from, enum pci_search_direction dir) +{ + struct pci_device_id id = { + .vendor = vendor, + .device = device, + .subvendor = ss_vendor, + .subdevice = ss_device, + }; + + if (dir == PCI_SEARCH_FORWARD) + return pci_get_dev_by_id(&id, from); + else + return pci_get_dev_by_id_reverse(&id, from); +} + /** * pci_get_subsys - begin or continue searching for a PCI device by vendor/subvendor/device/subdevice id * @vendor: PCI vendor id to match, or %PCI_ANY_ID to match all vendor ids @@ -302,14 +341,8 @@ struct pci_dev *pci_get_subsys(unsigned int vendor, unsigned int device, unsigned int ss_vendor, unsigned int ss_device, struct pci_dev *from) { - struct pci_device_id id = { - .vendor = vendor, - .device = device, - .subvendor = ss_vendor, - .subdevice = ss_device, - }; - - return pci_get_dev_by_id(&id, from); + return __pci_get_subsys(vendor, device, ss_vendor, ss_device, from, + PCI_SEARCH_FORWARD); } EXPORT_SYMBOL(pci_get_subsys); @@ -334,6 +367,19 @@ struct pci_dev *pci_get_device(unsigned int vendor, unsigned int device, } EXPORT_SYMBOL(pci_get_device); +/* + * Same semantics as pci_get_device(), except walks the PCI device list + * in reverse discovery order. + */ +struct pci_dev *pci_get_device_reverse(unsigned int vendor, + unsigned int device, + struct pci_dev *from) +{ + return __pci_get_subsys(vendor, device, PCI_ANY_ID, PCI_ANY_ID, from, + PCI_SEARCH_REVERSE); +} +EXPORT_SYMBOL(pci_get_device_reverse); + /** * pci_get_class - begin or continue searching for a PCI device by class * @class: search for a PCI device with this class designation diff --git a/drivers/pci/tsm.c b/drivers/pci/tsm.c new file mode 100644 index 000000000000..5fdcd7f2e820 --- /dev/null +++ b/drivers/pci/tsm.c @@ -0,0 +1,900 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Interface with platform TEE Security Manager (TSM) objects as defined by + * PCIe r7.0 section 11 TEE Device Interface Security Protocol (TDISP) + * + * Copyright(c) 2024-2025 Intel Corporation. All rights reserved. + */ + +#define dev_fmt(fmt) "PCI/TSM: " fmt + +#include <linux/bitfield.h> +#include <linux/pci.h> +#include <linux/pci-doe.h> +#include <linux/pci-tsm.h> +#include <linux/sysfs.h> +#include <linux/tsm.h> +#include <linux/xarray.h> +#include "pci.h" + +/* + * Provide a read/write lock against the init / exit of pdev tsm + * capabilities and arrival/departure of a TSM instance + */ +static DECLARE_RWSEM(pci_tsm_rwsem); + +/* + * Count of TSMs registered that support physical link operations vs device + * security state management. + */ +static int pci_tsm_link_count; +static int pci_tsm_devsec_count; + +static const struct pci_tsm_ops *to_pci_tsm_ops(struct pci_tsm *tsm) +{ + return tsm->tsm_dev->pci_ops; +} + +static inline bool is_dsm(struct pci_dev *pdev) +{ + return pdev->tsm && pdev->tsm->dsm_dev == pdev; +} + +static inline bool has_tee(struct pci_dev *pdev) +{ + return pdev->devcap & PCI_EXP_DEVCAP_TEE; +} + +/* 'struct pci_tsm_pf0' wraps 'struct pci_tsm' when ->dsm_dev == ->pdev (self) */ +static struct pci_tsm_pf0 *to_pci_tsm_pf0(struct pci_tsm *tsm) +{ + /* + * All "link" TSM contexts reference the device that hosts the DSM + * interface for a set of devices. Walk to the DSM device and cast its + * ->tsm context to a 'struct pci_tsm_pf0 *'. + */ + struct pci_dev *pf0 = tsm->dsm_dev; + + if (!is_pci_tsm_pf0(pf0) || !is_dsm(pf0)) { + pci_WARN_ONCE(tsm->pdev, 1, "invalid context object\n"); + return NULL; + } + + return container_of(pf0->tsm, struct pci_tsm_pf0, base_tsm); +} + +static void tsm_remove(struct pci_tsm *tsm) +{ + struct pci_dev *pdev; + + if (!tsm) + return; + + pdev = tsm->pdev; + to_pci_tsm_ops(tsm)->remove(tsm); + pdev->tsm = NULL; +} +DEFINE_FREE(tsm_remove, struct pci_tsm *, if (_T) tsm_remove(_T)) + +static void pci_tsm_walk_fns(struct pci_dev *pdev, + int (*cb)(struct pci_dev *pdev, void *data), + void *data) +{ + /* Walk subordinate physical functions */ + for (int i = 0; i < 8; i++) { + struct pci_dev *pf __free(pci_dev_put) = pci_get_slot( + pdev->bus, PCI_DEVFN(PCI_SLOT(pdev->devfn), i)); + + if (!pf) + continue; + + /* on entry function 0 has already run @cb */ + if (i > 0) + cb(pf, data); + + /* walk virtual functions of each pf */ + for (int j = 0; j < pci_num_vf(pf); j++) { + struct pci_dev *vf __free(pci_dev_put) = + pci_get_domain_bus_and_slot( + pci_domain_nr(pf->bus), + pci_iov_virtfn_bus(pf, j), + pci_iov_virtfn_devfn(pf, j)); + + if (!vf) + continue; + + cb(vf, data); + } + } + + /* + * Walk downstream devices, assumes that an upstream DSM is + * limited to downstream physical functions + */ + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_UPSTREAM && is_dsm(pdev)) + pci_walk_bus(pdev->subordinate, cb, data); +} + +static void pci_tsm_walk_fns_reverse(struct pci_dev *pdev, + int (*cb)(struct pci_dev *pdev, + void *data), + void *data) +{ + /* Reverse walk downstream devices */ + if (pci_pcie_type(pdev) == PCI_EXP_TYPE_UPSTREAM && is_dsm(pdev)) + pci_walk_bus_reverse(pdev->subordinate, cb, data); + + /* Reverse walk subordinate physical functions */ + for (int i = 7; i >= 0; i--) { + struct pci_dev *pf __free(pci_dev_put) = pci_get_slot( + pdev->bus, PCI_DEVFN(PCI_SLOT(pdev->devfn), i)); + + if (!pf) + continue; + + /* reverse walk virtual functions */ + for (int j = pci_num_vf(pf) - 1; j >= 0; j--) { + struct pci_dev *vf __free(pci_dev_put) = + pci_get_domain_bus_and_slot( + pci_domain_nr(pf->bus), + pci_iov_virtfn_bus(pf, j), + pci_iov_virtfn_devfn(pf, j)); + + if (!vf) + continue; + cb(vf, data); + } + + /* on exit, caller will run @cb on function 0 */ + if (i > 0) + cb(pf, data); + } +} + +static void link_sysfs_disable(struct pci_dev *pdev) +{ + sysfs_update_group(&pdev->dev.kobj, &pci_tsm_auth_attr_group); + sysfs_update_group(&pdev->dev.kobj, &pci_tsm_attr_group); +} + +static void link_sysfs_enable(struct pci_dev *pdev) +{ + bool tee = has_tee(pdev); + + pci_dbg(pdev, "%s Security Manager detected (%s%s%s)\n", + pdev->tsm ? "Device" : "Platform TEE", + pdev->ide_cap ? "IDE" : "", pdev->ide_cap && tee ? " " : "", + tee ? "TEE" : ""); + + sysfs_update_group(&pdev->dev.kobj, &pci_tsm_auth_attr_group); + sysfs_update_group(&pdev->dev.kobj, &pci_tsm_attr_group); +} + +static int probe_fn(struct pci_dev *pdev, void *dsm) +{ + struct pci_dev *dsm_dev = dsm; + const struct pci_tsm_ops *ops = to_pci_tsm_ops(dsm_dev->tsm); + + pdev->tsm = ops->probe(dsm_dev->tsm->tsm_dev, pdev); + pci_dbg(pdev, "setup TSM context: DSM: %s status: %s\n", + pci_name(dsm_dev), pdev->tsm ? "success" : "failed"); + if (pdev->tsm) + link_sysfs_enable(pdev); + return 0; +} + +static int pci_tsm_connect(struct pci_dev *pdev, struct tsm_dev *tsm_dev) +{ + int rc; + struct pci_tsm_pf0 *tsm_pf0; + const struct pci_tsm_ops *ops = tsm_dev->pci_ops; + struct pci_tsm *pci_tsm __free(tsm_remove) = ops->probe(tsm_dev, pdev); + + /* connect() mutually exclusive with subfunction pci_tsm_init() */ + lockdep_assert_held_write(&pci_tsm_rwsem); + + if (!pci_tsm) + return -ENXIO; + + pdev->tsm = pci_tsm; + tsm_pf0 = to_pci_tsm_pf0(pdev->tsm); + + /* mutex_intr assumes connect() is always sysfs/user driven */ + ACQUIRE(mutex_intr, lock)(&tsm_pf0->lock); + if ((rc = ACQUIRE_ERR(mutex_intr, &lock))) + return rc; + + rc = ops->connect(pdev); + if (rc) + return rc; + + pdev->tsm = no_free_ptr(pci_tsm); + + /* + * Now that the DSM is established, probe() all the potential + * dependent functions. Failure to probe a function is not fatal + * to connect(), it just disables subsequent security operations + * for that function. + * + * Note this is done unconditionally, without regard to finding + * PCI_EXP_DEVCAP_TEE on the dependent function, for robustness. The DSM + * is the ultimate arbiter of security state relative to a given + * interface id, and if it says it can manage TDISP state of a function, + * let it. + */ + if (has_tee(pdev)) + pci_tsm_walk_fns(pdev, probe_fn, pdev); + return 0; +} + +static ssize_t connect_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct tsm_dev *tsm_dev; + int rc; + + ACQUIRE(rwsem_read_intr, lock)(&pci_tsm_rwsem); + if ((rc = ACQUIRE_ERR(rwsem_read_intr, &lock))) + return rc; + + if (!pdev->tsm) + return sysfs_emit(buf, "\n"); + + tsm_dev = pdev->tsm->tsm_dev; + return sysfs_emit(buf, "%s\n", dev_name(&tsm_dev->dev)); +} + +/* Is @tsm_dev managing physical link / session properties... */ +static bool is_link_tsm(struct tsm_dev *tsm_dev) +{ + return tsm_dev && tsm_dev->pci_ops && tsm_dev->pci_ops->link_ops.probe; +} + +/* ...or is @tsm_dev managing device security state ? */ +static bool is_devsec_tsm(struct tsm_dev *tsm_dev) +{ + return tsm_dev && tsm_dev->pci_ops && tsm_dev->pci_ops->devsec_ops.lock; +} + +static ssize_t connect_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t len) +{ + struct pci_dev *pdev = to_pci_dev(dev); + int rc, id; + + rc = sscanf(buf, "tsm%d\n", &id); + if (rc != 1) + return -EINVAL; + + ACQUIRE(rwsem_write_kill, lock)(&pci_tsm_rwsem); + if ((rc = ACQUIRE_ERR(rwsem_write_kill, &lock))) + return rc; + + if (pdev->tsm) + return -EBUSY; + + struct tsm_dev *tsm_dev __free(put_tsm_dev) = find_tsm_dev(id); + if (!is_link_tsm(tsm_dev)) + return -ENXIO; + + rc = pci_tsm_connect(pdev, tsm_dev); + if (rc) + return rc; + return len; +} +static DEVICE_ATTR_RW(connect); + +static int remove_fn(struct pci_dev *pdev, void *data) +{ + tsm_remove(pdev->tsm); + link_sysfs_disable(pdev); + return 0; +} + +/* + * Note, this helper only returns an error code and takes an argument for + * compatibility with the pci_walk_bus() callback prototype. pci_tsm_unbind() + * always succeeds. + */ +static int __pci_tsm_unbind(struct pci_dev *pdev, void *data) +{ + struct pci_tdi *tdi; + struct pci_tsm_pf0 *tsm_pf0; + + lockdep_assert_held(&pci_tsm_rwsem); + + if (!pdev->tsm) + return 0; + + tsm_pf0 = to_pci_tsm_pf0(pdev->tsm); + guard(mutex)(&tsm_pf0->lock); + + tdi = pdev->tsm->tdi; + if (!tdi) + return 0; + + to_pci_tsm_ops(pdev->tsm)->unbind(tdi); + pdev->tsm->tdi = NULL; + + return 0; +} + +void pci_tsm_unbind(struct pci_dev *pdev) +{ + guard(rwsem_read)(&pci_tsm_rwsem); + __pci_tsm_unbind(pdev, NULL); +} +EXPORT_SYMBOL_GPL(pci_tsm_unbind); + +/** + * pci_tsm_bind() - Bind @pdev as a TDI for @kvm + * @pdev: PCI device function to bind + * @kvm: Private memory attach context + * @tdi_id: Identifier (virtual BDF) for the TDI as referenced by the TSM and DSM + * + * Returns 0 on success, or a negative error code on failure. + * + * Context: Caller is responsible for constraining the bind lifetime to the + * registered state of the device. For example, pci_tsm_bind() / + * pci_tsm_unbind() limited to the VFIO driver bound state of the device. + */ +int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u32 tdi_id) +{ + struct pci_tsm_pf0 *tsm_pf0; + struct pci_tdi *tdi; + + if (!kvm) + return -EINVAL; + + guard(rwsem_read)(&pci_tsm_rwsem); + + if (!pdev->tsm) + return -EINVAL; + + if (!is_link_tsm(pdev->tsm->tsm_dev)) + return -ENXIO; + + tsm_pf0 = to_pci_tsm_pf0(pdev->tsm); + guard(mutex)(&tsm_pf0->lock); + + /* Resolve races to bind a TDI */ + if (pdev->tsm->tdi) { + if (pdev->tsm->tdi->kvm != kvm) + return -EBUSY; + return 0; + } + + tdi = to_pci_tsm_ops(pdev->tsm)->bind(pdev, kvm, tdi_id); + if (IS_ERR(tdi)) + return PTR_ERR(tdi); + + pdev->tsm->tdi = tdi; + + return 0; +} +EXPORT_SYMBOL_GPL(pci_tsm_bind); + +/** + * pci_tsm_guest_req() - helper to marshal guest requests to the TSM driver + * @pdev: @pdev representing a bound tdi + * @scope: caller asserts this passthrough request is limited to TDISP operations + * @req_in: Input payload forwarded from the guest + * @in_len: Length of @req_in + * @req_out: Output payload buffer response to the guest + * @out_len: Length of @req_out on input, bytes filled in @req_out on output + * @tsm_code: Optional TSM arch specific result code for the guest TSM + * + * This is a common entry point for requests triggered by userspace KVM-exit + * service handlers responding to TDI information or state change requests. The + * scope parameter limits requests to TDISP state management, or limited debug. + * This path is only suitable for commands and results that are the host kernel + * has no use, the host is only facilitating guest to TSM communication. + * + * Returns 0 on success and -error on failure and positive "residue" on success + * but @req_out is filled with less then @out_len, or @req_out is NULL and a + * residue number of bytes were not consumed from @req_in. On success or + * failure @tsm_code may be populated with a TSM implementation specific result + * code for the guest to consume. + * + * Context: Caller is responsible for calling this within the pci_tsm_bind() + * state of the TDI. + */ +ssize_t pci_tsm_guest_req(struct pci_dev *pdev, enum pci_tsm_req_scope scope, + sockptr_t req_in, size_t in_len, sockptr_t req_out, + size_t out_len, u64 *tsm_code) +{ + struct pci_tsm_pf0 *tsm_pf0; + struct pci_tdi *tdi; + int rc; + + /* Forbid requests that are not directly related to TDISP operations */ + if (scope > PCI_TSM_REQ_STATE_CHANGE) + return -EINVAL; + + ACQUIRE(rwsem_read_intr, lock)(&pci_tsm_rwsem); + if ((rc = ACQUIRE_ERR(rwsem_read_intr, &lock))) + return rc; + + if (!pdev->tsm) + return -ENXIO; + + if (!is_link_tsm(pdev->tsm->tsm_dev)) + return -ENXIO; + + tsm_pf0 = to_pci_tsm_pf0(pdev->tsm); + ACQUIRE(mutex_intr, ops_lock)(&tsm_pf0->lock); + if ((rc = ACQUIRE_ERR(mutex_intr, &ops_lock))) + return rc; + + tdi = pdev->tsm->tdi; + if (!tdi) + return -ENXIO; + return to_pci_tsm_ops(pdev->tsm)->guest_req(tdi, scope, req_in, in_len, + req_out, out_len, tsm_code); +} +EXPORT_SYMBOL_GPL(pci_tsm_guest_req); + +static void pci_tsm_unbind_all(struct pci_dev *pdev) +{ + pci_tsm_walk_fns_reverse(pdev, __pci_tsm_unbind, NULL); + __pci_tsm_unbind(pdev, NULL); +} + +static void __pci_tsm_disconnect(struct pci_dev *pdev) +{ + struct pci_tsm_pf0 *tsm_pf0 = to_pci_tsm_pf0(pdev->tsm); + const struct pci_tsm_ops *ops = to_pci_tsm_ops(pdev->tsm); + + /* disconnect() mutually exclusive with subfunction pci_tsm_init() */ + lockdep_assert_held_write(&pci_tsm_rwsem); + + pci_tsm_unbind_all(pdev); + + /* + * disconnect() is uninterruptible as it may be called for device + * teardown + */ + guard(mutex)(&tsm_pf0->lock); + pci_tsm_walk_fns_reverse(pdev, remove_fn, NULL); + ops->disconnect(pdev); +} + +static void pci_tsm_disconnect(struct pci_dev *pdev) +{ + __pci_tsm_disconnect(pdev); + tsm_remove(pdev->tsm); +} + +static ssize_t disconnect_store(struct device *dev, + struct device_attribute *attr, const char *buf, + size_t len) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct tsm_dev *tsm_dev; + int rc; + + ACQUIRE(rwsem_write_kill, lock)(&pci_tsm_rwsem); + if ((rc = ACQUIRE_ERR(rwsem_write_kill, &lock))) + return rc; + + if (!pdev->tsm) + return -ENXIO; + + tsm_dev = pdev->tsm->tsm_dev; + if (!sysfs_streq(buf, dev_name(&tsm_dev->dev))) + return -EINVAL; + + pci_tsm_disconnect(pdev); + return len; +} +static DEVICE_ATTR_WO(disconnect); + +static ssize_t bound_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_tsm_pf0 *tsm_pf0; + struct pci_tsm *tsm; + int rc; + + ACQUIRE(rwsem_read_intr, lock)(&pci_tsm_rwsem); + if ((rc = ACQUIRE_ERR(rwsem_read_intr, &lock))) + return rc; + + tsm = pdev->tsm; + if (!tsm) + return sysfs_emit(buf, "\n"); + tsm_pf0 = to_pci_tsm_pf0(tsm); + + ACQUIRE(mutex_intr, ops_lock)(&tsm_pf0->lock); + if ((rc = ACQUIRE_ERR(mutex_intr, &ops_lock))) + return rc; + + if (!tsm->tdi) + return sysfs_emit(buf, "\n"); + return sysfs_emit(buf, "%s\n", dev_name(&tsm->tsm_dev->dev)); +} +static DEVICE_ATTR_RO(bound); + +static ssize_t dsm_show(struct device *dev, struct device_attribute *attr, + char *buf) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct pci_tsm *tsm; + int rc; + + ACQUIRE(rwsem_read_intr, lock)(&pci_tsm_rwsem); + if ((rc = ACQUIRE_ERR(rwsem_read_intr, &lock))) + return rc; + + tsm = pdev->tsm; + if (!tsm) + return sysfs_emit(buf, "\n"); + + return sysfs_emit(buf, "%s\n", pci_name(tsm->dsm_dev)); +} +static DEVICE_ATTR_RO(dsm); + +/* The 'authenticated' attribute is exclusive to the presence of a 'link' TSM */ +static bool pci_tsm_link_group_visible(struct kobject *kobj) +{ + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + + if (!pci_tsm_link_count) + return false; + + if (!pci_is_pcie(pdev)) + return false; + + if (is_pci_tsm_pf0(pdev)) + return true; + + /* + * Show 'authenticated' and other attributes for the managed + * sub-functions of a DSM. + */ + if (pdev->tsm) + return true; + + return false; +} +DEFINE_SIMPLE_SYSFS_GROUP_VISIBLE(pci_tsm_link); + +/* + * 'link' and 'devsec' TSMs share the same 'tsm/' sysfs group, so the TSM type + * specific attributes need individual visibility checks. + */ +static umode_t pci_tsm_attr_visible(struct kobject *kobj, + struct attribute *attr, int n) +{ + if (pci_tsm_link_group_visible(kobj)) { + struct pci_dev *pdev = to_pci_dev(kobj_to_dev(kobj)); + + if (attr == &dev_attr_bound.attr) { + if (is_pci_tsm_pf0(pdev) && has_tee(pdev)) + return attr->mode; + if (pdev->tsm && has_tee(pdev->tsm->dsm_dev)) + return attr->mode; + } + + if (attr == &dev_attr_dsm.attr) { + if (is_pci_tsm_pf0(pdev)) + return attr->mode; + if (pdev->tsm && has_tee(pdev->tsm->dsm_dev)) + return attr->mode; + } + + if (attr == &dev_attr_connect.attr || + attr == &dev_attr_disconnect.attr) { + if (is_pci_tsm_pf0(pdev)) + return attr->mode; + } + } + + return 0; +} + +static bool pci_tsm_group_visible(struct kobject *kobj) +{ + return pci_tsm_link_group_visible(kobj); +} +DEFINE_SYSFS_GROUP_VISIBLE(pci_tsm); + +static struct attribute *pci_tsm_attrs[] = { + &dev_attr_connect.attr, + &dev_attr_disconnect.attr, + &dev_attr_bound.attr, + &dev_attr_dsm.attr, + NULL +}; + +const struct attribute_group pci_tsm_attr_group = { + .name = "tsm", + .attrs = pci_tsm_attrs, + .is_visible = SYSFS_GROUP_VISIBLE(pci_tsm), +}; + +static ssize_t authenticated_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + /* + * When the SPDM session established via TSM the 'authenticated' state + * of the device is identical to the connect state. + */ + return connect_show(dev, attr, buf); +} +static DEVICE_ATTR_RO(authenticated); + +static struct attribute *pci_tsm_auth_attrs[] = { + &dev_attr_authenticated.attr, + NULL +}; + +const struct attribute_group pci_tsm_auth_attr_group = { + .attrs = pci_tsm_auth_attrs, + .is_visible = SYSFS_GROUP_VISIBLE(pci_tsm_link), +}; + +/* + * Retrieve physical function0 device whether it has TEE capability or not + */ +static struct pci_dev *pf0_dev_get(struct pci_dev *pdev) +{ + struct pci_dev *pf_dev = pci_physfn(pdev); + + if (PCI_FUNC(pf_dev->devfn) == 0) + return pci_dev_get(pf_dev); + + return pci_get_slot(pf_dev->bus, + pf_dev->devfn - PCI_FUNC(pf_dev->devfn)); +} + +/* + * Find the PCI Device instance that serves as the Device Security Manager (DSM) + * for @pdev. Note that no additional reference is held for the resulting device + * because that resulting object always has a registered lifetime + * greater-than-or-equal to that of the @pdev argument. This is by virtue of + * @pdev being a descendant of, or identical to, the returned DSM device. + */ +static struct pci_dev *find_dsm_dev(struct pci_dev *pdev) +{ + struct device *grandparent; + struct pci_dev *uport; + + if (is_pci_tsm_pf0(pdev)) + return pdev; + + struct pci_dev *pf0 __free(pci_dev_put) = pf0_dev_get(pdev); + if (!pf0) + return NULL; + + if (is_dsm(pf0)) + return pf0; + + /* + * For cases where a switch may be hosting TDISP services on behalf of + * downstream devices, check the first upstream port relative to this + * endpoint. + */ + if (!pdev->dev.parent) + return NULL; + grandparent = pdev->dev.parent->parent; + if (!grandparent) + return NULL; + if (!dev_is_pci(grandparent)) + return NULL; + uport = to_pci_dev(grandparent); + if (!pci_is_pcie(uport) || + pci_pcie_type(uport) != PCI_EXP_TYPE_UPSTREAM) + return NULL; + + if (is_dsm(uport)) + return uport; + return NULL; +} + +/** + * pci_tsm_tdi_constructor() - base 'struct pci_tdi' initialization for link TSMs + * @pdev: PCI device function representing the TDI + * @tdi: context to initialize + * @kvm: Private memory attach context + * @tdi_id: Identifier (virtual BDF) for the TDI as referenced by the TSM and DSM + */ +void pci_tsm_tdi_constructor(struct pci_dev *pdev, struct pci_tdi *tdi, + struct kvm *kvm, u32 tdi_id) +{ + tdi->pdev = pdev; + tdi->kvm = kvm; + tdi->tdi_id = tdi_id; +} +EXPORT_SYMBOL_GPL(pci_tsm_tdi_constructor); + +/** + * pci_tsm_link_constructor() - base 'struct pci_tsm' initialization for link TSMs + * @pdev: The PCI device + * @tsm: context to initialize + * @tsm_dev: Platform TEE Security Manager, initiator of security operations + */ +int pci_tsm_link_constructor(struct pci_dev *pdev, struct pci_tsm *tsm, + struct tsm_dev *tsm_dev) +{ + if (!is_link_tsm(tsm_dev)) + return -EINVAL; + + tsm->dsm_dev = find_dsm_dev(pdev); + if (!tsm->dsm_dev) { + pci_warn(pdev, "failed to find Device Security Manager\n"); + return -ENXIO; + } + tsm->pdev = pdev; + tsm->tsm_dev = tsm_dev; + + return 0; +} +EXPORT_SYMBOL_GPL(pci_tsm_link_constructor); + +/** + * pci_tsm_pf0_constructor() - common 'struct pci_tsm_pf0' (DSM) initialization + * @pdev: Physical Function 0 PCI device (as indicated by is_pci_tsm_pf0()) + * @tsm: context to initialize + * @tsm_dev: Platform TEE Security Manager, initiator of security operations + */ +int pci_tsm_pf0_constructor(struct pci_dev *pdev, struct pci_tsm_pf0 *tsm, + struct tsm_dev *tsm_dev) +{ + mutex_init(&tsm->lock); + tsm->doe_mb = pci_find_doe_mailbox(pdev, PCI_VENDOR_ID_PCI_SIG, + PCI_DOE_FEATURE_CMA); + if (!tsm->doe_mb) { + pci_warn(pdev, "TSM init failure, no CMA mailbox\n"); + return -ENODEV; + } + + return pci_tsm_link_constructor(pdev, &tsm->base_tsm, tsm_dev); +} +EXPORT_SYMBOL_GPL(pci_tsm_pf0_constructor); + +void pci_tsm_pf0_destructor(struct pci_tsm_pf0 *pf0_tsm) +{ + mutex_destroy(&pf0_tsm->lock); +} +EXPORT_SYMBOL_GPL(pci_tsm_pf0_destructor); + +int pci_tsm_register(struct tsm_dev *tsm_dev) +{ + struct pci_dev *pdev = NULL; + + if (!tsm_dev) + return -EINVAL; + + /* The TSM device must only implement one of link_ops or devsec_ops */ + if (!is_link_tsm(tsm_dev) && !is_devsec_tsm(tsm_dev)) + return -EINVAL; + + if (is_link_tsm(tsm_dev) && is_devsec_tsm(tsm_dev)) + return -EINVAL; + + guard(rwsem_write)(&pci_tsm_rwsem); + + /* On first enable, update sysfs groups */ + if (is_link_tsm(tsm_dev) && pci_tsm_link_count++ == 0) { + for_each_pci_dev(pdev) + if (is_pci_tsm_pf0(pdev)) + link_sysfs_enable(pdev); + } else if (is_devsec_tsm(tsm_dev)) { + pci_tsm_devsec_count++; + } + + return 0; +} + +static void pci_tsm_fn_exit(struct pci_dev *pdev) +{ + __pci_tsm_unbind(pdev, NULL); + tsm_remove(pdev->tsm); +} + +/** + * __pci_tsm_destroy() - destroy the TSM context for @pdev + * @pdev: device to cleanup + * @tsm_dev: the TSM device being removed, or NULL if @pdev is being removed. + * + * At device removal or TSM unregistration all established context + * with the TSM is torn down. Additionally, if there are no more TSMs + * registered, the PCI tsm/ sysfs attributes are hidden. + */ +static void __pci_tsm_destroy(struct pci_dev *pdev, struct tsm_dev *tsm_dev) +{ + struct pci_tsm *tsm = pdev->tsm; + + lockdep_assert_held_write(&pci_tsm_rwsem); + + /* + * First, handle the TSM removal case to shutdown @pdev sysfs, this is + * skipped if the device itself is being removed since sysfs goes away + * naturally at that point + */ + if (is_link_tsm(tsm_dev) && is_pci_tsm_pf0(pdev) && !pci_tsm_link_count) + link_sysfs_disable(pdev); + + /* Nothing else to do if this device never attached to the departing TSM */ + if (!tsm) + return; + + /* Now lookup the tsm_dev to destroy TSM context */ + if (!tsm_dev) + tsm_dev = tsm->tsm_dev; + else if (tsm_dev != tsm->tsm_dev) + return; + + if (is_link_tsm(tsm_dev) && is_pci_tsm_pf0(pdev)) + pci_tsm_disconnect(pdev); + else + pci_tsm_fn_exit(pdev); +} + +void pci_tsm_destroy(struct pci_dev *pdev) +{ + guard(rwsem_write)(&pci_tsm_rwsem); + __pci_tsm_destroy(pdev, NULL); +} + +void pci_tsm_init(struct pci_dev *pdev) +{ + guard(rwsem_read)(&pci_tsm_rwsem); + + /* + * Subfunctions are either probed synchronous with connect() or later + * when either the SR-IOV configuration is changed, or, unlikely, + * connect() raced initial bus scanning. + */ + if (pdev->tsm) + return; + + if (pci_tsm_link_count) { + struct pci_dev *dsm = find_dsm_dev(pdev); + + if (!dsm) + return; + + /* + * The only path to init a Device Security Manager capable + * device is via connect(). + */ + if (!dsm->tsm) + return; + + probe_fn(pdev, dsm); + } +} + +void pci_tsm_unregister(struct tsm_dev *tsm_dev) +{ + struct pci_dev *pdev = NULL; + + guard(rwsem_write)(&pci_tsm_rwsem); + if (is_link_tsm(tsm_dev)) + pci_tsm_link_count--; + if (is_devsec_tsm(tsm_dev)) + pci_tsm_devsec_count--; + for_each_pci_dev_reverse(pdev) + __pci_tsm_destroy(pdev, tsm_dev); +} + +int pci_tsm_doe_transfer(struct pci_dev *pdev, u8 type, const void *req, + size_t req_sz, void *resp, size_t resp_sz) +{ + struct pci_tsm_pf0 *tsm; + + if (!pdev->tsm || !is_pci_tsm_pf0(pdev)) + return -ENXIO; + + tsm = to_pci_tsm_pf0(pdev->tsm); + if (!tsm->doe_mb) + return -ENXIO; + + return pci_doe(tsm->doe_mb, PCI_VENDOR_ID_PCI_SIG, type, req, req_sz, + resp, resp_sz); +} +EXPORT_SYMBOL_GPL(pci_tsm_doe_transfer); diff --git a/drivers/pinctrl/nuvoton/pinctrl-ma35.c b/drivers/pinctrl/nuvoton/pinctrl-ma35.c index cdad01d68a37..8d71dc53cc1d 100644 --- a/drivers/pinctrl/nuvoton/pinctrl-ma35.c +++ b/drivers/pinctrl/nuvoton/pinctrl-ma35.c @@ -81,10 +81,6 @@ #define MVOLT_1800 0 #define MVOLT_3300 1 -/* Non-constant mask variant of FIELD_GET() and FIELD_PREP() */ -#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) -#define field_prep(_mask, _val) (((_val) << (ffs(_mask) - 1)) & (_mask)) - static const char * const gpio_group_name[] = { "gpioa", "gpiob", "gpioc", "gpiod", "gpioe", "gpiof", "gpiog", "gpioh", "gpioi", "gpioj", "gpiok", "gpiol", "gpiom", "gpion", diff --git a/drivers/remoteproc/imx_dsp_rproc.c b/drivers/remoteproc/imx_dsp_rproc.c index 6e78a01755c7..5130a35214c9 100644 --- a/drivers/remoteproc/imx_dsp_rproc.c +++ b/drivers/remoteproc/imx_dsp_rproc.c @@ -261,56 +261,6 @@ static int imx8ulp_dsp_reset(struct imx_dsp_rproc *priv) return 0; } -/* Specific configuration for i.MX8MP */ -static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8mp = { - .att = imx_dsp_rproc_att_imx8mp, - .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8mp), - .method = IMX_RPROC_RESET_CONTROLLER, -}; - -static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8mp = { - .dcfg = &dsp_rproc_cfg_imx8mp, - .reset = imx8mp_dsp_reset, -}; - -/* Specific configuration for i.MX8ULP */ -static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8ulp = { - .src_reg = IMX8ULP_SIM_LPAV_REG_SYSCTRL0, - .src_mask = IMX8ULP_SYSCTRL0_DSP_STALL, - .src_start = 0, - .src_stop = IMX8ULP_SYSCTRL0_DSP_STALL, - .att = imx_dsp_rproc_att_imx8ulp, - .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8ulp), - .method = IMX_RPROC_MMIO, -}; - -static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8ulp = { - .dcfg = &dsp_rproc_cfg_imx8ulp, - .reset = imx8ulp_dsp_reset, -}; - -/* Specific configuration for i.MX8QXP */ -static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qxp = { - .att = imx_dsp_rproc_att_imx8qxp, - .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qxp), - .method = IMX_RPROC_SCU_API, -}; - -static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qxp = { - .dcfg = &dsp_rproc_cfg_imx8qxp, -}; - -/* Specific configuration for i.MX8QM */ -static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qm = { - .att = imx_dsp_rproc_att_imx8qm, - .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qm), - .method = IMX_RPROC_SCU_API, -}; - -static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qm = { - .dcfg = &dsp_rproc_cfg_imx8qm, -}; - static int imx_dsp_rproc_ready(struct rproc *rproc) { struct imx_dsp_rproc *priv = rproc->priv; @@ -388,6 +338,28 @@ static int imx_dsp_rproc_handle_rsc(struct rproc *rproc, u32 rsc_type, return RSC_HANDLED; } +static int imx_dsp_rproc_mmio_start(struct rproc *rproc) +{ + struct imx_dsp_rproc *priv = rproc->priv; + const struct imx_rproc_dcfg *dcfg = priv->dsp_dcfg->dcfg; + + return regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask, dcfg->src_start); +} + +static int imx_dsp_rproc_reset_ctrl_start(struct rproc *rproc) +{ + struct imx_dsp_rproc *priv = rproc->priv; + + return reset_control_deassert(priv->run_stall); +} + +static int imx_dsp_rproc_scu_api_start(struct rproc *rproc) +{ + struct imx_dsp_rproc *priv = rproc->priv; + + return imx_sc_pm_cpu_start(priv->ipc_handle, IMX_SC_R_DSP, true, rproc->bootaddr); +} + /* * Start function for rproc_ops * @@ -404,32 +376,41 @@ static int imx_dsp_rproc_start(struct rproc *rproc) struct device *dev = rproc->dev.parent; int ret; - switch (dcfg->method) { - case IMX_RPROC_MMIO: - ret = regmap_update_bits(priv->regmap, - dcfg->src_reg, - dcfg->src_mask, - dcfg->src_start); - break; - case IMX_RPROC_SCU_API: - ret = imx_sc_pm_cpu_start(priv->ipc_handle, - IMX_SC_R_DSP, - true, - rproc->bootaddr); - break; - case IMX_RPROC_RESET_CONTROLLER: - ret = reset_control_deassert(priv->run_stall); - break; - default: + if (!dcfg->ops || !dcfg->ops->start) return -EOPNOTSUPP; - } - if (ret) + ret = dcfg->ops->start(rproc); + if (ret) { dev_err(dev, "Failed to enable remote core!\n"); - else if (priv->flags & WAIT_FW_READY) + return ret; + } + + if (priv->flags & WAIT_FW_READY) return imx_dsp_rproc_ready(rproc); - return ret; + return 0; +} + +static int imx_dsp_rproc_mmio_stop(struct rproc *rproc) +{ + struct imx_dsp_rproc *priv = rproc->priv; + const struct imx_rproc_dcfg *dcfg = priv->dsp_dcfg->dcfg; + + return regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask, dcfg->src_stop); +} + +static int imx_dsp_rproc_reset_ctrl_stop(struct rproc *rproc) +{ + struct imx_dsp_rproc *priv = rproc->priv; + + return reset_control_assert(priv->run_stall); +} + +static int imx_dsp_rproc_scu_api_stop(struct rproc *rproc) +{ + struct imx_dsp_rproc *priv = rproc->priv; + + return imx_sc_pm_cpu_start(priv->ipc_handle, IMX_SC_R_DSP, false, rproc->bootaddr); } /* @@ -449,30 +430,18 @@ static int imx_dsp_rproc_stop(struct rproc *rproc) return 0; } - switch (dcfg->method) { - case IMX_RPROC_MMIO: - ret = regmap_update_bits(priv->regmap, dcfg->src_reg, dcfg->src_mask, - dcfg->src_stop); - break; - case IMX_RPROC_SCU_API: - ret = imx_sc_pm_cpu_start(priv->ipc_handle, - IMX_SC_R_DSP, - false, - rproc->bootaddr); - break; - case IMX_RPROC_RESET_CONTROLLER: - ret = reset_control_assert(priv->run_stall); - break; - default: + if (!dcfg->ops || !dcfg->ops->stop) return -EOPNOTSUPP; - } - if (ret) + ret = dcfg->ops->stop(rproc); + if (ret) { dev_err(dev, "Failed to stop remote core\n"); - else - priv->flags &= ~REMOTE_IS_READY; + return ret; + } - return ret; + priv->flags &= ~REMOTE_IS_READY; + + return 0; } /** @@ -689,11 +658,9 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) struct rproc *rproc = priv->rproc; struct device *dev = rproc->dev.parent; struct device_node *np = dev->of_node; - struct of_phandle_iterator it; struct rproc_mem_entry *mem; - struct reserved_mem *rmem; void __iomem *cpu_addr; - int a; + int a, i = 0; u64 da; /* Remap required addresses */ @@ -724,49 +691,40 @@ static int imx_dsp_rproc_add_carveout(struct imx_dsp_rproc *priv) rproc_add_carveout(rproc, mem); } - of_phandle_iterator_init(&it, np, "memory-region", NULL, 0); - while (of_phandle_iterator_next(&it) == 0) { + while (1) { + int err; + struct resource res; + + err = of_reserved_mem_region_to_resource(np, i++, &res); + if (err) + return 0; + /* * Ignore the first memory region which will be used vdev buffer. * No need to do extra handlings, rproc_add_virtio_dev will handle it. */ - if (!strcmp(it.node->name, "vdev0buffer")) + if (strstarts(res.name, "vdev0buffer")) continue; - rmem = of_reserved_mem_lookup(it.node); - if (!rmem) { - of_node_put(it.node); - dev_err(dev, "unable to acquire memory-region\n"); + if (imx_dsp_rproc_sys_to_da(priv, res.start, resource_size(&res), &da)) return -EINVAL; - } - if (imx_dsp_rproc_sys_to_da(priv, rmem->base, rmem->size, &da)) { - of_node_put(it.node); - return -EINVAL; - } - - cpu_addr = devm_ioremap_wc(dev, rmem->base, rmem->size); - if (!cpu_addr) { - of_node_put(it.node); - dev_err(dev, "failed to map memory %p\n", &rmem->base); - return -ENOMEM; + cpu_addr = devm_ioremap_resource_wc(dev, &res); + if (IS_ERR(cpu_addr)) { + dev_err(dev, "failed to map memory %pR\n", &res); + return PTR_ERR(cpu_addr); } /* Register memory region */ - mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)rmem->base, - rmem->size, da, NULL, NULL, it.node->name); - - if (mem) { - rproc_coredump_add_segment(rproc, da, rmem->size); - } else { - of_node_put(it.node); + mem = rproc_mem_entry_init(dev, (void __force *)cpu_addr, (dma_addr_t)res.start, + resource_size(&res), da, NULL, NULL, + "%.*s", strchrnul(res.name, '@') - res.name, res.name); + if (!mem) return -ENOMEM; - } + rproc_coredump_add_segment(rproc, da, resource_size(&res)); rproc_add_carveout(rproc, mem); } - - return 0; } /* Prepare function for rproc_ops */ @@ -784,7 +742,7 @@ static int imx_dsp_rproc_prepare(struct rproc *rproc) pm_runtime_get_sync(dev); - return 0; + return 0; } /* Unprepare function for rproc_ops */ @@ -792,7 +750,7 @@ static int imx_dsp_rproc_unprepare(struct rproc *rproc) { pm_runtime_put_sync(rproc->dev.parent); - return 0; + return 0; } /* Kick function for rproc_ops */ @@ -1062,14 +1020,50 @@ static const struct rproc_ops imx_dsp_rproc_ops = { static int imx_dsp_attach_pm_domains(struct imx_dsp_rproc *priv) { struct device *dev = priv->rproc->dev.parent; - int ret; /* A single PM domain is already attached. */ if (dev->pm_domain) return 0; - ret = dev_pm_domain_attach_list(dev, NULL, &priv->pd_list); - return ret < 0 ? ret : 0; + return devm_pm_domain_attach_list(dev, NULL, &priv->pd_list); +} + +static int imx_dsp_rproc_mmio_detect_mode(struct rproc *rproc) +{ + struct imx_dsp_rproc *priv = rproc->priv; + struct device *dev = rproc->dev.parent; + struct regmap *regmap; + + regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "fsl,dsp-ctrl"); + if (IS_ERR(regmap)) { + dev_err(dev, "failed to find syscon\n"); + return PTR_ERR(regmap); + } + + priv->regmap = regmap; + + return 0; +} + +static int imx_dsp_rproc_reset_ctrl_detect_mode(struct rproc *rproc) +{ + struct imx_dsp_rproc *priv = rproc->priv; + struct device *dev = rproc->dev.parent; + + priv->run_stall = devm_reset_control_get_exclusive(dev, "runstall"); + if (IS_ERR(priv->run_stall)) { + dev_err(dev, "Failed to get DSP runstall reset control\n"); + return PTR_ERR(priv->run_stall); + } + + return 0; +} + +static int imx_dsp_rproc_scu_api_detect_mode(struct rproc *rproc) +{ + struct imx_dsp_rproc *priv = rproc->priv; + + return imx_scu_get_handle(&priv->ipc_handle); } /** @@ -1087,38 +1081,12 @@ static int imx_dsp_attach_pm_domains(struct imx_dsp_rproc *priv) static int imx_dsp_rproc_detect_mode(struct imx_dsp_rproc *priv) { const struct imx_dsp_rproc_dcfg *dsp_dcfg = priv->dsp_dcfg; - struct device *dev = priv->rproc->dev.parent; - struct regmap *regmap; - int ret = 0; - - switch (dsp_dcfg->dcfg->method) { - case IMX_RPROC_SCU_API: - ret = imx_scu_get_handle(&priv->ipc_handle); - if (ret) - return ret; - break; - case IMX_RPROC_MMIO: - regmap = syscon_regmap_lookup_by_phandle(dev->of_node, "fsl,dsp-ctrl"); - if (IS_ERR(regmap)) { - dev_err(dev, "failed to find syscon\n"); - return PTR_ERR(regmap); - } + const struct imx_rproc_dcfg *dcfg = dsp_dcfg->dcfg; - priv->regmap = regmap; - break; - case IMX_RPROC_RESET_CONTROLLER: - priv->run_stall = devm_reset_control_get_exclusive(dev, "runstall"); - if (IS_ERR(priv->run_stall)) { - dev_err(dev, "Failed to get DSP runstall reset control\n"); - return PTR_ERR(priv->run_stall); - } - break; - default: - ret = -EOPNOTSUPP; - break; - } + if (dcfg->ops && dcfg->ops->detect_mode) + return dcfg->ops->detect_mode(priv->rproc); - return ret; + return -EOPNOTSUPP; } static const char *imx_dsp_clks_names[DSP_RPROC_CLK_MAX] = { @@ -1152,11 +1120,8 @@ static int imx_dsp_rproc_probe(struct platform_device *pdev) return -ENODEV; ret = rproc_of_parse_firmware(dev, 0, &fw_name); - if (ret) { - dev_err(dev, "failed to parse firmware-name property, ret = %d\n", - ret); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "failed to parse firmware-name property\n"); rproc = devm_rproc_alloc(dev, "imx-dsp-rproc", &imx_dsp_rproc_ops, fw_name, sizeof(*priv)); @@ -1179,52 +1144,28 @@ static int imx_dsp_rproc_probe(struct platform_device *pdev) INIT_WORK(&priv->rproc_work, imx_dsp_rproc_vq_work); ret = imx_dsp_rproc_detect_mode(priv); - if (ret) { - dev_err(dev, "failed on imx_dsp_rproc_detect_mode\n"); - return ret; - } + if (ret) + return dev_err_probe(dev, ret, "failed on imx_dsp_rproc_detect_mode\n"); /* There are multiple power domains required by DSP on some platform */ ret = imx_dsp_attach_pm_domains(priv); - if (ret) { - dev_err(dev, "failed on imx_dsp_attach_pm_domains\n"); - return ret; - } + if (ret < 0) + return dev_err_probe(dev, ret, "failed on imx_dsp_attach_pm_domains\n"); + /* Get clocks */ ret = imx_dsp_rproc_clk_get(priv); - if (ret) { - dev_err(dev, "failed on imx_dsp_rproc_clk_get\n"); - goto err_detach_domains; - } + if (ret) + return dev_err_probe(dev, ret, "failed on imx_dsp_rproc_clk_get\n"); init_completion(&priv->pm_comp); rproc->auto_boot = false; - ret = rproc_add(rproc); - if (ret) { - dev_err(dev, "rproc_add failed\n"); - goto err_detach_domains; - } + ret = devm_rproc_add(dev, rproc); + if (ret) + return dev_err_probe(dev, ret, "rproc_add failed\n"); rproc_coredump_set_elf_info(rproc, ELFCLASS32, EM_XTENSA); - pm_runtime_enable(dev); - - return 0; - -err_detach_domains: - dev_pm_domain_detach_list(priv->pd_list); - - return ret; -} - -static void imx_dsp_rproc_remove(struct platform_device *pdev) -{ - struct rproc *rproc = platform_get_drvdata(pdev); - struct imx_dsp_rproc *priv = rproc->priv; - - pm_runtime_disable(&pdev->dev); - rproc_del(rproc); - dev_pm_domain_detach_list(priv->pd_list); + return devm_pm_runtime_enable(dev); } /* pm runtime functions */ @@ -1364,6 +1305,74 @@ static const struct dev_pm_ops imx_dsp_rproc_pm_ops = { RUNTIME_PM_OPS(imx_dsp_runtime_suspend, imx_dsp_runtime_resume, NULL) }; +static const struct imx_rproc_plat_ops imx_dsp_rproc_ops_mmio = { + .start = imx_dsp_rproc_mmio_start, + .stop = imx_dsp_rproc_mmio_stop, + .detect_mode = imx_dsp_rproc_mmio_detect_mode, +}; + +static const struct imx_rproc_plat_ops imx_dsp_rproc_ops_reset_ctrl = { + .start = imx_dsp_rproc_reset_ctrl_start, + .stop = imx_dsp_rproc_reset_ctrl_stop, + .detect_mode = imx_dsp_rproc_reset_ctrl_detect_mode, +}; + +static const struct imx_rproc_plat_ops imx_dsp_rproc_ops_scu_api = { + .start = imx_dsp_rproc_scu_api_start, + .stop = imx_dsp_rproc_scu_api_stop, + .detect_mode = imx_dsp_rproc_scu_api_detect_mode, +}; + +/* Specific configuration for i.MX8MP */ +static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8mp = { + .att = imx_dsp_rproc_att_imx8mp, + .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8mp), + .ops = &imx_dsp_rproc_ops_reset_ctrl, +}; + +static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8mp = { + .dcfg = &dsp_rproc_cfg_imx8mp, + .reset = imx8mp_dsp_reset, +}; + +/* Specific configuration for i.MX8ULP */ +static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8ulp = { + .src_reg = IMX8ULP_SIM_LPAV_REG_SYSCTRL0, + .src_mask = IMX8ULP_SYSCTRL0_DSP_STALL, + .src_start = 0, + .src_stop = IMX8ULP_SYSCTRL0_DSP_STALL, + .att = imx_dsp_rproc_att_imx8ulp, + .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8ulp), + .ops = &imx_dsp_rproc_ops_mmio, +}; + +static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8ulp = { + .dcfg = &dsp_rproc_cfg_imx8ulp, + .reset = imx8ulp_dsp_reset, +}; + +/* Specific configuration for i.MX8QXP */ +static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qxp = { + .att = imx_dsp_rproc_att_imx8qxp, + .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qxp), + .ops = &imx_dsp_rproc_ops_scu_api, +}; + +static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qxp = { + .dcfg = &dsp_rproc_cfg_imx8qxp, +}; + +/* Specific configuration for i.MX8QM */ +static const struct imx_rproc_dcfg dsp_rproc_cfg_imx8qm = { + .att = imx_dsp_rproc_att_imx8qm, + .att_size = ARRAY_SIZE(imx_dsp_rproc_att_imx8qm), + .ops = &imx_dsp_rproc_ops_scu_api, +}; + +static const struct imx_dsp_rproc_dcfg imx_dsp_rproc_cfg_imx8qm = { + .dcfg = &dsp_rproc_cfg_imx8qm, +}; + static const struct of_device_id imx_dsp_rproc_of_match[] = { { .compatible = "fsl,imx8qxp-hifi4", .data = &imx_dsp_rproc_cfg_imx8qxp }, { .compatible = "fsl,imx8qm-hifi4", .data = &imx_dsp_rproc_cfg_imx8qm }, @@ -1375,7 +1384,6 @@ MODULE_DEVICE_TABLE(of, imx_dsp_rproc_of_match); static struct platform_driver imx_dsp_rproc_driver = { .probe = imx_dsp_rproc_probe, - .remove = imx_dsp_rproc_remove, .driver = { .name = "imx-dsp-rproc", .of_match_table = imx_dsp_rproc_of_match, diff --git a/drivers/remoteproc/imx_rproc.c b/drivers/remoteproc/imx_rproc.c index bb25221a4a89..3be8790c14a2 100644 --- a/drivers/remoteproc/imx_rproc.c +++ b/drivers/remoteproc/imx_rproc.c @@ -93,7 +93,7 @@ struct imx_rproc_mem { #define ATT_CORE(I) BIT((I)) static int imx_rproc_xtr_mbox_init(struct rproc *rproc, bool tx_block); -static void imx_rproc_free_mbox(struct rproc *rproc); +static void imx_rproc_free_mbox(void *data); struct imx_rproc { struct device *dev; @@ -490,50 +490,44 @@ static int imx_rproc_prepare(struct rproc *rproc) { struct imx_rproc *priv = rproc->priv; struct device_node *np = priv->dev->of_node; - struct of_phandle_iterator it; struct rproc_mem_entry *mem; - struct reserved_mem *rmem; + int i = 0; u32 da; /* Register associated reserved memory regions */ - of_phandle_iterator_init(&it, np, "memory-region", NULL, 0); - while (of_phandle_iterator_next(&it) == 0) { + while (1) { + int err; + struct resource res; + + err = of_reserved_mem_region_to_resource(np, i++, &res); + if (err) + return 0; + /* * Ignore the first memory region which will be used vdev buffer. * No need to do extra handlings, rproc_add_virtio_dev will handle it. */ - if (!strcmp(it.node->name, "vdev0buffer")) + if (strstarts(res.name, "vdev0buffer")) continue; - if (!strcmp(it.node->name, "rsc-table")) + if (strstarts(res.name, "rsc-table")) continue; - rmem = of_reserved_mem_lookup(it.node); - if (!rmem) { - of_node_put(it.node); - dev_err(priv->dev, "unable to acquire memory-region\n"); - return -EINVAL; - } - /* No need to translate pa to da, i.MX use same map */ - da = rmem->base; + da = res.start; /* Register memory region */ - mem = rproc_mem_entry_init(priv->dev, NULL, (dma_addr_t)rmem->base, rmem->size, da, + mem = rproc_mem_entry_init(priv->dev, NULL, (dma_addr_t)res.start, + resource_size(&res), da, imx_rproc_mem_alloc, imx_rproc_mem_release, - it.node->name); - - if (mem) { - rproc_coredump_add_segment(rproc, da, rmem->size); - } else { - of_node_put(it.node); + "%.*s", strchrnul(res.name, '@') - res.name, + res.name); + if (!mem) return -ENOMEM; - } + rproc_coredump_add_segment(rproc, da, resource_size(&res)); rproc_add_carveout(rproc, mem); } - - return 0; } static int imx_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw) @@ -575,13 +569,9 @@ static int imx_rproc_attach(struct rproc *rproc) return imx_rproc_xtr_mbox_init(rproc, true); } -static int imx_rproc_detach(struct rproc *rproc) +static int imx_rproc_scu_api_detach(struct rproc *rproc) { struct imx_rproc *priv = rproc->priv; - const struct imx_rproc_dcfg *dcfg = priv->dcfg; - - if (dcfg->method != IMX_RPROC_SCU_API) - return -EOPNOTSUPP; if (imx_sc_rm_is_resource_owned(priv->ipc_handle, priv->rsrc_id)) return -EOPNOTSUPP; @@ -591,6 +581,17 @@ static int imx_rproc_detach(struct rproc *rproc) return 0; } +static int imx_rproc_detach(struct rproc *rproc) +{ + struct imx_rproc *priv = rproc->priv; + const struct imx_rproc_dcfg *dcfg = priv->dcfg; + + if (!dcfg->ops || !dcfg->ops->detach) + return -EOPNOTSUPP; + + return dcfg->ops->detach(rproc); +} + static struct resource_table *imx_rproc_get_loaded_rsc_table(struct rproc *rproc, size_t *table_sz) { struct imx_rproc *priv = rproc->priv; @@ -664,47 +665,37 @@ static int imx_rproc_addr_init(struct imx_rproc *priv, } /* memory-region is optional property */ - nph = of_count_phandle_with_args(np, "memory-region", NULL); + nph = of_reserved_mem_region_count(np); if (nph <= 0) return 0; /* remap optional addresses */ for (a = 0; a < nph; a++) { - struct device_node *node; struct resource res; - node = of_parse_phandle(np, "memory-region", a); - if (!node) - continue; - /* Not map vdevbuffer, vdevring region */ - if (!strncmp(node->name, "vdev", strlen("vdev"))) { - of_node_put(node); - continue; - } - err = of_address_to_resource(node, 0, &res); + err = of_reserved_mem_region_to_resource(np, a, &res); if (err) { dev_err(dev, "unable to resolve memory region\n"); - of_node_put(node); return err; } - if (b >= IMX_RPROC_MEM_MAX) { - of_node_put(node); + /* Not map vdevbuffer, vdevring region */ + if (strstarts(res.name, "vdev")) + continue; + + if (b >= IMX_RPROC_MEM_MAX) break; - } /* Not use resource version, because we might share region */ - priv->mem[b].cpu_addr = devm_ioremap_wc(&pdev->dev, res.start, resource_size(&res)); + priv->mem[b].cpu_addr = devm_ioremap_resource_wc(&pdev->dev, &res); if (!priv->mem[b].cpu_addr) { dev_err(dev, "failed to remap %pr\n", &res); - of_node_put(node); return -ENOMEM; } priv->mem[b].sys_addr = res.start; priv->mem[b].size = resource_size(&res); - if (!strcmp(node->name, "rsc-table")) + if (!strcmp(res.name, "rsc-table")) priv->rsc_table = priv->mem[b].cpu_addr; - of_node_put(node); b++; } @@ -780,8 +771,9 @@ static int imx_rproc_xtr_mbox_init(struct rproc *rproc, bool tx_block) return 0; } -static void imx_rproc_free_mbox(struct rproc *rproc) +static void imx_rproc_free_mbox(void *data) { + struct rproc *rproc = data; struct imx_rproc *priv = rproc->priv; if (priv->tx_ch) { @@ -795,13 +787,9 @@ static void imx_rproc_free_mbox(struct rproc *rproc) } } -static void imx_rproc_put_scu(struct rproc *rproc) +static void imx_rproc_put_scu(void *data) { - struct imx_rproc *priv = rproc->priv; - const struct imx_rproc_dcfg *dcfg = priv->dcfg; - - if (dcfg->method != IMX_RPROC_SCU_API) - return; + struct imx_rproc *priv = data; if (imx_sc_rm_is_resource_owned(priv->ipc_handle, priv->rsrc_id)) { dev_pm_domain_detach_list(priv->pd_list); @@ -943,6 +931,10 @@ static int imx_rproc_scu_api_detect_mode(struct rproc *rproc) else priv->core_index = 0; + ret = devm_add_action_or_reset(dev, imx_rproc_put_scu, priv); + if (ret) + return dev_err_probe(dev, ret, "Failed to add action for put scu\n"); + /* * If Mcore resource is not owned by Acore partition, It is kicked by ROM, * and Linux could only do IPC with Mcore and nothing else. @@ -1001,35 +993,6 @@ static int imx_rproc_detect_mode(struct imx_rproc *priv) return dcfg->ops->detect_mode(priv->rproc); } -static int imx_rproc_clk_enable(struct imx_rproc *priv) -{ - const struct imx_rproc_dcfg *dcfg = priv->dcfg; - struct device *dev = priv->dev; - int ret; - - /* Remote core is not under control of Linux or it is managed by SCU API */ - if (dcfg->method == IMX_RPROC_NONE || dcfg->method == IMX_RPROC_SCU_API) - return 0; - - priv->clk = devm_clk_get(dev, NULL); - if (IS_ERR(priv->clk)) { - dev_err(dev, "Failed to get clock\n"); - return PTR_ERR(priv->clk); - } - - /* - * clk for M4 block including memory. Should be - * enabled before .start for FW transfer. - */ - ret = clk_prepare_enable(priv->clk); - if (ret) { - dev_err(dev, "Failed to enable clock\n"); - return ret; - } - - return 0; -} - static int imx_rproc_sys_off_handler(struct sys_off_data *data) { struct rproc *rproc = data->cb_data; @@ -1046,6 +1009,13 @@ static int imx_rproc_sys_off_handler(struct sys_off_data *data) return NOTIFY_DONE; } +static void imx_rproc_destroy_workqueue(void *data) +{ + struct workqueue_struct *workqueue = data; + + destroy_workqueue(workqueue); +} + static int imx_rproc_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; @@ -1077,25 +1047,38 @@ static int imx_rproc_probe(struct platform_device *pdev) return -ENOMEM; } + ret = devm_add_action_or_reset(dev, imx_rproc_destroy_workqueue, priv->workqueue); + if (ret) + return dev_err_probe(dev, ret, "Failed to add devm destroy workqueue action\n"); + INIT_WORK(&priv->rproc_work, imx_rproc_vq_work); ret = imx_rproc_xtr_mbox_init(rproc, true); if (ret) - goto err_put_wkq; + return ret; + + ret = devm_add_action_or_reset(dev, imx_rproc_free_mbox, rproc); + if (ret) + return dev_err_probe(dev, ret, + "Failed to add devm free mbox action: %d\n", ret); ret = imx_rproc_addr_init(priv, pdev); - if (ret) { - dev_err(dev, "failed on imx_rproc_addr_init\n"); - goto err_put_mbox; - } + if (ret) + return dev_err_probe(dev, ret, "failed on imx_rproc_addr_init\n"); ret = imx_rproc_detect_mode(priv); if (ret) - goto err_put_mbox; + return dev_err_probe(dev, ret, "failed on detect mode\n"); - ret = imx_rproc_clk_enable(priv); - if (ret) - goto err_put_scu; + /* + * Handle clocks when remote core is under control of Linux AND the + * clocks are not managed by system firmware. + */ + if (dcfg->flags & IMX_RPROC_NEED_CLKS) { + priv->clk = devm_clk_get_enabled(dev, NULL); + if (IS_ERR(priv->clk)) + return dev_err_probe(dev, PTR_ERR(priv->clk), "Failed to enable clock\n"); + } if (rproc->state != RPROC_DETACHED) rproc->auto_boot = of_property_read_bool(np, "fsl,auto-boot"); @@ -1110,45 +1093,32 @@ static int imx_rproc_probe(struct platform_device *pdev) ret = devm_register_sys_off_handler(dev, SYS_OFF_MODE_POWER_OFF_PREPARE, SYS_OFF_PRIO_DEFAULT, imx_rproc_sys_off_handler, rproc); - if (ret) { - dev_err(dev, "register power off handler failure\n"); - goto err_put_clk; - } + if (ret) + return dev_err_probe(dev, ret, "register power off handler failure\n"); ret = devm_register_sys_off_handler(dev, SYS_OFF_MODE_RESTART_PREPARE, SYS_OFF_PRIO_DEFAULT, imx_rproc_sys_off_handler, rproc); - if (ret) { - dev_err(dev, "register restart handler failure\n"); - goto err_put_clk; - } + if (ret) + return dev_err_probe(dev, ret, "register restart handler failure\n"); } - if (dcfg->method == IMX_RPROC_SCU_API) { - pm_runtime_enable(dev); - ret = pm_runtime_resume_and_get(dev); - if (ret) { - dev_err(dev, "pm_runtime get failed: %d\n", ret); - goto err_put_clk; - } - } + pm_runtime_enable(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret) + return dev_err_probe(dev, ret, "pm_runtime get failed\n"); - ret = rproc_add(rproc); + ret = devm_rproc_add(dev, rproc); if (ret) { dev_err(dev, "rproc_add failed\n"); - goto err_put_clk; + goto err_put_pm; } return 0; -err_put_clk: - clk_disable_unprepare(priv->clk); -err_put_scu: - imx_rproc_put_scu(rproc); -err_put_mbox: - imx_rproc_free_mbox(rproc); -err_put_wkq: - destroy_workqueue(priv->workqueue); +err_put_pm: + pm_runtime_disable(dev); + pm_runtime_put_noidle(dev); return ret; } @@ -1158,15 +1128,8 @@ static void imx_rproc_remove(struct platform_device *pdev) struct rproc *rproc = platform_get_drvdata(pdev); struct imx_rproc *priv = rproc->priv; - if (priv->dcfg->method == IMX_RPROC_SCU_API) { - pm_runtime_disable(priv->dev); - pm_runtime_put(priv->dev); - } - clk_disable_unprepare(priv->clk); - rproc_del(rproc); - imx_rproc_put_scu(rproc); - imx_rproc_free_mbox(rproc); - destroy_workqueue(priv->workqueue); + pm_runtime_disable(priv->dev); + pm_runtime_put_noidle(priv->dev); } static const struct imx_rproc_plat_ops imx_rproc_ops_arm_smc = { @@ -1184,6 +1147,7 @@ static const struct imx_rproc_plat_ops imx_rproc_ops_mmio = { static const struct imx_rproc_plat_ops imx_rproc_ops_scu_api = { .start = imx_rproc_scu_api_start, .stop = imx_rproc_scu_api_stop, + .detach = imx_rproc_scu_api_detach, .detect_mode = imx_rproc_scu_api_detect_mode, }; @@ -1196,15 +1160,15 @@ static const struct imx_rproc_dcfg imx_rproc_cfg_imx8mn_mmio = { .gpr_wait = IMX8M_GPR22_CM7_CPUWAIT, .att = imx_rproc_att_imx8mn, .att_size = ARRAY_SIZE(imx_rproc_att_imx8mn), - .method = IMX_RPROC_MMIO, .ops = &imx_rproc_ops_mmio, + .flags = IMX_RPROC_NEED_CLKS, }; static const struct imx_rproc_dcfg imx_rproc_cfg_imx8mn = { .att = imx_rproc_att_imx8mn, .att_size = ARRAY_SIZE(imx_rproc_att_imx8mn), - .method = IMX_RPROC_SMC, .ops = &imx_rproc_ops_arm_smc, + .flags = IMX_RPROC_NEED_CLKS, }; static const struct imx_rproc_dcfg imx_rproc_cfg_imx8mq = { @@ -1214,34 +1178,30 @@ static const struct imx_rproc_dcfg imx_rproc_cfg_imx8mq = { .src_stop = IMX7D_M4_STOP, .att = imx_rproc_att_imx8mq, .att_size = ARRAY_SIZE(imx_rproc_att_imx8mq), - .method = IMX_RPROC_MMIO, .ops = &imx_rproc_ops_mmio, + .flags = IMX_RPROC_NEED_CLKS, }; static const struct imx_rproc_dcfg imx_rproc_cfg_imx8qm = { .att = imx_rproc_att_imx8qm, .att_size = ARRAY_SIZE(imx_rproc_att_imx8qm), - .method = IMX_RPROC_SCU_API, .ops = &imx_rproc_ops_scu_api, }; static const struct imx_rproc_dcfg imx_rproc_cfg_imx8qxp = { .att = imx_rproc_att_imx8qxp, .att_size = ARRAY_SIZE(imx_rproc_att_imx8qxp), - .method = IMX_RPROC_SCU_API, .ops = &imx_rproc_ops_scu_api, }; static const struct imx_rproc_dcfg imx_rproc_cfg_imx8ulp = { .att = imx_rproc_att_imx8ulp, .att_size = ARRAY_SIZE(imx_rproc_att_imx8ulp), - .method = IMX_RPROC_NONE, }; static const struct imx_rproc_dcfg imx_rproc_cfg_imx7ulp = { .att = imx_rproc_att_imx7ulp, .att_size = ARRAY_SIZE(imx_rproc_att_imx7ulp), - .method = IMX_RPROC_NONE, .flags = IMX_RPROC_NEED_SYSTEM_OFF, }; @@ -1252,8 +1212,8 @@ static const struct imx_rproc_dcfg imx_rproc_cfg_imx7d = { .src_stop = IMX7D_M4_STOP, .att = imx_rproc_att_imx7d, .att_size = ARRAY_SIZE(imx_rproc_att_imx7d), - .method = IMX_RPROC_MMIO, .ops = &imx_rproc_ops_mmio, + .flags = IMX_RPROC_NEED_CLKS, }; static const struct imx_rproc_dcfg imx_rproc_cfg_imx6sx = { @@ -1263,15 +1223,15 @@ static const struct imx_rproc_dcfg imx_rproc_cfg_imx6sx = { .src_stop = IMX6SX_M4_STOP, .att = imx_rproc_att_imx6sx, .att_size = ARRAY_SIZE(imx_rproc_att_imx6sx), - .method = IMX_RPROC_MMIO, .ops = &imx_rproc_ops_mmio, + .flags = IMX_RPROC_NEED_CLKS, }; static const struct imx_rproc_dcfg imx_rproc_cfg_imx93 = { .att = imx_rproc_att_imx93, .att_size = ARRAY_SIZE(imx_rproc_att_imx93), - .method = IMX_RPROC_SMC, .ops = &imx_rproc_ops_arm_smc, + .flags = IMX_RPROC_NEED_CLKS, }; static const struct of_device_id imx_rproc_of_match[] = { diff --git a/drivers/remoteproc/imx_rproc.h b/drivers/remoteproc/imx_rproc.h index 3a9adaaf048b..1b2d9f4d6d19 100644 --- a/drivers/remoteproc/imx_rproc.h +++ b/drivers/remoteproc/imx_rproc.h @@ -15,25 +15,14 @@ struct imx_rproc_att { int flags; }; -/* Remote core start/stop method */ -enum imx_rproc_method { - IMX_RPROC_NONE, - /* Through syscon regmap */ - IMX_RPROC_MMIO, - /* Through ARM SMCCC */ - IMX_RPROC_SMC, - /* Through System Control Unit API */ - IMX_RPROC_SCU_API, - /* Through Reset Controller API */ - IMX_RPROC_RESET_CONTROLLER, -}; - /* dcfg flags */ #define IMX_RPROC_NEED_SYSTEM_OFF BIT(0) +#define IMX_RPROC_NEED_CLKS BIT(1) struct imx_rproc_plat_ops { int (*start)(struct rproc *rproc); int (*stop)(struct rproc *rproc); + int (*detach)(struct rproc *rproc); int (*detect_mode)(struct rproc *rproc); }; @@ -46,7 +35,6 @@ struct imx_rproc_dcfg { u32 gpr_wait; const struct imx_rproc_att *att; size_t att_size; - enum imx_rproc_method method; u32 flags; const struct imx_rproc_plat_ops *ops; }; diff --git a/drivers/remoteproc/mtk_scp.c b/drivers/remoteproc/mtk_scp.c index 8206a1766481..db8fd045468d 100644 --- a/drivers/remoteproc/mtk_scp.c +++ b/drivers/remoteproc/mtk_scp.c @@ -16,6 +16,7 @@ #include <linux/remoteproc.h> #include <linux/remoteproc/mtk_scp.h> #include <linux/rpmsg/mtk_rpmsg.h> +#include <linux/string.h> #include "mtk_common.h" #include "remoteproc_internal.h" @@ -1093,22 +1094,74 @@ static void scp_remove_rpmsg_subdev(struct mtk_scp *scp) } } +/** + * scp_get_default_fw_path() - Get default SCP firmware path + * @dev: SCP Device + * @core_id: SCP Core number + * + * This function generates a path based on the following format: + * mediatek/(soc_model)/scp(_cX).img; for multi-core or + * mediatek/(soc_model)/scp.img for single core SCP HW + * + * Return: A devm allocated string containing the full path to + * a SCP firmware or an error pointer + */ +static const char *scp_get_default_fw_path(struct device *dev, int core_id) +{ + struct device_node *np = core_id < 0 ? dev->of_node : dev->parent->of_node; + const char *compatible, *soc; + char scp_fw_file[7]; + int ret; + + /* Use only the first compatible string */ + ret = of_property_read_string_index(np, "compatible", 0, &compatible); + if (ret) + return ERR_PTR(ret); + + /* If the compatible string's length is implausible bail out early */ + if (strlen(compatible) < strlen("mediatek,mtXXXX-scp")) + return ERR_PTR(-EINVAL); + + /* If the compatible string starts with "mediatek,mt" assume that it's ok */ + if (!str_has_prefix(compatible, "mediatek,mt")) + return ERR_PTR(-EINVAL); + + if (core_id >= 0) + ret = snprintf(scp_fw_file, sizeof(scp_fw_file), "scp_c%d", core_id); + else + ret = snprintf(scp_fw_file, sizeof(scp_fw_file), "scp"); + if (ret >= sizeof(scp_fw_file)) + return ERR_PTR(-ENAMETOOLONG); + + /* Not using strchr here, as strlen of a const gets optimized by compiler */ + soc = &compatible[strlen("mediatek,")]; + + return devm_kasprintf(dev, GFP_KERNEL, "mediatek/%.*s/%s.img", + (int)strlen("mtXXXX"), soc, scp_fw_file); +} + static struct mtk_scp *scp_rproc_init(struct platform_device *pdev, struct mtk_scp_of_cluster *scp_cluster, - const struct mtk_scp_of_data *of_data) + const struct mtk_scp_of_data *of_data, + int core_id) { struct device *dev = &pdev->dev; struct device_node *np = dev->of_node; struct mtk_scp *scp; struct rproc *rproc; struct resource *res; - const char *fw_name = "scp.img"; + const char *fw_name; int ret, i; const struct mtk_scp_sizes_data *scp_sizes; ret = rproc_of_parse_firmware(dev, 0, &fw_name); - if (ret < 0 && ret != -EINVAL) - return ERR_PTR(ret); + if (ret) { + fw_name = scp_get_default_fw_path(dev, core_id); + if (IS_ERR(fw_name)) { + dev_err(dev, "Cannot get firmware path: %ld\n", PTR_ERR(fw_name)); + return ERR_CAST(fw_name); + } + } rproc = devm_rproc_alloc(dev, np->name, &scp_ops, fw_name, sizeof(*scp)); if (!rproc) { @@ -1212,7 +1265,7 @@ static int scp_add_single_core(struct platform_device *pdev, struct mtk_scp *scp; int ret; - scp = scp_rproc_init(pdev, scp_cluster, of_device_get_match_data(dev)); + scp = scp_rproc_init(pdev, scp_cluster, of_device_get_match_data(dev), -1); if (IS_ERR(scp)) return PTR_ERR(scp); @@ -1259,7 +1312,7 @@ static int scp_add_multi_core(struct platform_device *pdev, goto init_fail; } - scp = scp_rproc_init(cpdev, scp_cluster, cluster_of_data[core_id]); + scp = scp_rproc_init(cpdev, scp_cluster, cluster_of_data[core_id], core_id); put_device(&cpdev->dev); if (IS_ERR(scp)) { ret = PTR_ERR(scp); diff --git a/drivers/remoteproc/omap_remoteproc.c b/drivers/remoteproc/omap_remoteproc.c index 9c9e9c3cf378..cb01354248af 100644 --- a/drivers/remoteproc/omap_remoteproc.c +++ b/drivers/remoteproc/omap_remoteproc.c @@ -555,7 +555,6 @@ static void omap_rproc_kick(struct rproc *rproc, int vqid) dev_err(dev, "failed to send mailbox message, status = %d\n", ret); - pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); } @@ -656,7 +655,6 @@ static int omap_rproc_start(struct rproc *rproc) pm_runtime_use_autosuspend(dev); pm_runtime_get_noresume(dev); pm_runtime_enable(dev); - pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); return 0; @@ -714,7 +712,6 @@ enable_device: reset_control_deassert(oproc->reset); out: /* schedule the next auto-suspend */ - pm_runtime_mark_last_busy(dev); pm_runtime_put_autosuspend(dev); return ret; } diff --git a/drivers/remoteproc/qcom_q6v5_adsp.c b/drivers/remoteproc/qcom_q6v5_adsp.c index e98b7e03162c..b5c8d6d38c9c 100644 --- a/drivers/remoteproc/qcom_q6v5_adsp.c +++ b/drivers/remoteproc/qcom_q6v5_adsp.c @@ -625,27 +625,22 @@ static int adsp_init_mmio(struct qcom_adsp *adsp, static int adsp_alloc_memory_region(struct qcom_adsp *adsp) { - struct reserved_mem *rmem = NULL; - struct device_node *node; - - node = of_parse_phandle(adsp->dev->of_node, "memory-region", 0); - if (node) - rmem = of_reserved_mem_lookup(node); - of_node_put(node); + int ret; + struct resource res; - if (!rmem) { + ret = of_reserved_mem_region_to_resource(adsp->dev->of_node, 0, &res); + if (ret) { dev_err(adsp->dev, "unable to resolve memory-region\n"); - return -EINVAL; + return ret; } - adsp->mem_phys = adsp->mem_reloc = rmem->base; - adsp->mem_size = rmem->size; - adsp->mem_region = devm_ioremap_wc(adsp->dev, - adsp->mem_phys, adsp->mem_size); - if (!adsp->mem_region) { - dev_err(adsp->dev, "unable to map memory region: %pa+%zx\n", - &rmem->base, adsp->mem_size); - return -EBUSY; + adsp->mem_phys = adsp->mem_reloc = res.start; + adsp->mem_size = resource_size(&res); + adsp->mem_region = devm_ioremap_resource_wc(adsp->dev, &res); + if (IS_ERR(adsp->mem_region)) { + dev_err(adsp->dev, "unable to map memory region: %pR\n", &res); + return PTR_ERR(adsp->mem_region); + } return 0; diff --git a/drivers/remoteproc/qcom_q6v5_mss.c b/drivers/remoteproc/qcom_q6v5_mss.c index 3087d895b87f..91940977ca89 100644 --- a/drivers/remoteproc/qcom_q6v5_mss.c +++ b/drivers/remoteproc/qcom_q6v5_mss.c @@ -1970,8 +1970,8 @@ static int q6v5_init_reset(struct q6v5 *qproc) static int q6v5_alloc_memory_region(struct q6v5 *qproc) { struct device_node *child; - struct reserved_mem *rmem; - struct device_node *node; + struct resource res; + int ret; /* * In the absence of mba/mpss sub-child, extract the mba and mpss @@ -1979,71 +1979,49 @@ static int q6v5_alloc_memory_region(struct q6v5 *qproc) */ child = of_get_child_by_name(qproc->dev->of_node, "mba"); if (!child) { - node = of_parse_phandle(qproc->dev->of_node, - "memory-region", 0); + ret = of_reserved_mem_region_to_resource(qproc->dev->of_node, 0, &res); } else { - node = of_parse_phandle(child, "memory-region", 0); + ret = of_reserved_mem_region_to_resource(child, 0, &res); of_node_put(child); } - if (!node) { - dev_err(qproc->dev, "no mba memory-region specified\n"); - return -EINVAL; - } - - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) { + if (ret) { dev_err(qproc->dev, "unable to resolve mba region\n"); - return -EINVAL; + return ret; } - qproc->mba_phys = rmem->base; - qproc->mba_size = rmem->size; + qproc->mba_phys = res.start; + qproc->mba_size = resource_size(&res); if (!child) { - node = of_parse_phandle(qproc->dev->of_node, - "memory-region", 1); + ret = of_reserved_mem_region_to_resource(qproc->dev->of_node, 1, &res); } else { child = of_get_child_by_name(qproc->dev->of_node, "mpss"); - node = of_parse_phandle(child, "memory-region", 0); + ret = of_reserved_mem_region_to_resource(child, 0, &res); of_node_put(child); } - if (!node) { - dev_err(qproc->dev, "no mpss memory-region specified\n"); - return -EINVAL; - } - - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) { + if (ret) { dev_err(qproc->dev, "unable to resolve mpss region\n"); - return -EINVAL; + return ret; } - qproc->mpss_phys = qproc->mpss_reloc = rmem->base; - qproc->mpss_size = rmem->size; + qproc->mpss_phys = qproc->mpss_reloc = res.start; + qproc->mpss_size = resource_size(&res); if (!child) { - node = of_parse_phandle(qproc->dev->of_node, "memory-region", 2); + ret = of_reserved_mem_region_to_resource(qproc->dev->of_node, 2, &res); } else { child = of_get_child_by_name(qproc->dev->of_node, "metadata"); - node = of_parse_phandle(child, "memory-region", 0); + ret = of_reserved_mem_region_to_resource(child, 0, &res); of_node_put(child); } - if (!node) + if (ret) return 0; - rmem = of_reserved_mem_lookup(node); - if (!rmem) { - dev_err(qproc->dev, "unable to resolve metadata region\n"); - return -EINVAL; - } - - qproc->mdata_phys = rmem->base; - qproc->mdata_size = rmem->size; + qproc->mdata_phys = res.start; + qproc->mdata_size = resource_size(&res); return 0; } diff --git a/drivers/remoteproc/qcom_q6v5_pas.c b/drivers/remoteproc/qcom_q6v5_pas.c index 158bcd6cc85c..52680ac99589 100644 --- a/drivers/remoteproc/qcom_q6v5_pas.c +++ b/drivers/remoteproc/qcom_q6v5_pas.c @@ -547,54 +547,38 @@ static void qcom_pas_pds_detach(struct qcom_pas *pas, struct device **pds, size_ static int qcom_pas_alloc_memory_region(struct qcom_pas *pas) { - struct reserved_mem *rmem; - struct device_node *node; - - node = of_parse_phandle(pas->dev->of_node, "memory-region", 0); - if (!node) { - dev_err(pas->dev, "no memory-region specified\n"); - return -EINVAL; - } + struct resource res; + int ret; - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) { + ret = of_reserved_mem_region_to_resource(pas->dev->of_node, 0, &res); + if (ret) { dev_err(pas->dev, "unable to resolve memory-region\n"); - return -EINVAL; + return ret; } - pas->mem_phys = pas->mem_reloc = rmem->base; - pas->mem_size = rmem->size; - pas->mem_region = devm_ioremap_wc(pas->dev, pas->mem_phys, pas->mem_size); - if (!pas->mem_region) { - dev_err(pas->dev, "unable to map memory region: %pa+%zx\n", - &rmem->base, pas->mem_size); - return -EBUSY; + pas->mem_phys = pas->mem_reloc = res.start; + pas->mem_size = resource_size(&res); + pas->mem_region = devm_ioremap_resource_wc(pas->dev, &res); + if (IS_ERR(pas->mem_region)) { + dev_err(pas->dev, "unable to map memory region: %pR\n", &res); + return PTR_ERR(pas->mem_region); } if (!pas->dtb_pas_id) return 0; - node = of_parse_phandle(pas->dev->of_node, "memory-region", 1); - if (!node) { - dev_err(pas->dev, "no dtb memory-region specified\n"); - return -EINVAL; - } - - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) { + ret = of_reserved_mem_region_to_resource(pas->dev->of_node, 1, &res); + if (ret) { dev_err(pas->dev, "unable to resolve dtb memory-region\n"); - return -EINVAL; + return ret; } - pas->dtb_mem_phys = pas->dtb_mem_reloc = rmem->base; - pas->dtb_mem_size = rmem->size; - pas->dtb_mem_region = devm_ioremap_wc(pas->dev, pas->dtb_mem_phys, pas->dtb_mem_size); - if (!pas->dtb_mem_region) { - dev_err(pas->dev, "unable to map dtb memory region: %pa+%zx\n", - &rmem->base, pas->dtb_mem_size); - return -EBUSY; + pas->dtb_mem_phys = pas->dtb_mem_reloc = res.start; + pas->dtb_mem_size = resource_size(&res); + pas->dtb_mem_region = devm_ioremap_resource_wc(pas->dev, &res); + if (IS_ERR(pas->dtb_mem_region)) { + dev_err(pas->dev, "unable to map dtb memory region: %pR\n", &res); + return PTR_ERR(pas->dtb_mem_region); } return 0; @@ -603,7 +587,6 @@ static int qcom_pas_alloc_memory_region(struct qcom_pas *pas) static int qcom_pas_assign_memory_region(struct qcom_pas *pas) { struct qcom_scm_vmperm perm[MAX_ASSIGN_COUNT]; - struct device_node *node; unsigned int perm_size; int offset; int ret; @@ -612,17 +595,15 @@ static int qcom_pas_assign_memory_region(struct qcom_pas *pas) return 0; for (offset = 0; offset < pas->region_assign_count; ++offset) { - struct reserved_mem *rmem = NULL; - - node = of_parse_phandle(pas->dev->of_node, "memory-region", - pas->region_assign_idx + offset); - if (node) - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - if (!rmem) { + struct resource res; + + ret = of_reserved_mem_region_to_resource(pas->dev->of_node, + pas->region_assign_idx + offset, + &res); + if (ret) { dev_err(pas->dev, "unable to resolve shareable memory-region index %d\n", offset); - return -EINVAL; + return ret; } if (pas->region_assign_shared) { @@ -637,8 +618,8 @@ static int qcom_pas_assign_memory_region(struct qcom_pas *pas) perm_size = 1; } - pas->region_assign_phys[offset] = rmem->base; - pas->region_assign_size[offset] = rmem->size; + pas->region_assign_phys[offset] = res.start; + pas->region_assign_size[offset] = resource_size(&res); pas->region_assign_owners[offset] = BIT(QCOM_SCM_VMID_HLOS); ret = qcom_scm_assign_mem(pas->region_assign_phys[offset], @@ -1461,7 +1442,7 @@ static const struct of_device_id qcom_pas_of_match[] = { { .compatible = "qcom,milos-wpss-pas", .data = &sc7280_wpss_resource}, { .compatible = "qcom,msm8226-adsp-pil", .data = &msm8996_adsp_resource}, { .compatible = "qcom,msm8953-adsp-pil", .data = &msm8996_adsp_resource}, - { .compatible = "qcom,msm8974-adsp-pil", .data = &adsp_resource_init}, + { .compatible = "qcom,msm8974-adsp-pil", .data = &msm8996_adsp_resource}, { .compatible = "qcom,msm8996-adsp-pil", .data = &msm8996_adsp_resource}, { .compatible = "qcom,msm8996-slpi-pil", .data = &msm8996_slpi_resource_init}, { .compatible = "qcom,msm8998-adsp-pas", .data = &msm8996_adsp_resource}, @@ -1488,6 +1469,7 @@ static const struct of_device_id qcom_pas_of_match[] = { { .compatible = "qcom,sc8280xp-nsp0-pas", .data = &sc8280xp_nsp0_resource}, { .compatible = "qcom,sc8280xp-nsp1-pas", .data = &sc8280xp_nsp1_resource}, { .compatible = "qcom,sdm660-adsp-pas", .data = &adsp_resource_init}, + { .compatible = "qcom,sdm660-cdsp-pas", .data = &cdsp_resource_init}, { .compatible = "qcom,sdm845-adsp-pas", .data = &sdm845_adsp_resource_init}, { .compatible = "qcom,sdm845-cdsp-pas", .data = &sdm845_cdsp_resource_init}, { .compatible = "qcom,sdm845-slpi-pas", .data = &sdm845_slpi_resource_init}, diff --git a/drivers/remoteproc/qcom_q6v5_wcss.c b/drivers/remoteproc/qcom_q6v5_wcss.c index 07c88623f597..c27200159a88 100644 --- a/drivers/remoteproc/qcom_q6v5_wcss.c +++ b/drivers/remoteproc/qcom_q6v5_wcss.c @@ -85,7 +85,7 @@ #define TCSR_WCSS_CLK_MASK 0x1F #define TCSR_WCSS_CLK_ENABLE 0x14 -#define MAX_HALT_REG 3 +#define MAX_HALT_REG 4 enum { WCSS_IPQ8074, WCSS_QCS404, @@ -811,7 +811,8 @@ static int q6v5_wcss_init_reset(struct q6v5_wcss *wcss, } } - wcss->wcss_q6_bcr_reset = devm_reset_control_get_exclusive(dev, "wcss_q6_bcr_reset"); + wcss->wcss_q6_bcr_reset = devm_reset_control_get_optional_exclusive(dev, + "wcss_q6_bcr_reset"); if (IS_ERR(wcss->wcss_q6_bcr_reset)) { dev_err(wcss->dev, "unable to acquire wcss_q6_bcr_reset\n"); return PTR_ERR(wcss->wcss_q6_bcr_reset); @@ -864,37 +865,32 @@ static int q6v5_wcss_init_mmio(struct q6v5_wcss *wcss, return -EINVAL; } - wcss->halt_q6 = halt_reg[0]; - wcss->halt_wcss = halt_reg[1]; - wcss->halt_nc = halt_reg[2]; + wcss->halt_q6 = halt_reg[1]; + wcss->halt_wcss = halt_reg[2]; + wcss->halt_nc = halt_reg[3]; return 0; } static int q6v5_alloc_memory_region(struct q6v5_wcss *wcss) { - struct reserved_mem *rmem = NULL; - struct device_node *node; struct device *dev = wcss->dev; + struct resource res; + int ret; - node = of_parse_phandle(dev->of_node, "memory-region", 0); - if (node) - rmem = of_reserved_mem_lookup(node); - of_node_put(node); - - if (!rmem) { + ret = of_reserved_mem_region_to_resource(dev->of_node, 0, &res); + if (ret) { dev_err(dev, "unable to acquire memory-region\n"); - return -EINVAL; + return ret; } - wcss->mem_phys = rmem->base; - wcss->mem_reloc = rmem->base; - wcss->mem_size = rmem->size; - wcss->mem_region = devm_ioremap_wc(dev, wcss->mem_phys, wcss->mem_size); - if (!wcss->mem_region) { - dev_err(dev, "unable to map memory region: %pa+%pa\n", - &rmem->base, &rmem->size); - return -EBUSY; + wcss->mem_phys = res.start; + wcss->mem_reloc = res.start; + wcss->mem_size = resource_size(&res); + wcss->mem_region = devm_ioremap_resource_wc(dev, &res); + if (IS_ERR(wcss->mem_region)) { + dev_err(dev, "unable to map memory region: %pR\n", &res); + return PTR_ERR(wcss->mem_region); } return 0; diff --git a/drivers/remoteproc/qcom_wcnss.c b/drivers/remoteproc/qcom_wcnss.c index 2c7e519a2254..ee18bf2e8054 100644 --- a/drivers/remoteproc/qcom_wcnss.c +++ b/drivers/remoteproc/qcom_wcnss.c @@ -526,26 +526,21 @@ static int wcnss_request_irq(struct qcom_wcnss *wcnss, static int wcnss_alloc_memory_region(struct qcom_wcnss *wcnss) { - struct reserved_mem *rmem = NULL; - struct device_node *node; - - node = of_parse_phandle(wcnss->dev->of_node, "memory-region", 0); - if (node) - rmem = of_reserved_mem_lookup(node); - of_node_put(node); + struct resource res; + int ret; - if (!rmem) { + ret = of_reserved_mem_region_to_resource(wcnss->dev->of_node, 0, &res); + if (ret) { dev_err(wcnss->dev, "unable to resolve memory-region\n"); - return -EINVAL; + return ret; } - wcnss->mem_phys = wcnss->mem_reloc = rmem->base; - wcnss->mem_size = rmem->size; - wcnss->mem_region = devm_ioremap_wc(wcnss->dev, wcnss->mem_phys, wcnss->mem_size); - if (!wcnss->mem_region) { - dev_err(wcnss->dev, "unable to map memory region: %pa+%zx\n", - &rmem->base, wcnss->mem_size); - return -EBUSY; + wcnss->mem_phys = wcnss->mem_reloc = res.start; + wcnss->mem_size = resource_size(&res); + wcnss->mem_region = devm_ioremap_resource_wc(wcnss->dev, &res); + if (IS_ERR(wcnss->mem_region)) { + dev_err(wcnss->dev, "unable to map memory region: %pR\n", &res); + return PTR_ERR(wcnss->mem_region); } return 0; diff --git a/drivers/remoteproc/rcar_rproc.c b/drivers/remoteproc/rcar_rproc.c index 921d853594f4..3c25625f966d 100644 --- a/drivers/remoteproc/rcar_rproc.c +++ b/drivers/remoteproc/rcar_rproc.c @@ -52,46 +52,36 @@ static int rcar_rproc_prepare(struct rproc *rproc) { struct device *dev = rproc->dev.parent; struct device_node *np = dev->of_node; - struct of_phandle_iterator it; struct rproc_mem_entry *mem; - struct reserved_mem *rmem; + int i = 0; u32 da; /* Register associated reserved memory regions */ - of_phandle_iterator_init(&it, np, "memory-region", NULL, 0); - while (of_phandle_iterator_next(&it) == 0) { - - rmem = of_reserved_mem_lookup(it.node); - if (!rmem) { - of_node_put(it.node); - dev_err(&rproc->dev, - "unable to acquire memory-region\n"); - return -EINVAL; - } + while (1) { + struct resource res; + int ret; + + ret = of_reserved_mem_region_to_resource(np, i++, &res); + if (ret) + return 0; - if (rmem->base > U32_MAX) { - of_node_put(it.node); + if (res.start > U32_MAX) return -EINVAL; - } /* No need to translate pa to da, R-Car use same map */ - da = rmem->base; + da = res.start; mem = rproc_mem_entry_init(dev, NULL, - rmem->base, - rmem->size, da, + res.start, + resource_size(&res), da, rcar_rproc_mem_alloc, rcar_rproc_mem_release, - it.node->name); + res.name); - if (!mem) { - of_node_put(it.node); + if (!mem) return -ENOMEM; - } rproc_add_carveout(rproc, mem); } - - return 0; } static int rcar_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw) diff --git a/drivers/remoteproc/remoteproc_core.c b/drivers/remoteproc/remoteproc_core.c index 825672100528..aada2780b343 100644 --- a/drivers/remoteproc/remoteproc_core.c +++ b/drivers/remoteproc/remoteproc_core.c @@ -16,29 +16,25 @@ #define pr_fmt(fmt) "%s: " fmt, __func__ +#include <asm/byteorder.h> #include <linux/delay.h> -#include <linux/kernel.h> -#include <linux/module.h> #include <linux/device.h> -#include <linux/panic_notifier.h> -#include <linux/slab.h> -#include <linux/mutex.h> #include <linux/dma-mapping.h> +#include <linux/elf.h> #include <linux/firmware.h> -#include <linux/string.h> -#include <linux/debugfs.h> -#include <linux/rculist.h> -#include <linux/remoteproc.h> -#include <linux/iommu.h> #include <linux/idr.h> -#include <linux/elf.h> -#include <linux/crc32.h> +#include <linux/iommu.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mutex.h> #include <linux/of_platform.h> -#include <linux/of_reserved_mem.h> -#include <linux/virtio_ids.h> -#include <linux/virtio_ring.h> -#include <asm/byteorder.h> +#include <linux/panic_notifier.h> #include <linux/platform_device.h> +#include <linux/rculist.h> +#include <linux/remoteproc.h> +#include <linux/slab.h> +#include <linux/string.h> +#include <linux/virtio_ring.h> #include "remoteproc_internal.h" @@ -159,7 +155,6 @@ phys_addr_t rproc_va_to_pa(void *cpu_addr) WARN_ON(!virt_addr_valid(cpu_addr)); return virt_to_phys(cpu_addr); } -EXPORT_SYMBOL(rproc_va_to_pa); /** * rproc_da_to_va() - lookup the kernel virtual address for a remoteproc address @@ -1989,7 +1984,7 @@ EXPORT_SYMBOL(rproc_boot); int rproc_shutdown(struct rproc *rproc) { struct device *dev = &rproc->dev; - int ret = 0; + int ret; ret = mutex_lock_interruptible(&rproc->lock); if (ret) { diff --git a/drivers/remoteproc/st_remoteproc.c b/drivers/remoteproc/st_remoteproc.c index e6566a9839dc..a07edf7217d2 100644 --- a/drivers/remoteproc/st_remoteproc.c +++ b/drivers/remoteproc/st_remoteproc.c @@ -120,43 +120,41 @@ static int st_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw) struct device *dev = rproc->dev.parent; struct device_node *np = dev->of_node; struct rproc_mem_entry *mem; - struct reserved_mem *rmem; - struct of_phandle_iterator it; - int index = 0; - - of_phandle_iterator_init(&it, np, "memory-region", NULL, 0); - while (of_phandle_iterator_next(&it) == 0) { - rmem = of_reserved_mem_lookup(it.node); - if (!rmem) { - of_node_put(it.node); - dev_err(dev, "unable to acquire memory-region\n"); - return -EINVAL; - } + int entries; + + entries = of_reserved_mem_region_count(np); + + for (int index = 0; index < entries; index++) { + struct resource res; + int ret; + + ret = of_reserved_mem_region_to_resource(np, index, &res); + if (ret) + return ret; /* No need to map vdev buffer */ - if (strcmp(it.node->name, "vdev0buffer")) { + if (!strstarts(res.name, "vdev0buffer")) { /* Register memory region */ mem = rproc_mem_entry_init(dev, NULL, - (dma_addr_t)rmem->base, - rmem->size, rmem->base, + (dma_addr_t)res.start, + resource_size(&res), res.start, st_rproc_mem_alloc, st_rproc_mem_release, - it.node->name); + "%.*s", + strchrnul(res.name, '@') - res.name, + res.name); } else { /* Register reserved memory for vdev buffer allocation */ mem = rproc_of_resm_mem_entry_init(dev, index, - rmem->size, - rmem->base, - it.node->name); + resource_size(&res), + res.start, + "vdev0buffer"); } - if (!mem) { - of_node_put(it.node); + if (!mem) return -ENOMEM; - } rproc_add_carveout(rproc, mem); - index++; } return rproc_elf_load_rsc_table(rproc, fw); diff --git a/drivers/remoteproc/stm32_rproc.c b/drivers/remoteproc/stm32_rproc.c index 431648607d53..c28679d3b43c 100644 --- a/drivers/remoteproc/stm32_rproc.c +++ b/drivers/remoteproc/stm32_rproc.c @@ -213,60 +213,52 @@ static int stm32_rproc_prepare(struct rproc *rproc) { struct device *dev = rproc->dev.parent; struct device_node *np = dev->of_node; - struct of_phandle_iterator it; struct rproc_mem_entry *mem; - struct reserved_mem *rmem; u64 da; - int index = 0; + int index = 0, mr = 0; /* Register associated reserved memory regions */ - of_phandle_iterator_init(&it, np, "memory-region", NULL, 0); - while (of_phandle_iterator_next(&it) == 0) { - rmem = of_reserved_mem_lookup(it.node); - if (!rmem) { - of_node_put(it.node); - dev_err(dev, "unable to acquire memory-region\n"); - return -EINVAL; - } + while (1) { + struct resource res; + int ret; + + ret = of_reserved_mem_region_to_resource(np, mr++, &res); + if (ret) + return 0; - if (stm32_rproc_pa_to_da(rproc, rmem->base, &da) < 0) { - of_node_put(it.node); - dev_err(dev, "memory region not valid %pa\n", - &rmem->base); + if (stm32_rproc_pa_to_da(rproc, res.start, &da) < 0) { + dev_err(dev, "memory region not valid %pR\n", &res); return -EINVAL; } /* No need to map vdev buffer */ - if (strcmp(it.node->name, "vdev0buffer")) { + if (!strstarts(res.name, "vdev0buffer")) { /* Register memory region */ mem = rproc_mem_entry_init(dev, NULL, - (dma_addr_t)rmem->base, - rmem->size, da, + (dma_addr_t)res.start, + resource_size(&res), da, stm32_rproc_mem_alloc, stm32_rproc_mem_release, - it.node->name); - + "%.*s", strchrnul(res.name, '@') - res.name, + res.name); if (mem) rproc_coredump_add_segment(rproc, da, - rmem->size); + resource_size(&res)); } else { /* Register reserved memory for vdev buffer alloc */ mem = rproc_of_resm_mem_entry_init(dev, index, - rmem->size, - rmem->base, - it.node->name); + resource_size(&res), + res.start, + "vdev0buffer"); } if (!mem) { - of_node_put(it.node); return -ENOMEM; } rproc_add_carveout(rproc, mem); index++; } - - return 0; } static int stm32_rproc_parse_fw(struct rproc *rproc, const struct firmware *fw) diff --git a/drivers/remoteproc/ti_k3_common.c b/drivers/remoteproc/ti_k3_common.c index 56b71652e449..32aa954dc5be 100644 --- a/drivers/remoteproc/ti_k3_common.c +++ b/drivers/remoteproc/ti_k3_common.c @@ -470,13 +470,10 @@ int k3_reserved_mem_init(struct k3_rproc *kproc) { struct device *dev = kproc->dev; struct device_node *np = dev->of_node; - struct device_node *rmem_np; - struct reserved_mem *rmem; int num_rmems; int ret, i; - num_rmems = of_property_count_elems_of_size(np, "memory-region", - sizeof(phandle)); + num_rmems = of_reserved_mem_region_count(np); if (num_rmems < 0) { dev_err(dev, "device does not reserved memory regions (%d)\n", num_rmems); @@ -505,23 +502,20 @@ int k3_reserved_mem_init(struct k3_rproc *kproc) /* use remaining reserved memory regions for static carveouts */ for (i = 0; i < num_rmems; i++) { - rmem_np = of_parse_phandle(np, "memory-region", i + 1); - if (!rmem_np) - return -EINVAL; + struct resource res; - rmem = of_reserved_mem_lookup(rmem_np); - of_node_put(rmem_np); - if (!rmem) - return -EINVAL; + ret = of_reserved_mem_region_to_resource(np, i + 1, &res); + if (ret) + return ret; - kproc->rmem[i].bus_addr = rmem->base; + kproc->rmem[i].bus_addr = res.start; /* 64-bit address regions currently not supported */ - kproc->rmem[i].dev_addr = (u32)rmem->base; - kproc->rmem[i].size = rmem->size; - kproc->rmem[i].cpu_addr = devm_ioremap_wc(dev, rmem->base, rmem->size); + kproc->rmem[i].dev_addr = (u32)res.start; + kproc->rmem[i].size = resource_size(&res); + kproc->rmem[i].cpu_addr = devm_ioremap_resource_wc(dev, &res); if (!kproc->rmem[i].cpu_addr) { - dev_err(dev, "failed to map reserved memory#%d at %pa of size %pa\n", - i + 1, &rmem->base, &rmem->size); + dev_err(dev, "failed to map reserved memory#%d at %pR\n", + i + 1, &res); return -ENOMEM; } diff --git a/drivers/remoteproc/xlnx_r5_remoteproc.c b/drivers/remoteproc/xlnx_r5_remoteproc.c index 0b7b173d0d26..a7b75235f53e 100644 --- a/drivers/remoteproc/xlnx_r5_remoteproc.c +++ b/drivers/remoteproc/xlnx_r5_remoteproc.c @@ -492,53 +492,46 @@ static int add_mem_regions_carveout(struct rproc *rproc) { struct rproc_mem_entry *rproc_mem; struct zynqmp_r5_core *r5_core; - struct of_phandle_iterator it; - struct reserved_mem *rmem; int i = 0; r5_core = rproc->priv; /* Register associated reserved memory regions */ - of_phandle_iterator_init(&it, r5_core->np, "memory-region", NULL, 0); + while (1) { + int err; + struct resource res; - while (of_phandle_iterator_next(&it) == 0) { - rmem = of_reserved_mem_lookup(it.node); - if (!rmem) { - of_node_put(it.node); - dev_err(&rproc->dev, "unable to acquire memory-region\n"); - return -EINVAL; - } + err = of_reserved_mem_region_to_resource(r5_core->np, i, &res); + if (err) + return 0; - if (!strcmp(it.node->name, "vdev0buffer")) { + if (strstarts(res.name, "vdev0buffer")) { /* Init reserved memory for vdev buffer */ rproc_mem = rproc_of_resm_mem_entry_init(&rproc->dev, i, - rmem->size, - rmem->base, - it.node->name); + resource_size(&res), + res.start, + "vdev0buffer"); } else { /* Register associated reserved memory regions */ rproc_mem = rproc_mem_entry_init(&rproc->dev, NULL, - (dma_addr_t)rmem->base, - rmem->size, rmem->base, + (dma_addr_t)res.start, + resource_size(&res), res.start, zynqmp_r5_mem_region_map, zynqmp_r5_mem_region_unmap, - it.node->name); + "%.*s", + strchrnul(res.name, '@') - res.name, + res.name); } - if (!rproc_mem) { - of_node_put(it.node); + if (!rproc_mem) return -ENOMEM; - } rproc_add_carveout(rproc, rproc_mem); - rproc_coredump_add_segment(rproc, rmem->base, rmem->size); + rproc_coredump_add_segment(rproc, res.start, resource_size(&res)); - dev_dbg(&rproc->dev, "reserved mem carveout %s addr=%llx, size=0x%llx", - it.node->name, rmem->base, rmem->size); + dev_dbg(&rproc->dev, "reserved mem carveout %pR\n", &res); i++; } - - return 0; } static int add_sram_carveouts(struct rproc *rproc) @@ -808,7 +801,6 @@ static int zynqmp_r5_get_rsc_table_va(struct zynqmp_r5_core *r5_core) struct device *dev = r5_core->dev; struct rsc_tbl_data *rsc_data_va; struct resource res_mem; - struct device_node *np; int ret; /* @@ -818,14 +810,7 @@ static int zynqmp_r5_get_rsc_table_va(struct zynqmp_r5_core *r5_core) * contains that data structure which holds resource table address, size * and some magic number to validate correct resource table entry. */ - np = of_parse_phandle(r5_core->np, "memory-region", 0); - if (!np) { - dev_err(dev, "failed to get memory region dev node\n"); - return -EINVAL; - } - - ret = of_address_to_resource(np, 0, &res_mem); - of_node_put(np); + ret = of_reserved_mem_region_to_resource(r5_core->np, 0, &res_mem); if (ret) { dev_err(dev, "failed to get memory-region resource addr\n"); return -EINVAL; diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c index 820a6ca5b1d7..5ea096acc858 100644 --- a/drivers/rpmsg/qcom_glink_native.c +++ b/drivers/rpmsg/qcom_glink_native.c @@ -1395,6 +1395,19 @@ static int qcom_glink_announce_create(struct rpmsg_device *rpdev) return 0; } +static void qcom_glink_remove_rpmsg_device(struct qcom_glink *glink, struct glink_channel *channel) +{ + struct rpmsg_channel_info chinfo; + + if (channel->rpdev) { + strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); + chinfo.src = RPMSG_ADDR_ANY; + chinfo.dst = RPMSG_ADDR_ANY; + rpmsg_unregister_device(glink->dev, &chinfo); + } + channel->rpdev = NULL; +} + static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) { struct glink_channel *channel = to_glink_channel(ept); @@ -1406,7 +1419,7 @@ static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) spin_unlock_irqrestore(&channel->recv_lock, flags); /* Decouple the potential rpdev from the channel */ - channel->rpdev = NULL; + qcom_glink_remove_rpmsg_device(glink, channel); qcom_glink_send_close_req(glink, channel); } @@ -1697,7 +1710,6 @@ free_channel: static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) { - struct rpmsg_channel_info chinfo; struct glink_channel *channel; unsigned long flags; @@ -1713,14 +1725,7 @@ static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) /* cancel pending rx_done work */ cancel_work_sync(&channel->intent_work); - if (channel->rpdev) { - strscpy_pad(chinfo.name, channel->name, sizeof(chinfo.name)); - chinfo.src = RPMSG_ADDR_ANY; - chinfo.dst = RPMSG_ADDR_ANY; - - rpmsg_unregister_device(glink->dev, &chinfo); - } - channel->rpdev = NULL; + qcom_glink_remove_rpmsg_device(glink, channel); qcom_glink_send_close_ack(glink, channel); @@ -1734,7 +1739,6 @@ static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) { - struct rpmsg_channel_info chinfo; struct glink_channel *channel; unsigned long flags; @@ -1756,14 +1760,7 @@ static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) spin_unlock_irqrestore(&glink->idr_lock, flags); /* Decouple the potential rpdev from the channel */ - if (channel->rpdev) { - strscpy(chinfo.name, channel->name, sizeof(chinfo.name)); - chinfo.src = RPMSG_ADDR_ANY; - chinfo.dst = RPMSG_ADDR_ANY; - - rpmsg_unregister_device(glink->dev, &chinfo); - } - channel->rpdev = NULL; + qcom_glink_remove_rpmsg_device(glink, channel); kref_put(&channel->refcount, qcom_glink_channel_release); } diff --git a/drivers/soc/renesas/renesas-soc.c b/drivers/soc/renesas/renesas-soc.c index 1eb52356b996..ee4f17bb4db4 100644 --- a/drivers/soc/renesas/renesas-soc.c +++ b/drivers/soc/renesas/renesas-soc.c @@ -5,6 +5,7 @@ * Copyright (C) 2014-2016 Glider bvba */ +#include <linux/bitfield.h> #include <linux/io.h> #include <linux/of.h> #include <linux/of_address.h> @@ -524,8 +525,7 @@ static int __init renesas_soc_init(void) eshi, eslo); } - if (soc->id && - ((product & id->mask) >> __ffs(id->mask)) != soc->id) { + if (soc->id && field_get(id->mask, product) != soc->id) { pr_warn("SoC mismatch (product = 0x%x)\n", product); ret = -ENODEV; goto free_soc_dev_attr; diff --git a/drivers/soc/renesas/rz-sysc.c b/drivers/soc/renesas/rz-sysc.c index 19c1e666279b..ae727d9c8cc5 100644 --- a/drivers/soc/renesas/rz-sysc.c +++ b/drivers/soc/renesas/rz-sysc.c @@ -5,6 +5,7 @@ * Copyright (C) 2024 Renesas Electronics Corp. */ +#include <linux/bitfield.h> #include <linux/cleanup.h> #include <linux/io.h> #include <linux/mfd/syscon.h> @@ -16,8 +17,6 @@ #include "rz-sysc.h" -#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) - /** * struct rz_sysc - RZ SYSC private data structure * @base: SYSC base address diff --git a/drivers/video/fbdev/gbefb.c b/drivers/video/fbdev/gbefb.c index 4c36a3e409be..cb6ff15a21db 100644 --- a/drivers/video/fbdev/gbefb.c +++ b/drivers/video/fbdev/gbefb.c @@ -12,6 +12,7 @@ #include <linux/delay.h> #include <linux/platform_device.h> #include <linux/dma-mapping.h> +#include <linux/dma-direct.h> #include <linux/errno.h> #include <linux/gfp.h> #include <linux/fb.h> @@ -65,7 +66,7 @@ struct gbefb_par { static unsigned int gbe_mem_size = CONFIG_FB_GBE_MEM * 1024*1024; static void *gbe_mem; static dma_addr_t gbe_dma_addr; -static unsigned long gbe_mem_phys; +static phys_addr_t gbe_mem_phys; static struct { uint16_t *cpu; @@ -1183,7 +1184,7 @@ static int gbefb_probe(struct platform_device *p_dev) goto out_release_mem_region; } - gbe_mem_phys = (unsigned long) gbe_dma_addr; + gbe_mem_phys = dma_to_phys(&p_dev->dev, gbe_dma_addr); } par = info->par; diff --git a/drivers/video/fbdev/gxt4500.c b/drivers/video/fbdev/gxt4500.c index 15a82c6b609e..1ee0a1efa18e 100644 --- a/drivers/video/fbdev/gxt4500.c +++ b/drivers/video/fbdev/gxt4500.c @@ -704,7 +704,7 @@ static int gxt4500_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } info->var = var; if (gxt4500_set_par(info)) { - printk(KERN_ERR "gxt4500: cannot set video mode\n"); + dev_err(&pdev->dev, "cannot set video mode\n"); goto err_free_cmap; } diff --git a/drivers/video/fbdev/i810/i810_main.c b/drivers/video/fbdev/i810/i810_main.c index d73a795fe1be..10b914a24114 100644 --- a/drivers/video/fbdev/i810/i810_main.c +++ b/drivers/video/fbdev/i810/i810_main.c @@ -1012,7 +1012,7 @@ static int i810_check_params(struct fb_var_screeninfo *var, var->bits_per_pixel); vidmem = line_length * info->var.yres; if (vxres < var->xres) { - printk("i810fb: required video memory, " + dev_info(&par->dev->dev, "required video memory, " "%d bytes, for %dx%d-%d (virtual) " "is out of range\n", vidmem, vxres, vyres, @@ -1067,9 +1067,9 @@ static int i810_check_params(struct fb_var_screeninfo *var, |(info->monspecs.hfmax-HFMAX) |(info->monspecs.vfmin-VFMIN) |(info->monspecs.vfmax-VFMAX); - printk("i810fb: invalid video mode%s\n", - default_sync ? "" : ". Specifying " - "vsyncN/hsyncN parameters may help"); + dev_err(&par->dev->dev, "invalid video mode%s\n", + default_sync ? "" : ". Specifying " + "vsyncN/hsyncN parameters may help"); retval = -EINVAL; } } @@ -1674,19 +1674,19 @@ static int i810_alloc_agp_mem(struct fb_info *info) size = par->fb.size + par->iring.size; if (!(bridge = agp_backend_acquire(par->dev))) { - printk("i810fb_alloc_fbmem: cannot acquire agpgart\n"); + dev_warn(&par->dev->dev, "cannot acquire agpgart\n"); return -ENODEV; } if (!(par->i810_gtt.i810_fb_memory = agp_allocate_memory(bridge, size >> 12, AGP_NORMAL_MEMORY))) { - printk("i810fb_alloc_fbmem: can't allocate framebuffer " + dev_warn(&par->dev->dev, "can't allocate framebuffer " "memory\n"); agp_backend_release(bridge); return -ENOMEM; } if (agp_bind_memory(par->i810_gtt.i810_fb_memory, par->fb.offset)) { - printk("i810fb_alloc_fbmem: can't bind framebuffer memory\n"); + dev_warn(&par->dev->dev, "can't bind framebuffer memory\n"); agp_backend_release(bridge); return -EBUSY; } @@ -1694,14 +1694,14 @@ static int i810_alloc_agp_mem(struct fb_info *info) if (!(par->i810_gtt.i810_cursor_memory = agp_allocate_memory(bridge, par->cursor_heap.size >> 12, AGP_PHYSICAL_MEMORY))) { - printk("i810fb_alloc_cursormem: can't allocate " + dev_warn(&par->dev->dev, "can't allocate " "cursor memory\n"); agp_backend_release(bridge); return -ENOMEM; } if (agp_bind_memory(par->i810_gtt.i810_cursor_memory, par->cursor_heap.offset)) { - printk("i810fb_alloc_cursormem: cannot bind cursor memory\n"); + dev_warn(&par->dev->dev, "cannot bind cursor memory\n"); agp_backend_release(bridge); return -EBUSY; } @@ -1844,7 +1844,7 @@ static int i810_allocate_pci_resource(struct i810fb_par *par, int err; if ((err = pci_enable_device(par->dev))) { - printk("i810fb_init: cannot enable device\n"); + dev_err(&par->dev->dev, "cannot enable device\n"); return err; } par->res_flags |= PCI_DEVICE_ENABLED; @@ -1859,14 +1859,14 @@ static int i810_allocate_pci_resource(struct i810fb_par *par, par->mmio_start_phys = pci_resource_start(par->dev, 0); } if (!par->aperture.size) { - printk("i810fb_init: device is disabled\n"); + dev_warn(&par->dev->dev, "device is disabled\n"); return -ENOMEM; } if (!request_mem_region(par->aperture.physical, par->aperture.size, i810_pci_list[entry->driver_data])) { - printk("i810fb_init: cannot request framebuffer region\n"); + dev_warn(&par->dev->dev, "cannot request framebuffer region\n"); return -ENODEV; } par->res_flags |= FRAMEBUFFER_REQ; @@ -1874,14 +1874,14 @@ static int i810_allocate_pci_resource(struct i810fb_par *par, par->aperture.virtual = ioremap_wc(par->aperture.physical, par->aperture.size); if (!par->aperture.virtual) { - printk("i810fb_init: cannot remap framebuffer region\n"); + dev_warn(&par->dev->dev, "cannot remap framebuffer region\n"); return -ENODEV; } if (!request_mem_region(par->mmio_start_phys, MMIO_SIZE, i810_pci_list[entry->driver_data])) { - printk("i810fb_init: cannot request mmio region\n"); + dev_warn(&par->dev->dev, "cannot request mmio region\n"); return -ENODEV; } par->res_flags |= MMIO_REQ; @@ -1889,7 +1889,7 @@ static int i810_allocate_pci_resource(struct i810fb_par *par, par->mmio_start_virtual = ioremap(par->mmio_start_phys, MMIO_SIZE); if (!par->mmio_start_virtual) { - printk("i810fb_init: cannot remap mmio region\n"); + dev_warn(&par->dev->dev, "cannot remap mmio region\n"); return -ENODEV; } @@ -1921,12 +1921,12 @@ static void i810fb_find_init_mode(struct fb_info *info) } if (!err) - printk("i810fb_init_pci: DDC probe successful\n"); + dev_info(&par->dev->dev, "DDC probe successful\n"); fb_edid_to_monspecs(par->edid, specs); if (specs->modedb == NULL) - printk("i810fb_init_pci: Unable to get Mode Database\n"); + dev_info(&par->dev->dev, "Unable to get Mode Database\n"); fb_videomode_to_modelist(specs->modedb, specs->modedb_len, &info->modelist); @@ -2072,7 +2072,7 @@ static int i810fb_init_pci(struct pci_dev *dev, if (err < 0) { i810fb_release_resource(info, par); - printk("i810fb_init: cannot register framebuffer device\n"); + dev_warn(&par->dev->dev, "cannot register framebuffer device\n"); return err; } @@ -2084,10 +2084,10 @@ static int i810fb_init_pci(struct pci_dev *dev, vfreq = hfreq/(info->var.yres + info->var.upper_margin + info->var.vsync_len + info->var.lower_margin); - printk("I810FB: fb%d : %s v%d.%d.%d%s\n" - "I810FB: Video RAM : %dK\n" - "I810FB: Monitor : H: %d-%d KHz V: %d-%d Hz\n" - "I810FB: Mode : %dx%d-%dbpp@%dHz\n", + dev_info(&par->dev->dev, "fb%d : %s v%d.%d.%d%s\n" + "Video RAM : %dK\n" + "Monitor : H: %d-%d KHz V: %d-%d Hz\n" + "Mode : %dx%d-%dbpp@%dHz\n", info->node, i810_pci_list[entry->driver_data], VERSION_MAJOR, VERSION_MINOR, VERSION_TEENIE, BRANCH_VERSION, @@ -2137,7 +2137,7 @@ static void i810fb_remove_pci(struct pci_dev *dev) unregister_framebuffer(info); i810fb_release_resource(info, par); - printk("cleanup_module: unloaded i810 framebuffer device\n"); + dev_info(&par->dev->dev, "unloaded i810 framebuffer device\n"); } #ifndef MODULE diff --git a/drivers/video/fbdev/pxafb.c b/drivers/video/fbdev/pxafb.c index b96a8a96bce8..e418eee825fb 100644 --- a/drivers/video/fbdev/pxafb.c +++ b/drivers/video/fbdev/pxafb.c @@ -419,12 +419,12 @@ static int pxafb_adjust_timing(struct pxafb_info *fbi, var->yres = max_t(int, var->yres, MIN_YRES); if (!(fbi->lccr0 & LCCR0_LCDT)) { - clamp_val(var->hsync_len, 1, 64); - clamp_val(var->vsync_len, 1, 64); - clamp_val(var->left_margin, 1, 255); - clamp_val(var->right_margin, 1, 255); - clamp_val(var->upper_margin, 1, 255); - clamp_val(var->lower_margin, 1, 255); + var->hsync_len = clamp(var->hsync_len, 1, 64); + var->vsync_len = clamp(var->vsync_len, 1, 64); + var->left_margin = clamp(var->left_margin, 1, 255); + var->right_margin = clamp(var->right_margin, 1, 255); + var->upper_margin = clamp(var->upper_margin, 1, 255); + var->lower_margin = clamp(var->lower_margin, 1, 255); } /* make sure each line is aligned on word boundary */ diff --git a/drivers/video/fbdev/ssd1307fb.c b/drivers/video/fbdev/ssd1307fb.c index aa6cc0a8151a..83dd31fa1fab 100644 --- a/drivers/video/fbdev/ssd1307fb.c +++ b/drivers/video/fbdev/ssd1307fb.c @@ -680,7 +680,7 @@ static int ssd1307fb_probe(struct i2c_client *client) if (!ssd1307fb_defio) { dev_err(dev, "Couldn't allocate deferred io.\n"); ret = -ENOMEM; - goto fb_alloc_error; + goto fb_defio_error; } ssd1307fb_defio->delay = HZ / refreshrate; @@ -757,6 +757,8 @@ regulator_enable_error: regulator_disable(par->vbat_reg); reset_oled_error: fb_deferred_io_cleanup(info); +fb_defio_error: + __free_pages(vmem, get_order(vmem_size)); fb_alloc_error: framebuffer_release(info); return ret; diff --git a/drivers/video/fbdev/tcx.c b/drivers/video/fbdev/tcx.c index f9a0085ad72b..ca9e84e8d860 100644 --- a/drivers/video/fbdev/tcx.c +++ b/drivers/video/fbdev/tcx.c @@ -428,7 +428,7 @@ static int tcx_probe(struct platform_device *op) j = i; break; } - par->mmap_map[i].poff = op->resource[j].start; + par->mmap_map[i].poff = op->resource[j].start - info->fix.smem_start; } info->fbops = &tcx_ops; diff --git a/drivers/video/fbdev/tridentfb.c b/drivers/video/fbdev/tridentfb.c index 516cf2a18757..17b7253b8fbe 100644 --- a/drivers/video/fbdev/tridentfb.c +++ b/drivers/video/fbdev/tridentfb.c @@ -1631,7 +1631,7 @@ static int trident_pci_probe(struct pci_dev *dev, } if (noaccel) { - printk(KERN_DEBUG "disabling acceleration\n"); + dev_dbg(&dev->dev, "disabling acceleration\n"); info->flags |= FBINFO_HWACCEL_DISABLED; info->pixmap.scan_align = 1; } @@ -1693,7 +1693,7 @@ static int trident_pci_probe(struct pci_dev *dev, info->var.activate |= FB_ACTIVATE_NOW; info->device = &dev->dev; if (register_framebuffer(info) < 0) { - printk(KERN_ERR "tridentfb: could not register framebuffer\n"); + dev_err(&dev->dev, "could not register framebuffer\n"); fb_dealloc_cmap(&info->cmap); err = -EINVAL; goto out_unmap2; diff --git a/drivers/video/fbdev/vesafb.c b/drivers/video/fbdev/vesafb.c index a81df8865143..f135033c22fb 100644 --- a/drivers/video/fbdev/vesafb.c +++ b/drivers/video/fbdev/vesafb.c @@ -314,8 +314,8 @@ static int vesafb_probe(struct platform_device *dev) #endif if (!request_mem_region(vesafb_fix.smem_start, size_total, "vesafb")) { - printk(KERN_WARNING - "vesafb: cannot reserve video memory at 0x%lx\n", + dev_warn(&dev->dev, + "cannot reserve video memory at 0x%lx\n", vesafb_fix.smem_start); /* We cannot make this fatal. Sometimes this comes from magic spaces our resource handlers simply don't know about */ @@ -333,12 +333,12 @@ static int vesafb_probe(struct platform_device *dev) par->base = si->lfb_base; par->size = size_total; - printk(KERN_INFO "vesafb: mode is %dx%dx%d, linelength=%d, pages=%d\n", + dev_info(&dev->dev, "mode is %dx%dx%d, linelength=%d, pages=%d\n", vesafb_defined.xres, vesafb_defined.yres, vesafb_defined.bits_per_pixel, vesafb_fix.line_length, si->pages); if (si->vesapm_seg) { - printk(KERN_INFO "vesafb: protected mode interface info at %04x:%04x\n", + dev_info(&dev->dev, "protected mode interface info at %04x:%04x\n", si->vesapm_seg, si->vesapm_off); } @@ -352,9 +352,10 @@ static int vesafb_probe(struct platform_device *dev) pmi_base = (unsigned short *)phys_to_virt(pmi_phys); pmi_start = (void*)((char*)pmi_base + pmi_base[1]); pmi_pal = (void*)((char*)pmi_base + pmi_base[2]); - printk(KERN_INFO "vesafb: pmi: set display start = %p, set palette = %p\n",pmi_start,pmi_pal); + dev_info(&dev->dev, "pmi: set display start = %p, set palette = %p\n", + pmi_start, pmi_pal); if (pmi_base[3]) { - printk(KERN_INFO "vesafb: pmi: ports = "); + dev_info(&dev->dev, "pmi: ports = "); for (i = pmi_base[3]/2; pmi_base[i] != 0xffff; i++) printk("%x ", pmi_base[i]); printk("\n"); @@ -365,14 +366,14 @@ static int vesafb_probe(struct platform_device *dev) * Rules are: we have to set up a descriptor for the requested * memory area and pass it in the ES register to the BIOS function. */ - printk(KERN_INFO "vesafb: can't handle memory requests, pmi disabled\n"); + dev_info(&dev->dev, "can't handle memory requests, pmi disabled\n"); ypan = pmi_setpal = 0; } } } if (vesafb_defined.bits_per_pixel == 8 && !pmi_setpal && !vga_compat) { - printk(KERN_WARNING "vesafb: hardware palette is unchangeable,\n" + dev_warn(&dev->dev, "hardware palette is unchangeable,\n" " colors may be incorrect\n"); vesafb_fix.visual = FB_VISUAL_STATIC_PSEUDOCOLOR; } @@ -380,10 +381,10 @@ static int vesafb_probe(struct platform_device *dev) vesafb_defined.xres_virtual = vesafb_defined.xres; vesafb_defined.yres_virtual = vesafb_fix.smem_len / vesafb_fix.line_length; if (ypan && vesafb_defined.yres_virtual > vesafb_defined.yres) { - printk(KERN_INFO "vesafb: scrolling: %s using protected mode interface, yres_virtual=%d\n", + dev_info(&dev->dev, "scrolling: %s using protected mode interface, yres_virtual=%d\n", (ypan > 1) ? "ywrap" : "ypan",vesafb_defined.yres_virtual); } else { - printk(KERN_INFO "vesafb: scrolling: redraw\n"); + dev_info(&dev->dev, "scrolling: redraw\n"); vesafb_defined.yres_virtual = vesafb_defined.yres; ypan = 0; } @@ -410,7 +411,7 @@ static int vesafb_probe(struct platform_device *dev) vesafb_defined.bits_per_pixel; } - printk(KERN_INFO "vesafb: %s: " + dev_info(&dev->dev, "%s: " "size=%d:%d:%d:%d, shift=%d:%d:%d:%d\n", (vesafb_defined.bits_per_pixel > 8) ? "Truecolor" : (vga_compat || pmi_setpal) ? @@ -453,14 +454,14 @@ static int vesafb_probe(struct platform_device *dev) } if (!info->screen_base) { - printk(KERN_ERR - "vesafb: abort, cannot ioremap video memory 0x%x @ 0x%lx\n", + dev_err(&dev->dev, + "abort, cannot ioremap video memory 0x%x @ 0x%lx\n", vesafb_fix.smem_len, vesafb_fix.smem_start); err = -EIO; goto err_release_region; } - printk(KERN_INFO "vesafb: framebuffer at 0x%lx, mapped to 0x%p, " + dev_info(&dev->dev, "framebuffer at 0x%lx, mapped to 0x%p, " "using %dk, total %dk\n", vesafb_fix.smem_start, info->screen_base, size_remap/1024, size_total/1024); diff --git a/drivers/video/fbdev/vga16fb.c b/drivers/video/fbdev/vga16fb.c index eedab14c7d51..6b81337a4909 100644 --- a/drivers/video/fbdev/vga16fb.c +++ b/drivers/video/fbdev/vga16fb.c @@ -1319,7 +1319,12 @@ static int vga16fb_probe(struct platform_device *dev) if (ret) return ret; - printk(KERN_DEBUG "vga16fb: initializing\n"); + dev_dbg(&dev->dev, "initializing\n"); + if (!request_mem_region(vga16fb_fix.smem_start, vga16fb_fix.smem_len, + "vga16b")) { + dev_err(&dev->dev, "cannot reserve video memory at 0x%lx\n", + vga16fb_fix.smem_start); + } info = framebuffer_alloc(sizeof(struct vga16fb_par), &dev->dev); if (!info) { @@ -1331,12 +1336,12 @@ static int vga16fb_probe(struct platform_device *dev) info->screen_base = (void __iomem *)VGA_MAP_MEM(VGA_FB_PHYS_BASE, 0); if (!info->screen_base) { - printk(KERN_ERR "vga16fb: unable to map device\n"); + dev_err(&dev->dev, "unable to map device\n"); ret = -ENOMEM; goto err_ioremap; } - printk(KERN_INFO "vga16fb: mapped to 0x%p\n", info->screen_base); + dev_info(&dev->dev, "mapped to 0x%p\n", info->screen_base); par = info->par; par->isVGA = screen_info_video_type(si) == VIDEO_TYPE_VGAC; @@ -1364,13 +1369,13 @@ static int vga16fb_probe(struct platform_device *dev) i = (info->var.bits_per_pixel == 8) ? 256 : 16; ret = fb_alloc_cmap(&info->cmap, i, 0); if (ret) { - printk(KERN_ERR "vga16fb: unable to allocate colormap\n"); + dev_err(&dev->dev, "unable to allocate colormap\n"); ret = -ENOMEM; goto err_alloc_cmap; } if (vga16fb_check_var(&info->var, info)) { - printk(KERN_ERR "vga16fb: unable to validate variable\n"); + dev_err(&dev->dev, "unable to validate variable\n"); ret = -EINVAL; goto err_check_var; } @@ -1381,7 +1386,7 @@ static int vga16fb_probe(struct platform_device *dev) if (ret) goto err_check_var; if (register_framebuffer(info) < 0) { - printk(KERN_ERR "vga16fb: unable to register framebuffer\n"); + dev_err(&dev->dev, "unable to register framebuffer\n"); ret = -EINVAL; goto err_check_var; } @@ -1398,6 +1403,8 @@ static int vga16fb_probe(struct platform_device *dev) err_ioremap: framebuffer_release(info); err_fb_alloc: + release_mem_region(vga16fb_fix.smem_start, + vga16fb_fix.smem_len); return ret; } @@ -1407,6 +1414,8 @@ static void vga16fb_remove(struct platform_device *dev) if (info) unregister_framebuffer(info); + release_mem_region(vga16fb_fix.smem_start, + vga16fb_fix.smem_len); } static const struct platform_device_id vga16fb_driver_id_table[] = { diff --git a/drivers/virt/Kconfig b/drivers/virt/Kconfig index d8c848cf09a6..52eb7e4ba71f 100644 --- a/drivers/virt/Kconfig +++ b/drivers/virt/Kconfig @@ -47,6 +47,6 @@ source "drivers/virt/nitro_enclaves/Kconfig" source "drivers/virt/acrn/Kconfig" -source "drivers/virt/coco/Kconfig" - endif + +source "drivers/virt/coco/Kconfig" diff --git a/drivers/virt/coco/Kconfig b/drivers/virt/coco/Kconfig index 819a97e8ba99..df1cfaf26c65 100644 --- a/drivers/virt/coco/Kconfig +++ b/drivers/virt/coco/Kconfig @@ -3,6 +3,7 @@ # Confidential computing related collateral # +if VIRT_DRIVERS source "drivers/virt/coco/efi_secret/Kconfig" source "drivers/virt/coco/pkvm-guest/Kconfig" @@ -14,3 +15,7 @@ source "drivers/virt/coco/tdx-guest/Kconfig" source "drivers/virt/coco/arm-cca-guest/Kconfig" source "drivers/virt/coco/guest/Kconfig" +endif + +config TSM + bool diff --git a/drivers/virt/coco/Makefile b/drivers/virt/coco/Makefile index f918bbb61737..cb52021912b3 100644 --- a/drivers/virt/coco/Makefile +++ b/drivers/virt/coco/Makefile @@ -7,4 +7,5 @@ obj-$(CONFIG_ARM_PKVM_GUEST) += pkvm-guest/ obj-$(CONFIG_SEV_GUEST) += sev-guest/ obj-$(CONFIG_INTEL_TDX_GUEST) += tdx-guest/ obj-$(CONFIG_ARM_CCA_GUEST) += arm-cca-guest/ +obj-$(CONFIG_TSM) += tsm-core.o obj-$(CONFIG_TSM_GUEST) += guest/ diff --git a/drivers/virt/coco/tsm-core.c b/drivers/virt/coco/tsm-core.c new file mode 100644 index 000000000000..f027876a2f19 --- /dev/null +++ b/drivers/virt/coco/tsm-core.c @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2024-2025 Intel Corporation. All rights reserved. */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/tsm.h> +#include <linux/pci.h> +#include <linux/rwsem.h> +#include <linux/device.h> +#include <linux/module.h> +#include <linux/cleanup.h> +#include <linux/pci-tsm.h> +#include <linux/pci-ide.h> + +static struct class *tsm_class; +static DECLARE_RWSEM(tsm_rwsem); +static DEFINE_IDA(tsm_ida); + +static int match_id(struct device *dev, const void *data) +{ + struct tsm_dev *tsm_dev = container_of(dev, struct tsm_dev, dev); + int id = *(const int *)data; + + return tsm_dev->id == id; +} + +struct tsm_dev *find_tsm_dev(int id) +{ + struct device *dev = class_find_device(tsm_class, NULL, &id, match_id); + + if (!dev) + return NULL; + return container_of(dev, struct tsm_dev, dev); +} + +static struct tsm_dev *alloc_tsm_dev(struct device *parent) +{ + struct device *dev; + int id; + + struct tsm_dev *tsm_dev __free(kfree) = + kzalloc(sizeof(*tsm_dev), GFP_KERNEL); + if (!tsm_dev) + return ERR_PTR(-ENOMEM); + + id = ida_alloc(&tsm_ida, GFP_KERNEL); + if (id < 0) + return ERR_PTR(id); + + tsm_dev->id = id; + dev = &tsm_dev->dev; + dev->parent = parent; + dev->class = tsm_class; + device_initialize(dev); + + return no_free_ptr(tsm_dev); +} + +static struct tsm_dev *tsm_register_pci_or_reset(struct tsm_dev *tsm_dev, + struct pci_tsm_ops *pci_ops) +{ + int rc; + + if (!pci_ops) + return tsm_dev; + + tsm_dev->pci_ops = pci_ops; + rc = pci_tsm_register(tsm_dev); + if (rc) { + dev_err(tsm_dev->dev.parent, + "PCI/TSM registration failure: %d\n", rc); + device_unregister(&tsm_dev->dev); + return ERR_PTR(rc); + } + + /* Notify TSM userspace that PCI/TSM operations are now possible */ + kobject_uevent(&tsm_dev->dev.kobj, KOBJ_CHANGE); + return tsm_dev; +} + +struct tsm_dev *tsm_register(struct device *parent, struct pci_tsm_ops *pci_ops) +{ + struct tsm_dev *tsm_dev __free(put_tsm_dev) = alloc_tsm_dev(parent); + struct device *dev; + int rc; + + if (IS_ERR(tsm_dev)) + return tsm_dev; + + dev = &tsm_dev->dev; + rc = dev_set_name(dev, "tsm%d", tsm_dev->id); + if (rc) + return ERR_PTR(rc); + + rc = device_add(dev); + if (rc) + return ERR_PTR(rc); + + return tsm_register_pci_or_reset(no_free_ptr(tsm_dev), pci_ops); +} +EXPORT_SYMBOL_GPL(tsm_register); + +void tsm_unregister(struct tsm_dev *tsm_dev) +{ + if (tsm_dev->pci_ops) + pci_tsm_unregister(tsm_dev); + device_unregister(&tsm_dev->dev); +} +EXPORT_SYMBOL_GPL(tsm_unregister); + +/* must be invoked between tsm_register / tsm_unregister */ +int tsm_ide_stream_register(struct pci_ide *ide) +{ + struct pci_dev *pdev = ide->pdev; + struct pci_tsm *tsm = pdev->tsm; + struct tsm_dev *tsm_dev = tsm->tsm_dev; + int rc; + + rc = sysfs_create_link(&tsm_dev->dev.kobj, &pdev->dev.kobj, ide->name); + if (rc) + return rc; + + ide->tsm_dev = tsm_dev; + return 0; +} +EXPORT_SYMBOL_GPL(tsm_ide_stream_register); + +void tsm_ide_stream_unregister(struct pci_ide *ide) +{ + struct tsm_dev *tsm_dev = ide->tsm_dev; + + ide->tsm_dev = NULL; + sysfs_remove_link(&tsm_dev->dev.kobj, ide->name); +} +EXPORT_SYMBOL_GPL(tsm_ide_stream_unregister); + +static void tsm_release(struct device *dev) +{ + struct tsm_dev *tsm_dev = container_of(dev, typeof(*tsm_dev), dev); + + ida_free(&tsm_ida, tsm_dev->id); + kfree(tsm_dev); +} + +static int __init tsm_init(void) +{ + tsm_class = class_create("tsm"); + if (IS_ERR(tsm_class)) + return PTR_ERR(tsm_class); + + tsm_class->dev_release = tsm_release; + return 0; +} +module_init(tsm_init) + +static void __exit tsm_exit(void) +{ + class_destroy(tsm_class); +} +module_exit(tsm_exit) + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("TEE Security Manager Class Device"); diff --git a/drivers/watchdog/Kconfig b/drivers/watchdog/Kconfig index 05008d937e40..d3b9df7d466b 100644 --- a/drivers/watchdog/Kconfig +++ b/drivers/watchdog/Kconfig @@ -969,6 +969,14 @@ config RENESAS_WDT This driver adds watchdog support for the integrated watchdogs in the Renesas R-Car and other SH-Mobile SoCs (usually named RWDT or SWDT). +config RENESAS_WWDT + tristate "Renesas Window WWDT Watchdog" + depends on ARCH_RENESAS || COMPILE_TEST + select WATCHDOG_CORE + help + This driver adds watchdog support for a window timer found in some + Renesas R-Car Gen3 and later SoCs. + config RENESAS_RZAWDT tristate "Renesas RZ/A WDT Watchdog" depends on ARCH_RENESAS || COMPILE_TEST @@ -1976,10 +1984,10 @@ config LANTIQ_WDT config LOONGSON1_WDT tristate "Loongson1 SoC hardware watchdog" - depends on MACH_LOONGSON32 || COMPILE_TEST + depends on MACH_LOONGSON32 || MACH_LOONGSON64 || COMPILE_TEST select WATCHDOG_CORE help - Hardware driver for the Loongson1 SoC Watchdog Timer. + Hardware driver for the Loongson family Watchdog Timer. config RALINK_WDT tristate "Ralink SoC watchdog" diff --git a/drivers/watchdog/Makefile b/drivers/watchdog/Makefile index b680e4d3c1bc..ba52099b1253 100644 --- a/drivers/watchdog/Makefile +++ b/drivers/watchdog/Makefile @@ -85,6 +85,7 @@ obj-$(CONFIG_DIGICOLOR_WATCHDOG) += digicolor_wdt.o obj-$(CONFIG_LPC18XX_WATCHDOG) += lpc18xx_wdt.o obj-$(CONFIG_BCM7038_WDT) += bcm7038_wdt.o obj-$(CONFIG_RENESAS_WDT) += renesas_wdt.o +obj-$(CONFIG_RENESAS_WWDT) += renesas_wwdt.o obj-$(CONFIG_RENESAS_RZAWDT) += rza_wdt.o obj-$(CONFIG_RENESAS_RZN1WDT) += rzn1_wdt.o obj-$(CONFIG_RENESAS_RZG2LWDT) += rzg2l_wdt.o diff --git a/drivers/watchdog/aspeed_wdt.c b/drivers/watchdog/aspeed_wdt.c index 837e15701c0e..c9e79851504c 100644 --- a/drivers/watchdog/aspeed_wdt.c +++ b/drivers/watchdog/aspeed_wdt.c @@ -35,6 +35,7 @@ struct aspeed_wdt_config { u32 irq_shift; u32 irq_mask; struct aspeed_wdt_scu scu; + u32 num_reset_masks; }; struct aspeed_wdt { @@ -66,6 +67,7 @@ static const struct aspeed_wdt_config ast2500_config = { .wdt_reset_mask = 0x1, .wdt_reset_mask_shift = 2, }, + .num_reset_masks = 1, }; static const struct aspeed_wdt_config ast2600_config = { @@ -78,12 +80,27 @@ static const struct aspeed_wdt_config ast2600_config = { .wdt_reset_mask = 0xf, .wdt_reset_mask_shift = 16, }, + .num_reset_masks = 2, +}; + +static const struct aspeed_wdt_config ast2700_config = { + .ext_pulse_width_mask = 0xfffff, + .irq_shift = 0, + .irq_mask = GENMASK(31, 10), + .scu = { + .compatible = "aspeed,ast2700-scu0", + .reset_status_reg = 0x70, + .wdt_reset_mask = 0xf, + .wdt_reset_mask_shift = 0, + }, + .num_reset_masks = 5, }; static const struct of_device_id aspeed_wdt_of_table[] = { { .compatible = "aspeed,ast2400-wdt", .data = &ast2400_config }, { .compatible = "aspeed,ast2500-wdt", .data = &ast2500_config }, { .compatible = "aspeed,ast2600-wdt", .data = &ast2600_config }, + { .compatible = "aspeed,ast2700-wdt", .data = &ast2700_config }, { }, }; MODULE_DEVICE_TABLE(of, aspeed_wdt_of_table); @@ -479,11 +496,11 @@ static int aspeed_wdt_probe(struct platform_device *pdev) set_bit(WDOG_HW_RUNNING, &wdt->wdd.status); } - if ((of_device_is_compatible(np, "aspeed,ast2500-wdt")) || - (of_device_is_compatible(np, "aspeed,ast2600-wdt"))) { - u32 reset_mask[2]; - size_t nrstmask = of_device_is_compatible(np, "aspeed,ast2600-wdt") ? 2 : 1; + if (!of_device_is_compatible(np, "aspeed,ast2400-wdt")) { + u32 reset_mask[5]; + size_t nrstmask = wdt->cfg->num_reset_masks; u32 reg = readl(wdt->base + WDT_RESET_WIDTH); + int i; reg &= wdt->cfg->ext_pulse_width_mask; if (of_property_read_bool(np, "aspeed,ext-active-high")) @@ -503,9 +520,8 @@ static int aspeed_wdt_probe(struct platform_device *pdev) ret = of_property_read_u32_array(np, "aspeed,reset-mask", reset_mask, nrstmask); if (!ret) { - writel(reset_mask[0], wdt->base + WDT_RESET_MASK1); - if (nrstmask > 1) - writel(reset_mask[1], wdt->base + WDT_RESET_MASK2); + for (i = 0; i < nrstmask; i++) + writel(reset_mask[i], wdt->base + WDT_RESET_MASK1 + i * 4); } } diff --git a/drivers/watchdog/diag288_wdt.c b/drivers/watchdog/diag288_wdt.c index 9daed2758ae5..ea2b0171e70b 100644 --- a/drivers/watchdog/diag288_wdt.c +++ b/drivers/watchdog/diag288_wdt.c @@ -6,10 +6,10 @@ * to CP. * * The command can be altered using the module parameter "cmd". This is - * not recommended because it's only supported on z/VM but not whith LPAR. + * not recommended because it's only supported on z/VM but not with LPAR. * - * On LPAR, the watchdog will always trigger a system restart. the module - * paramter cmd is meaningless here. + * On LPAR, the watchdog will always trigger a system restart. The module + * parameter cmd is meaningless here. * * * Copyright IBM Corp. 2004, 2013 diff --git a/drivers/watchdog/loongson1_wdt.c b/drivers/watchdog/loongson1_wdt.c index 0587ff44d3a1..2417519717cc 100644 --- a/drivers/watchdog/loongson1_wdt.c +++ b/drivers/watchdog/loongson1_wdt.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* * Copyright (c) 2016 Yang Ling <gnaygnil@gmail.com> + * Copyright (C) 2025 Binbin Zhou <zhoubinbin@loongson.cn> */ #include <linux/clk.h> @@ -10,31 +11,52 @@ #include <linux/platform_device.h> #include <linux/watchdog.h> -/* Loongson 1 Watchdog Register Definitions */ +/* Loongson Watchdog Register Definitions */ #define WDT_EN 0x0 -#define WDT_TIMER 0x4 -#define WDT_SET 0x8 #define DEFAULT_HEARTBEAT 30 static bool nowayout = WATCHDOG_NOWAYOUT; -module_param(nowayout, bool, 0444); +module_param(nowayout, bool, 0); +MODULE_PARM_DESC(nowayout, "Watchdog cannot be stopped once started (default=" + __MODULE_STRING(WATCHDOG_NOWAYOUT) ")"); static unsigned int heartbeat; -module_param(heartbeat, uint, 0444); +module_param(heartbeat, uint, 0); +MODULE_PARM_DESC(heartbeat, "Watchdog heartbeat in seconds. (default=" + __MODULE_STRING(DEFAULT_HEARTBEAT) ")"); + +struct ls1x_wdt_pdata { + u32 timer_offset; + u32 set_offset; + u32 wdt_en_bit; +}; + +static const struct ls1x_wdt_pdata ls1b_wdt_pdata = { + .timer_offset = 0x4, + .set_offset = 0x8, + .wdt_en_bit = BIT(0), +}; + +static const struct ls1x_wdt_pdata ls2k0300_wdt_pdata = { + .timer_offset = 0x8, + .set_offset = 0x4, + .wdt_en_bit = BIT(1), +}; struct ls1x_wdt_drvdata { void __iomem *base; struct clk *clk; unsigned long clk_rate; struct watchdog_device wdt; + const struct ls1x_wdt_pdata *pdata; }; static int ls1x_wdt_ping(struct watchdog_device *wdt_dev) { struct ls1x_wdt_drvdata *drvdata = watchdog_get_drvdata(wdt_dev); - writel(0x1, drvdata->base + WDT_SET); + writel(0x1, drvdata->base + drvdata->pdata->set_offset); return 0; } @@ -49,7 +71,7 @@ static int ls1x_wdt_set_timeout(struct watchdog_device *wdt_dev, wdt_dev->timeout = timeout; counts = drvdata->clk_rate * min(timeout, max_hw_heartbeat); - writel(counts, drvdata->base + WDT_TIMER); + writel(counts, drvdata->base + drvdata->pdata->timer_offset); return 0; } @@ -58,7 +80,7 @@ static int ls1x_wdt_start(struct watchdog_device *wdt_dev) { struct ls1x_wdt_drvdata *drvdata = watchdog_get_drvdata(wdt_dev); - writel(0x1, drvdata->base + WDT_EN); + writel(drvdata->pdata->wdt_en_bit, drvdata->base + WDT_EN); return 0; } @@ -66,8 +88,10 @@ static int ls1x_wdt_start(struct watchdog_device *wdt_dev) static int ls1x_wdt_stop(struct watchdog_device *wdt_dev) { struct ls1x_wdt_drvdata *drvdata = watchdog_get_drvdata(wdt_dev); + u32 val = readl(drvdata->base + WDT_EN); - writel(0x0, drvdata->base + WDT_EN); + val &= ~(drvdata->pdata->wdt_en_bit); + writel(val, drvdata->base + WDT_EN); return 0; } @@ -77,9 +101,9 @@ static int ls1x_wdt_restart(struct watchdog_device *wdt_dev, { struct ls1x_wdt_drvdata *drvdata = watchdog_get_drvdata(wdt_dev); - writel(0x1, drvdata->base + WDT_EN); - writel(0x1, drvdata->base + WDT_TIMER); - writel(0x1, drvdata->base + WDT_SET); + writel(drvdata->pdata->wdt_en_bit, drvdata->base + WDT_EN); + writel(0x1, drvdata->base + drvdata->pdata->timer_offset); + writel(0x1, drvdata->base + drvdata->pdata->set_offset); return 0; } @@ -104,11 +128,13 @@ static int ls1x_wdt_probe(struct platform_device *pdev) struct ls1x_wdt_drvdata *drvdata; struct watchdog_device *ls1x_wdt; unsigned long clk_rate; - int err; drvdata = devm_kzalloc(dev, sizeof(*drvdata), GFP_KERNEL); if (!drvdata) return -ENOMEM; + platform_set_drvdata(pdev, drvdata); + + drvdata->pdata = of_device_get_match_data(dev); drvdata->base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(drvdata->base)) @@ -135,36 +161,51 @@ static int ls1x_wdt_probe(struct platform_device *pdev) watchdog_set_nowayout(ls1x_wdt, nowayout); watchdog_set_drvdata(ls1x_wdt, drvdata); - err = devm_watchdog_register_device(dev, &drvdata->wdt); - if (err) - return err; + return devm_watchdog_register_device(dev, &drvdata->wdt); +} - platform_set_drvdata(pdev, drvdata); +static int ls1x_wdt_resume(struct device *dev) +{ + struct ls1x_wdt_drvdata *data = dev_get_drvdata(dev); + + if (watchdog_active(&data->wdt)) + ls1x_wdt_start(&data->wdt); + + return 0; +} + +static int ls1x_wdt_suspend(struct device *dev) +{ + struct ls1x_wdt_drvdata *data = dev_get_drvdata(dev); - dev_info(dev, "Loongson1 Watchdog driver registered\n"); + if (watchdog_active(&data->wdt)) + ls1x_wdt_stop(&data->wdt); return 0; } -#ifdef CONFIG_OF +static DEFINE_SIMPLE_DEV_PM_OPS(ls1x_wdt_pm_ops, ls1x_wdt_suspend, ls1x_wdt_resume); + static const struct of_device_id ls1x_wdt_dt_ids[] = { - { .compatible = "loongson,ls1b-wdt", }, - { .compatible = "loongson,ls1c-wdt", }, + { .compatible = "loongson,ls1b-wdt", .data = &ls1b_wdt_pdata }, + { .compatible = "loongson,ls1c-wdt", .data = &ls1b_wdt_pdata }, + { .compatible = "loongson,ls2k0300-wdt", .data = &ls2k0300_wdt_pdata }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(of, ls1x_wdt_dt_ids); -#endif static struct platform_driver ls1x_wdt_driver = { .probe = ls1x_wdt_probe, .driver = { .name = "ls1x-wdt", - .of_match_table = of_match_ptr(ls1x_wdt_dt_ids), + .of_match_table = ls1x_wdt_dt_ids, + .pm = pm_ptr(&ls1x_wdt_pm_ops), }, }; module_platform_driver(ls1x_wdt_driver); MODULE_AUTHOR("Yang Ling <gnaygnil@gmail.com>"); -MODULE_DESCRIPTION("Loongson1 Watchdog Driver"); +MODULE_AUTHOR("Binbin Zhou <zhoubinbin@loongson.cn>"); +MODULE_DESCRIPTION("Loongson Watchdog Driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/watchdog/renesas_wwdt.c b/drivers/watchdog/renesas_wwdt.c new file mode 100644 index 000000000000..b250913c349a --- /dev/null +++ b/drivers/watchdog/renesas_wwdt.c @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for the Renesas Window Watchdog Timer (WWDT) + * + * The WWDT can only be setup once after boot. Because we cannot know if this + * already happened in early boot stages, it is mandated that the firmware + * configures the watchdog. Linux then adapts according to the given setup. + * Note that this watchdog reports in the default configuration an overflow to + * the Error Control Module which then decides further actions. Or the WWDT is + * configured to generate an interrupt. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/watchdog.h> + +#define WDTA0WDTE 0x00 +#define WDTA0RUN BIT(7) +#define WDTA0_KEY 0x2c + +#define WDTA0MD 0x0c +#define WDTA0OVF(x) FIELD_GET(GENMASK(6, 4), x) +#define WDTA0WIE BIT(3) +#define WDTA0ERM BIT(2) +#define WDTA0WS(x) FIELD_GET(GENMASK(1, 0), x) + +struct wwdt_priv { + void __iomem *base; + struct watchdog_device wdev; +}; + +static int wwdt_start(struct watchdog_device *wdev) +{ + struct wwdt_priv *priv = container_of(wdev, struct wwdt_priv, wdev); + + writeb(WDTA0RUN | WDTA0_KEY, priv->base + WDTA0WDTE); + return 0; +} + +static const struct watchdog_info wwdt_ident = { + .options = WDIOF_KEEPALIVEPING | WDIOF_ALARMONLY, + .identity = "Renesas Window Watchdog", +}; + +static const struct watchdog_ops wwdt_ops = { + .owner = THIS_MODULE, + .start = wwdt_start, +}; + +static irqreturn_t wwdt_error_irq(int irq, void *dev_id) +{ + struct device *dev = dev_id; + + dev_warn(dev, "Watchdog timed out\n"); + return IRQ_HANDLED; +} + +static irqreturn_t wwdt_pretimeout_irq(int irq, void *dev_id) +{ + struct watchdog_device *wdev = dev_id; + + watchdog_notify_pretimeout(wdev); + return IRQ_HANDLED; +} + +static int wwdt_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct wwdt_priv *priv; + struct watchdog_device *wdev; + struct clk *clk; + unsigned long rate; + unsigned int interval, window_size; + int ret; + u8 val; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + clk = devm_clk_get(dev, "cnt"); + if (IS_ERR(clk)) + return PTR_ERR(clk); + + rate = clk_get_rate(clk); + if (!rate) + return -EINVAL; + + wdev = &priv->wdev; + + val = readb(priv->base + WDTA0WDTE); + if (val & WDTA0RUN) + set_bit(WDOG_HW_RUNNING, &wdev->status); + + val = readb(priv->base + WDTA0MD); + interval = 1 << (9 + WDTA0OVF(val)); + /* size of the closed(!) window per mille */ + window_size = 250 * (3 - WDTA0WS(val)); + + wdev->info = &wwdt_ident; + wdev->ops = &wwdt_ops; + wdev->parent = dev; + wdev->min_hw_heartbeat_ms = window_size * interval / rate; + wdev->max_hw_heartbeat_ms = 1000 * interval / rate; + wdev->timeout = DIV_ROUND_UP(wdev->max_hw_heartbeat_ms, 1000); + watchdog_set_nowayout(wdev, true); + + if (!(val & WDTA0ERM)) { + ret = platform_get_irq_byname(pdev, "error"); + if (ret < 0) + return ret; + + ret = devm_request_threaded_irq(dev, ret, NULL, wwdt_error_irq, + IRQF_ONESHOT, NULL, dev); + if (ret < 0) + return ret; + } + + if (val & WDTA0WIE) { + ret = platform_get_irq_byname(pdev, "pretimeout"); + if (ret < 0) + return ret; + + ret = devm_request_threaded_irq(dev, ret, NULL, wwdt_pretimeout_irq, + IRQF_ONESHOT, NULL, wdev); + if (ret < 0) + return ret; + } + + devm_watchdog_register_device(dev, wdev); + + return 0; +} + +static const struct of_device_id renesas_wwdt_ids[] = { + { .compatible = "renesas,rcar-gen3-wwdt", }, + { .compatible = "renesas,rcar-gen4-wwdt", }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(of, renesas_wwdt_ids); + +static struct platform_driver renesas_wwdt_driver = { + .driver = { + .name = "renesas_wwdt", + .of_match_table = renesas_wwdt_ids, + }, + .probe = wwdt_probe, +}; +module_platform_driver(renesas_wwdt_driver); + +MODULE_DESCRIPTION("Renesas Window Watchdog (WWDT) Driver"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Wolfram Sang <wsa+renesas@sang-engineering.com>"); diff --git a/drivers/watchdog/starfive-wdt.c b/drivers/watchdog/starfive-wdt.c index 355918d62f63..ed71d3960a0f 100644 --- a/drivers/watchdog/starfive-wdt.c +++ b/drivers/watchdog/starfive-wdt.c @@ -500,12 +500,14 @@ static int starfive_wdt_probe(struct platform_device *pdev) if (pm_runtime_enabled(&pdev->dev)) { ret = pm_runtime_put_sync(&pdev->dev); if (ret) - goto err_exit; + goto err_unregister_wdt; } } return 0; +err_unregister_wdt: + watchdog_unregister_device(&wdt->wdd); err_exit: starfive_wdt_disable_clock(wdt); pm_runtime_disable(&pdev->dev); diff --git a/drivers/watchdog/via_wdt.c b/drivers/watchdog/via_wdt.c index d647923d68fe..f55576392651 100644 --- a/drivers/watchdog/via_wdt.c +++ b/drivers/watchdog/via_wdt.c @@ -165,6 +165,7 @@ static int wdt_probe(struct pci_dev *pdev, dev_err(&pdev->dev, "cannot enable PCI device\n"); return -ENODEV; } + wdt_res.name = "via_wdt"; /* * Allocate a MMIO region which contains watchdog control register diff --git a/drivers/watchdog/wdat_wdt.c b/drivers/watchdog/wdat_wdt.c index 650fdc7996e1..dd3c2d69c9df 100644 --- a/drivers/watchdog/wdat_wdt.c +++ b/drivers/watchdog/wdat_wdt.c @@ -326,19 +326,27 @@ static int wdat_wdt_probe(struct platform_device *pdev) return -ENODEV; wdat = devm_kzalloc(dev, sizeof(*wdat), GFP_KERNEL); - if (!wdat) - return -ENOMEM; + if (!wdat) { + ret = -ENOMEM; + goto out_put_table; + } regs = devm_kcalloc(dev, pdev->num_resources, sizeof(*regs), GFP_KERNEL); - if (!regs) - return -ENOMEM; + if (!regs) { + ret = -ENOMEM; + goto out_put_table; + } /* WDAT specification wants to have >= 1ms period */ - if (tbl->timer_period < 1) - return -EINVAL; - if (tbl->min_count > tbl->max_count) - return -EINVAL; + if (tbl->timer_period < 1) { + ret = -EINVAL; + goto out_put_table; + } + if (tbl->min_count > tbl->max_count) { + ret = -EINVAL; + goto out_put_table; + } wdat->period = tbl->timer_period; wdat->wdd.min_timeout = DIV_ROUND_UP(wdat->period * tbl->min_count, 1000); @@ -355,15 +363,20 @@ static int wdat_wdt_probe(struct platform_device *pdev) res = &pdev->resource[i]; if (resource_type(res) == IORESOURCE_MEM) { reg = devm_ioremap_resource(dev, res); - if (IS_ERR(reg)) - return PTR_ERR(reg); + if (IS_ERR(reg)) { + ret = PTR_ERR(reg); + goto out_put_table; + } } else if (resource_type(res) == IORESOURCE_IO) { reg = devm_ioport_map(dev, res->start, 1); - if (!reg) - return -ENOMEM; + if (!reg) { + ret = -ENOMEM; + goto out_put_table; + } } else { dev_err(dev, "Unsupported resource\n"); - return -EINVAL; + ret = -EINVAL; + goto out_put_table; } regs[i] = reg; @@ -385,8 +398,10 @@ static int wdat_wdt_probe(struct platform_device *pdev) } instr = devm_kzalloc(dev, sizeof(*instr), GFP_KERNEL); - if (!instr) - return -ENOMEM; + if (!instr) { + ret = -ENOMEM; + goto out_put_table; + } INIT_LIST_HEAD(&instr->node); instr->entry = entries[i]; @@ -417,7 +432,8 @@ static int wdat_wdt_probe(struct platform_device *pdev) if (!instr->reg) { dev_err(dev, "I/O resource not found\n"); - return -EINVAL; + ret = -EINVAL; + goto out_put_table; } instructions = wdat->instructions[action]; @@ -425,8 +441,10 @@ static int wdat_wdt_probe(struct platform_device *pdev) instructions = devm_kzalloc(dev, sizeof(*instructions), GFP_KERNEL); - if (!instructions) - return -ENOMEM; + if (!instructions) { + ret = -ENOMEM; + goto out_put_table; + } INIT_LIST_HEAD(instructions); wdat->instructions[action] = instructions; @@ -443,7 +461,7 @@ static int wdat_wdt_probe(struct platform_device *pdev) ret = wdat_wdt_enable_reboot(wdat); if (ret) - return ret; + goto out_put_table; platform_set_drvdata(pdev, wdat); @@ -460,12 +478,16 @@ static int wdat_wdt_probe(struct platform_device *pdev) ret = wdat_wdt_set_timeout(&wdat->wdd, timeout); if (ret) - return ret; + goto out_put_table; watchdog_set_nowayout(&wdat->wdd, nowayout); watchdog_stop_on_reboot(&wdat->wdd); watchdog_stop_on_unregister(&wdat->wdd); - return devm_watchdog_register_device(dev, &wdat->wdd); + ret = devm_watchdog_register_device(dev, &wdat->wdd); + +out_put_table: + acpi_put_table((struct acpi_table_header *)tbl); + return ret; } static int wdat_wdt_suspend_noirq(struct device *dev) diff --git a/drivers/xen/grant-dma-ops.c b/drivers/xen/grant-dma-ops.c index 29257d2639db..14077d23f2a1 100644 --- a/drivers/xen/grant-dma-ops.c +++ b/drivers/xen/grant-dma-ops.c @@ -163,18 +163,22 @@ static void xen_grant_dma_free_pages(struct device *dev, size_t size, xen_grant_dma_free(dev, size, page_to_virt(vaddr), dma_handle, 0); } -static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, +static dma_addr_t xen_grant_dma_map_phys(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, unsigned long attrs) { struct xen_grant_dma_data *data; + unsigned long offset = offset_in_page(phys); unsigned long dma_offset = xen_offset_in_page(offset), pfn_offset = XEN_PFN_DOWN(offset); unsigned int i, n_pages = XEN_PFN_UP(dma_offset + size); grant_ref_t grant; dma_addr_t dma_handle; + if (unlikely(attrs & DMA_ATTR_MMIO)) + return DMA_MAPPING_ERROR; + if (WARN_ON(dir == DMA_NONE)) return DMA_MAPPING_ERROR; @@ -190,7 +194,7 @@ static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page, for (i = 0; i < n_pages; i++) { gnttab_grant_foreign_access_ref(grant + i, data->backend_domid, - pfn_to_gfn(page_to_xen_pfn(page) + i + pfn_offset), + pfn_to_gfn(page_to_xen_pfn(phys_to_page(phys)) + i + pfn_offset), dir == DMA_TO_DEVICE); } @@ -199,7 +203,7 @@ static dma_addr_t xen_grant_dma_map_page(struct device *dev, struct page *page, return dma_handle; } -static void xen_grant_dma_unmap_page(struct device *dev, dma_addr_t dma_handle, +static void xen_grant_dma_unmap_phys(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { @@ -242,7 +246,7 @@ static void xen_grant_dma_unmap_sg(struct device *dev, struct scatterlist *sg, return; for_each_sg(sg, s, nents, i) - xen_grant_dma_unmap_page(dev, s->dma_address, sg_dma_len(s), dir, + xen_grant_dma_unmap_phys(dev, s->dma_address, sg_dma_len(s), dir, attrs); } @@ -257,7 +261,7 @@ static int xen_grant_dma_map_sg(struct device *dev, struct scatterlist *sg, return -EINVAL; for_each_sg(sg, s, nents, i) { - s->dma_address = xen_grant_dma_map_page(dev, sg_page(s), s->offset, + s->dma_address = xen_grant_dma_map_phys(dev, sg_phys(s), s->length, dir, attrs); if (s->dma_address == DMA_MAPPING_ERROR) goto out; @@ -286,8 +290,8 @@ static const struct dma_map_ops xen_grant_dma_ops = { .free_pages = xen_grant_dma_free_pages, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, - .map_page = xen_grant_dma_map_page, - .unmap_page = xen_grant_dma_unmap_page, + .map_phys = xen_grant_dma_map_phys, + .unmap_phys = xen_grant_dma_unmap_phys, .map_sg = xen_grant_dma_map_sg, .unmap_sg = xen_grant_dma_unmap_sg, .dma_supported = xen_grant_dma_supported, diff --git a/drivers/xen/grant-table.c b/drivers/xen/grant-table.c index 478d2ad725ac..3e76e33f6e08 100644 --- a/drivers/xen/grant-table.c +++ b/drivers/xen/grant-table.c @@ -1204,7 +1204,7 @@ void gnttab_foreach_grant_in_range(struct page *page, unsigned int glen; unsigned long xen_pfn; - len = min_t(unsigned int, PAGE_SIZE - offset, len); + len = min(PAGE_SIZE - offset, len); goffset = xen_offset_in_page(offset); xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(offset); diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index dd7747a2de87..ccf25027bec1 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -200,17 +200,32 @@ xen_swiotlb_free_coherent(struct device *dev, size_t size, void *vaddr, * physical address to use is returned. * * Once the device is given the dma address, the device owns this memory until - * either xen_swiotlb_unmap_page or xen_swiotlb_dma_sync_single is performed. + * either xen_swiotlb_unmap_phys or xen_swiotlb_dma_sync_single is performed. */ -static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, +static dma_addr_t xen_swiotlb_map_phys(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, unsigned long attrs) { - phys_addr_t map, phys = page_to_phys(page) + offset; - dma_addr_t dev_addr = xen_phys_to_dma(dev, phys); + dma_addr_t dev_addr; + phys_addr_t map; BUG_ON(dir == DMA_NONE); + + if (attrs & DMA_ATTR_MMIO) { + if (unlikely(!dma_capable(dev, phys, size, false))) { + dev_err_once( + dev, + "DMA addr %pa+%zu overflow (mask %llx, bus limit %llx).\n", + &phys, size, *dev->dma_mask, + dev->bus_dma_limit); + WARN_ON_ONCE(1); + return DMA_MAPPING_ERROR; + } + return phys; + } + + dev_addr = xen_phys_to_dma(dev, phys); + /* * If the address happens to be in the device's DMA window, * we can safely return the device addr and not worry about bounce @@ -257,13 +272,13 @@ done: /* * Unmap a single streaming mode DMA translation. The dma_addr and size must - * match what was provided for in a previous xen_swiotlb_map_page call. All + * match what was provided for in a previous xen_swiotlb_map_phys call. All * other usages are undefined. * * After this call, reads by the cpu to the buffer are guaranteed to see * whatever the device wrote there. */ -static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, +static void xen_swiotlb_unmap_phys(struct device *hwdev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { phys_addr_t paddr = xen_dma_to_phys(hwdev, dev_addr); @@ -325,7 +340,7 @@ xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, /* * Unmap a set of streaming mode DMA translations. Again, cpu read rules - * concerning calls here are the same as for swiotlb_unmap_page() above. + * concerning calls here are the same as for swiotlb_unmap_phys() above. */ static void xen_swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, @@ -337,7 +352,7 @@ xen_swiotlb_unmap_sg(struct device *hwdev, struct scatterlist *sgl, int nelems, BUG_ON(dir == DMA_NONE); for_each_sg(sgl, sg, nelems, i) - xen_swiotlb_unmap_page(hwdev, sg->dma_address, sg_dma_len(sg), + xen_swiotlb_unmap_phys(hwdev, sg->dma_address, sg_dma_len(sg), dir, attrs); } @@ -352,8 +367,8 @@ xen_swiotlb_map_sg(struct device *dev, struct scatterlist *sgl, int nelems, BUG_ON(dir == DMA_NONE); for_each_sg(sgl, sg, nelems, i) { - sg->dma_address = xen_swiotlb_map_page(dev, sg_page(sg), - sg->offset, sg->length, dir, attrs); + sg->dma_address = xen_swiotlb_map_phys(dev, sg_phys(sg), + sg->length, dir, attrs); if (sg->dma_address == DMA_MAPPING_ERROR) goto out_unmap; sg_dma_len(sg) = sg->length; @@ -392,25 +407,6 @@ xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl, } } -static dma_addr_t xen_swiotlb_direct_map_resource(struct device *dev, - phys_addr_t paddr, - size_t size, - enum dma_data_direction dir, - unsigned long attrs) -{ - dma_addr_t dma_addr = paddr; - - if (unlikely(!dma_capable(dev, dma_addr, size, false))) { - dev_err_once(dev, - "DMA addr %pad+%zu overflow (mask %llx, bus limit %llx).\n", - &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit); - WARN_ON_ONCE(1); - return DMA_MAPPING_ERROR; - } - - return dma_addr; -} - /* * Return whether the given device DMA address mask can be supported * properly. For example, if your device can only drive the low 24-bits @@ -437,13 +433,12 @@ const struct dma_map_ops xen_swiotlb_dma_ops = { .sync_sg_for_device = xen_swiotlb_sync_sg_for_device, .map_sg = xen_swiotlb_map_sg, .unmap_sg = xen_swiotlb_unmap_sg, - .map_page = xen_swiotlb_map_page, - .unmap_page = xen_swiotlb_unmap_page, + .map_phys = xen_swiotlb_map_phys, + .unmap_phys = xen_swiotlb_unmap_phys, .dma_supported = xen_swiotlb_dma_supported, .mmap = dma_common_mmap, .get_sgtable = dma_common_get_sgtable, .alloc_pages_op = dma_common_alloc_pages, .free_pages = dma_common_free_pages, .max_mapping_size = swiotlb_max_mapping_size, - .map_resource = xen_swiotlb_direct_map_resource, }; diff --git a/drivers/xen/xenbus/xenbus_xs.c b/drivers/xen/xenbus/xenbus_xs.c index f794014814be..15f18374020e 100644 --- a/drivers/xen/xenbus/xenbus_xs.c +++ b/drivers/xen/xenbus/xenbus_xs.c @@ -407,7 +407,7 @@ static char *join(const char *dir, const char *name) buffer = kasprintf(GFP_NOIO | __GFP_HIGH, "%s", dir); else buffer = kasprintf(GFP_NOIO | __GFP_HIGH, "%s/%s", dir, name); - return (!buffer) ? ERR_PTR(-ENOMEM) : buffer; + return buffer ?: ERR_PTR(-ENOMEM); } static char **split_strings(char *strings, unsigned int len, unsigned int *num) @@ -546,18 +546,12 @@ int xenbus_transaction_start(struct xenbus_transaction *t) EXPORT_SYMBOL_GPL(xenbus_transaction_start); /* End a transaction. - * If abandon is true, transaction is discarded instead of committed. + * If abort is true, transaction is discarded instead of committed. */ -int xenbus_transaction_end(struct xenbus_transaction t, int abort) +int xenbus_transaction_end(struct xenbus_transaction t, bool abort) { - char abortstr[2]; - - if (abort) - strcpy(abortstr, "F"); - else - strcpy(abortstr, "T"); - - return xs_error(xs_single(t, XS_TRANSACTION_END, abortstr, NULL)); + return xs_error(xs_single(t, XS_TRANSACTION_END, abort ? "F" : "T", + NULL)); } EXPORT_SYMBOL_GPL(xenbus_transaction_end); diff --git a/fs/ceph/crypto.c b/fs/ceph/crypto.c index 928746b92512..0ea4db650f85 100644 --- a/fs/ceph/crypto.c +++ b/fs/ceph/crypto.c @@ -15,59 +15,6 @@ #include "mds_client.h" #include "crypto.h" -/* - * The base64url encoding used by fscrypt includes the '_' character, which may - * cause problems in snapshot names (which can not start with '_'). Thus, we - * used the base64 encoding defined for IMAP mailbox names (RFC 3501) instead, - * which replaces '-' and '_' by '+' and ','. - */ -static const char base64_table[65] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,"; - -int ceph_base64_encode(const u8 *src, int srclen, char *dst) -{ - u32 ac = 0; - int bits = 0; - int i; - char *cp = dst; - - for (i = 0; i < srclen; i++) { - ac = (ac << 8) | src[i]; - bits += 8; - do { - bits -= 6; - *cp++ = base64_table[(ac >> bits) & 0x3f]; - } while (bits >= 6); - } - if (bits) - *cp++ = base64_table[(ac << (6 - bits)) & 0x3f]; - return cp - dst; -} - -int ceph_base64_decode(const char *src, int srclen, u8 *dst) -{ - u32 ac = 0; - int bits = 0; - int i; - u8 *bp = dst; - - for (i = 0; i < srclen; i++) { - const char *p = strchr(base64_table, src[i]); - - if (p == NULL || src[i] == 0) - return -1; - ac = (ac << 6) | (p - base64_table); - bits += 6; - if (bits >= 8) { - bits -= 8; - *bp++ = (u8)(ac >> bits); - } - } - if (ac & ((1 << bits) - 1)) - return -1; - return bp - dst; -} - static int ceph_crypt_get_context(struct inode *inode, void *ctx, size_t len) { struct ceph_inode_info *ci = ceph_inode(inode); @@ -318,7 +265,7 @@ int ceph_encode_encrypted_dname(struct inode *parent, char *buf, int elen) } /* base64 encode the encrypted name */ - elen = ceph_base64_encode(cryptbuf, len, p); + elen = base64_encode(cryptbuf, len, p, false, BASE64_IMAP); doutc(cl, "base64-encoded ciphertext name = %.*s\n", elen, p); /* To understand the 240 limit, see CEPH_NOHASH_NAME_MAX comments */ @@ -412,7 +359,8 @@ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, tname = &_tname; } - declen = ceph_base64_decode(name, name_len, tname->name); + declen = base64_decode(name, name_len, + tname->name, false, BASE64_IMAP); if (declen <= 0) { ret = -EIO; goto out; @@ -426,7 +374,7 @@ int ceph_fname_to_usr(const struct ceph_fname *fname, struct fscrypt_str *tname, ret = fscrypt_fname_disk_to_usr(dir, 0, 0, &iname, oname); if (!ret && (dir != fname->dir)) { - char tmp_buf[CEPH_BASE64_CHARS(NAME_MAX)]; + char tmp_buf[BASE64_CHARS(NAME_MAX)]; name_len = snprintf(tmp_buf, sizeof(tmp_buf), "_%.*s_%ld", oname->len, oname->name, dir->i_ino); diff --git a/fs/ceph/crypto.h b/fs/ceph/crypto.h index 23612b2e9837..b748e2060bc9 100644 --- a/fs/ceph/crypto.h +++ b/fs/ceph/crypto.h @@ -8,6 +8,7 @@ #include <crypto/sha2.h> #include <linux/fscrypt.h> +#include <linux/base64.h> #define CEPH_FSCRYPT_BLOCK_SHIFT 12 #define CEPH_FSCRYPT_BLOCK_SIZE (_AC(1, UL) << CEPH_FSCRYPT_BLOCK_SHIFT) @@ -89,11 +90,6 @@ static inline u32 ceph_fscrypt_auth_len(struct ceph_fscrypt_auth *fa) */ #define CEPH_NOHASH_NAME_MAX (180 - SHA256_DIGEST_SIZE) -#define CEPH_BASE64_CHARS(nbytes) DIV_ROUND_UP((nbytes) * 4, 3) - -int ceph_base64_encode(const u8 *src, int srclen, char *dst); -int ceph_base64_decode(const char *src, int srclen, u8 *dst); - void ceph_fscrypt_set_ops(struct super_block *sb); void ceph_fscrypt_free_dummy_policy(struct ceph_fs_client *fsc); diff --git a/fs/ceph/dir.c b/fs/ceph/dir.c index bf50c6e7a029..86d7aa594ea9 100644 --- a/fs/ceph/dir.c +++ b/fs/ceph/dir.c @@ -998,13 +998,14 @@ static int prep_encrypted_symlink_target(struct ceph_mds_request *req, if (err) goto out; - req->r_path2 = kmalloc(CEPH_BASE64_CHARS(osd_link.len) + 1, GFP_KERNEL); + req->r_path2 = kmalloc(BASE64_CHARS(osd_link.len) + 1, GFP_KERNEL); if (!req->r_path2) { err = -ENOMEM; goto out; } - len = ceph_base64_encode(osd_link.name, osd_link.len, req->r_path2); + len = base64_encode(osd_link.name, osd_link.len, + req->r_path2, false, BASE64_IMAP); req->r_path2[len] = '\0'; out: fscrypt_fname_free_buffer(&osd_link); diff --git a/fs/ceph/inode.c b/fs/ceph/inode.c index a596cb53f1ac..2966f88310e3 100644 --- a/fs/ceph/inode.c +++ b/fs/ceph/inode.c @@ -947,7 +947,7 @@ static int decode_encrypted_symlink(struct ceph_mds_client *mdsc, if (!sym) return -ENOMEM; - declen = ceph_base64_decode(encsym, enclen, sym); + declen = base64_decode(encsym, enclen, sym, false, BASE64_IMAP); if (declen < 0) { pr_err_client(cl, "can't decode symlink (%d). Content: %.*s\n", diff --git a/fs/crypto/fname.c b/fs/crypto/fname.c index 8e4c213d418b..a9a4432d12ba 100644 --- a/fs/crypto/fname.c +++ b/fs/crypto/fname.c @@ -16,6 +16,7 @@ #include <linux/export.h> #include <linux/namei.h> #include <linux/scatterlist.h> +#include <linux/base64.h> #include "fscrypt_private.h" @@ -71,7 +72,7 @@ struct fscrypt_nokey_name { /* Encoded size of max-size no-key name */ #define FSCRYPT_NOKEY_NAME_MAX_ENCODED \ - FSCRYPT_BASE64URL_CHARS(FSCRYPT_NOKEY_NAME_MAX) + BASE64_CHARS(FSCRYPT_NOKEY_NAME_MAX) static inline bool fscrypt_is_dot_dotdot(const struct qstr *str) { @@ -162,84 +163,6 @@ static int fname_decrypt(const struct inode *inode, return 0; } -static const char base64url_table[65] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; - -#define FSCRYPT_BASE64URL_CHARS(nbytes) DIV_ROUND_UP((nbytes) * 4, 3) - -/** - * fscrypt_base64url_encode() - base64url-encode some binary data - * @src: the binary data to encode - * @srclen: the length of @src in bytes - * @dst: (output) the base64url-encoded string. Not NUL-terminated. - * - * Encodes data using base64url encoding, i.e. the "Base 64 Encoding with URL - * and Filename Safe Alphabet" specified by RFC 4648. '='-padding isn't used, - * as it's unneeded and not required by the RFC. base64url is used instead of - * base64 to avoid the '/' character, which isn't allowed in filenames. - * - * Return: the length of the resulting base64url-encoded string in bytes. - * This will be equal to FSCRYPT_BASE64URL_CHARS(srclen). - */ -static int fscrypt_base64url_encode(const u8 *src, int srclen, char *dst) -{ - u32 ac = 0; - int bits = 0; - int i; - char *cp = dst; - - for (i = 0; i < srclen; i++) { - ac = (ac << 8) | src[i]; - bits += 8; - do { - bits -= 6; - *cp++ = base64url_table[(ac >> bits) & 0x3f]; - } while (bits >= 6); - } - if (bits) - *cp++ = base64url_table[(ac << (6 - bits)) & 0x3f]; - return cp - dst; -} - -/** - * fscrypt_base64url_decode() - base64url-decode a string - * @src: the string to decode. Doesn't need to be NUL-terminated. - * @srclen: the length of @src in bytes - * @dst: (output) the decoded binary data - * - * Decodes a string using base64url encoding, i.e. the "Base 64 Encoding with - * URL and Filename Safe Alphabet" specified by RFC 4648. '='-padding isn't - * accepted, nor are non-encoding characters such as whitespace. - * - * This implementation hasn't been optimized for performance. - * - * Return: the length of the resulting decoded binary data in bytes, - * or -1 if the string isn't a valid base64url string. - */ -static int fscrypt_base64url_decode(const char *src, int srclen, u8 *dst) -{ - u32 ac = 0; - int bits = 0; - int i; - u8 *bp = dst; - - for (i = 0; i < srclen; i++) { - const char *p = strchr(base64url_table, src[i]); - - if (p == NULL || src[i] == 0) - return -1; - ac = (ac << 6) | (p - base64url_table); - bits += 6; - if (bits >= 8) { - bits -= 8; - *bp++ = (u8)(ac >> bits); - } - } - if (ac & ((1 << bits) - 1)) - return -1; - return bp - dst; -} - bool __fscrypt_fname_encrypted_size(const union fscrypt_policy *policy, u32 orig_len, u32 max_len, u32 *encrypted_len_ret) @@ -387,8 +310,8 @@ int fscrypt_fname_disk_to_usr(const struct inode *inode, nokey_name.sha256); size = FSCRYPT_NOKEY_NAME_MAX; } - oname->len = fscrypt_base64url_encode((const u8 *)&nokey_name, size, - oname->name); + oname->len = base64_encode((const u8 *)&nokey_name, size, + oname->name, false, BASE64_URLSAFE); return 0; } EXPORT_SYMBOL(fscrypt_fname_disk_to_usr); @@ -467,8 +390,8 @@ int fscrypt_setup_filename(struct inode *dir, const struct qstr *iname, if (fname->crypto_buf.name == NULL) return -ENOMEM; - ret = fscrypt_base64url_decode(iname->name, iname->len, - fname->crypto_buf.name); + ret = base64_decode(iname->name, iname->len, + fname->crypto_buf.name, false, BASE64_URLSAFE); if (ret < (int)offsetof(struct fscrypt_nokey_name, bytes[1]) || (ret > offsetof(struct fscrypt_nokey_name, sha256) && ret != FSCRYPT_NOKEY_NAME_MAX)) { diff --git a/fs/lockd/svclock.c b/fs/lockd/svclock.c index a31dc9588eb8..3a3d05cfe09a 100644 --- a/fs/lockd/svclock.c +++ b/fs/lockd/svclock.c @@ -495,6 +495,9 @@ nlmsvc_lock(struct svc_rqst *rqstp, struct nlm_file *file, (long long)lock->fl.fl_end, wait); + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + if (!locks_can_async_lock(nlmsvc_file_file(file)->f_op)) { async_block = wait; wait = 0; @@ -621,6 +624,9 @@ nlmsvc_testlock(struct svc_rqst *rqstp, struct nlm_file *file, (long long)lock->fl.fl_start, (long long)lock->fl.fl_end); + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + if (locks_in_grace(SVC_NET(rqstp))) { ret = nlm_lck_denied_grace_period; goto out; @@ -678,6 +684,9 @@ nlmsvc_unlock(struct net *net, struct nlm_file *file, struct nlm_lock *lock) (long long)lock->fl.fl_start, (long long)lock->fl.fl_end); + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + /* First, cancel any lock that might be there */ nlmsvc_cancel_blocked(net, file, lock); @@ -715,6 +724,9 @@ nlmsvc_cancel_blocked(struct net *net, struct nlm_file *file, struct nlm_lock *l (long long)lock->fl.fl_start, (long long)lock->fl.fl_end); + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + if (locks_in_grace(net)) return nlm_lck_denied_grace_period; diff --git a/fs/lockd/svcshare.c b/fs/lockd/svcshare.c index ade4931b2da2..88c81ce1148d 100644 --- a/fs/lockd/svcshare.c +++ b/fs/lockd/svcshare.c @@ -32,6 +32,9 @@ nlmsvc_share_file(struct nlm_host *host, struct nlm_file *file, struct xdr_netobj *oh = &argp->lock.oh; u8 *ohdata; + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + for (share = file->f_shares; share; share = share->s_next) { if (share->s_host == host && nlm_cmp_owner(share, oh)) goto update; @@ -72,6 +75,9 @@ nlmsvc_unshare_file(struct nlm_host *host, struct nlm_file *file, struct nlm_share *share, **shpp; struct xdr_netobj *oh = &argp->lock.oh; + if (nlmsvc_file_cannot_lock(file)) + return nlm_lck_denied_nolocks; + for (shpp = &file->f_shares; (share = *shpp) != NULL; shpp = &share->s_next) { if (share->s_host == host && nlm_cmp_owner(share, oh)) { diff --git a/fs/nfsd/Kconfig b/fs/nfsd/Kconfig index e134dce45e35..0b5c1a0bf1cf 100644 --- a/fs/nfsd/Kconfig +++ b/fs/nfsd/Kconfig @@ -5,6 +5,7 @@ config NFSD depends on FILE_LOCKING depends on FSNOTIFY select CRC32 + select CRYPTO_LIB_MD5 if NFSD_LEGACY_CLIENT_TRACKING select CRYPTO_LIB_SHA256 if NFSD_V4 select LOCKD select SUNRPC @@ -77,8 +78,7 @@ config NFSD_V4 depends on NFSD && PROC_FS select FS_POSIX_ACL select RPCSEC_GSS_KRB5 - select CRYPTO - select CRYPTO_MD5 + select CRYPTO # required by RPCSEC_GSS_KRB5 select GRACE_PERIOD select NFS_V4_2_SSC_HELPER if NFS_V4_2 help @@ -164,7 +164,7 @@ config NFSD_V4_SECURITY_LABEL config NFSD_LEGACY_CLIENT_TRACKING bool "Support legacy NFSv4 client tracking methods (DEPRECATED)" depends on NFSD_V4 - default y + default n help The NFSv4 server needs to store a small amount of information on stable storage in order to handle state recovery after reboot. Most diff --git a/fs/nfsd/blocklayout.c b/fs/nfsd/blocklayout.c index fde5539cf6a6..afa16d7a8013 100644 --- a/fs/nfsd/blocklayout.c +++ b/fs/nfsd/blocklayout.c @@ -13,70 +13,49 @@ #include "pnfs.h" #include "filecache.h" #include "vfs.h" +#include "trace.h" #define NFSDDBG_FACILITY NFSDDBG_PNFS +/* + * Get an extent from the file system that starts at offset or below + * and may be shorter than the requested length. + */ static __be32 -nfsd4_block_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode, - const struct svc_fh *fhp, struct nfsd4_layoutget *args) +nfsd4_block_map_extent(struct inode *inode, const struct svc_fh *fhp, + u64 offset, u64 length, u32 iomode, u64 minlength, + struct pnfs_block_extent *bex) { - struct nfsd4_layout_seg *seg = &args->lg_seg; struct super_block *sb = inode->i_sb; - u32 block_size = i_blocksize(inode); - struct pnfs_block_extent *bex; struct iomap iomap; u32 device_generation = 0; int error; - if (locks_in_grace(SVC_NET(rqstp))) - return nfserr_grace; - - if (seg->offset & (block_size - 1)) { - dprintk("pnfsd: I/O misaligned\n"); - goto out_layoutunavailable; - } - - /* - * Some clients barf on non-zero block numbers for NONE or INVALID - * layouts, so make sure to zero the whole structure. - */ - error = -ENOMEM; - bex = kzalloc(sizeof(*bex), GFP_KERNEL); - if (!bex) - goto out_error; - args->lg_content = bex; - - error = sb->s_export_op->map_blocks(inode, seg->offset, seg->length, - &iomap, seg->iomode != IOMODE_READ, - &device_generation); + error = sb->s_export_op->map_blocks(inode, offset, length, &iomap, + iomode != IOMODE_READ, &device_generation); if (error) { if (error == -ENXIO) - goto out_layoutunavailable; - goto out_error; - } - - if (iomap.length < args->lg_minlength) { - dprintk("pnfsd: extent smaller than minlength\n"); - goto out_layoutunavailable; + return nfserr_layoutunavailable; + return nfserrno(error); } switch (iomap.type) { case IOMAP_MAPPED: - if (seg->iomode == IOMODE_READ) + if (iomode == IOMODE_READ) bex->es = PNFS_BLOCK_READ_DATA; else bex->es = PNFS_BLOCK_READWRITE_DATA; bex->soff = iomap.addr; break; case IOMAP_UNWRITTEN: - if (seg->iomode & IOMODE_RW) { + if (iomode & IOMODE_RW) { /* * Crack monkey special case from section 2.3.1. */ - if (args->lg_minlength == 0) { + if (minlength == 0) { dprintk("pnfsd: no soup for you!\n"); - goto out_layoutunavailable; + return nfserr_layoutunavailable; } bex->es = PNFS_BLOCK_INVALID_DATA; @@ -85,7 +64,7 @@ nfsd4_block_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode, } fallthrough; case IOMAP_HOLE: - if (seg->iomode == IOMODE_READ) { + if (iomode == IOMODE_READ) { bex->es = PNFS_BLOCK_NONE_DATA; break; } @@ -93,27 +72,107 @@ nfsd4_block_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode, case IOMAP_DELALLOC: default: WARN(1, "pnfsd: filesystem returned %d extent\n", iomap.type); - goto out_layoutunavailable; + return nfserr_layoutunavailable; } error = nfsd4_set_deviceid(&bex->vol_id, fhp, device_generation); if (error) - goto out_error; + return nfserrno(error); + bex->foff = iomap.offset; bex->len = iomap.length; + return nfs_ok; +} - seg->offset = iomap.offset; - seg->length = iomap.length; +static __be32 +nfsd4_block_proc_layoutget(struct svc_rqst *rqstp, struct inode *inode, + const struct svc_fh *fhp, struct nfsd4_layoutget *args) +{ + struct nfsd4_layout_seg *seg = &args->lg_seg; + struct pnfs_block_layout *bl; + struct pnfs_block_extent *first_bex, *last_bex; + u64 offset = seg->offset, length = seg->length; + u32 i, nr_extents_max, block_size = i_blocksize(inode); + __be32 nfserr; - dprintk("GET: 0x%llx:0x%llx %d\n", bex->foff, bex->len, bex->es); - return 0; + if (locks_in_grace(SVC_NET(rqstp))) + return nfserr_grace; + + nfserr = nfserr_layoutunavailable; + if (seg->offset & (block_size - 1)) { + dprintk("pnfsd: I/O misaligned\n"); + goto out_error; + } + + /* + * RFC 8881, section 3.3.17: + * The layout4 data type defines a layout for a file. + * + * RFC 8881, section 18.43.3: + * The loga_maxcount field specifies the maximum layout size + * (in bytes) that the client can handle. If the size of the + * layout structure exceeds the size specified by maxcount, + * the metadata server will return the NFS4ERR_TOOSMALL error. + */ + nfserr = nfserr_toosmall; + if (args->lg_maxcount < PNFS_BLOCK_LAYOUT4_SIZE + + PNFS_BLOCK_EXTENT_SIZE) + goto out_error; + + /* + * Limit the maximum layout size to avoid allocating + * a large buffer on the server for each layout request. + */ + nr_extents_max = (min(args->lg_maxcount, PAGE_SIZE) - + PNFS_BLOCK_LAYOUT4_SIZE) / PNFS_BLOCK_EXTENT_SIZE; + + /* + * Some clients barf on non-zero block numbers for NONE or INVALID + * layouts, so make sure to zero the whole structure. + */ + nfserr = nfserrno(-ENOMEM); + bl = kzalloc(struct_size(bl, extents, nr_extents_max), GFP_KERNEL); + if (!bl) + goto out_error; + bl->nr_extents = nr_extents_max; + args->lg_content = bl; + + for (i = 0; i < bl->nr_extents; i++) { + struct pnfs_block_extent *bex = bl->extents + i; + u64 bex_length; + + nfserr = nfsd4_block_map_extent(inode, fhp, offset, length, + seg->iomode, args->lg_minlength, bex); + if (nfserr != nfs_ok) + goto out_error; + + bex_length = bex->len - (offset - bex->foff); + if (bex_length >= length) { + bl->nr_extents = i + 1; + break; + } + + offset = bex->foff + bex->len; + length -= bex_length; + } + + first_bex = bl->extents; + last_bex = bl->extents + bl->nr_extents - 1; + + nfserr = nfserr_layoutunavailable; + length = last_bex->foff + last_bex->len - seg->offset; + if (length < args->lg_minlength) { + dprintk("pnfsd: extent smaller than minlength\n"); + goto out_error; + } + + seg->offset = first_bex->foff; + seg->length = last_bex->foff - first_bex->foff + last_bex->len; + return nfs_ok; out_error: seg->length = 0; - return nfserrno(error); -out_layoutunavailable: - seg->length = 0; - return nfserr_layoutunavailable; + return nfserr; } static __be32 @@ -340,9 +399,12 @@ nfsd4_scsi_fence_client(struct nfs4_layout_stateid *ls, struct nfsd_file *file) { struct nfs4_client *clp = ls->ls_stid.sc_client; struct block_device *bdev = file->nf_file->f_path.mnt->mnt_sb->s_bdev; + int status; - bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY, - nfsd4_scsi_pr_key(clp), 0, true); + status = bdev->bd_disk->fops->pr_ops->pr_preempt(bdev, NFSD_MDS_PR_KEY, + nfsd4_scsi_pr_key(clp), + PR_EXCLUSIVE_ACCESS_REG_ONLY, true); + trace_nfsd_pnfs_fence(clp, bdev->bd_disk->disk_name, status); } const struct nfsd4_layout_ops scsi_layout_ops = { diff --git a/fs/nfsd/blocklayoutxdr.c b/fs/nfsd/blocklayoutxdr.c index e50afe340737..196ef4245604 100644 --- a/fs/nfsd/blocklayoutxdr.c +++ b/fs/nfsd/blocklayoutxdr.c @@ -14,12 +14,25 @@ #define NFSDDBG_FACILITY NFSDDBG_PNFS +/** + * nfsd4_block_encode_layoutget - encode block/scsi layout extent array + * @xdr: stream for data encoding + * @lgp: layoutget content, actually an array of extents to encode + * + * Encode the opaque loc_body field in the layoutget response. Since the + * pnfs_block_layout4 and pnfs_scsi_layout4 structures on the wire are + * the same, this function is used by both layout drivers. + * + * Return values: + * %nfs_ok: Success, all extents encoded into @xdr + * %nfserr_toosmall: Not enough space in @xdr to encode all the data + */ __be32 nfsd4_block_encode_layoutget(struct xdr_stream *xdr, const struct nfsd4_layoutget *lgp) { - const struct pnfs_block_extent *b = lgp->lg_content; - int len = sizeof(__be32) + 5 * sizeof(__be64) + sizeof(__be32); + const struct pnfs_block_layout *bl = lgp->lg_content; + u32 i, len = sizeof(__be32) + bl->nr_extents * PNFS_BLOCK_EXTENT_SIZE; __be32 *p; p = xdr_reserve_space(xdr, sizeof(__be32) + len); @@ -27,14 +40,19 @@ nfsd4_block_encode_layoutget(struct xdr_stream *xdr, return nfserr_toosmall; *p++ = cpu_to_be32(len); - *p++ = cpu_to_be32(1); /* we always return a single extent */ + *p++ = cpu_to_be32(bl->nr_extents); - p = svcxdr_encode_deviceid4(p, &b->vol_id); - p = xdr_encode_hyper(p, b->foff); - p = xdr_encode_hyper(p, b->len); - p = xdr_encode_hyper(p, b->soff); - *p++ = cpu_to_be32(b->es); - return 0; + for (i = 0; i < bl->nr_extents; i++) { + const struct pnfs_block_extent *bex = bl->extents + i; + + p = svcxdr_encode_deviceid4(p, &bex->vol_id); + p = xdr_encode_hyper(p, bex->foff); + p = xdr_encode_hyper(p, bex->len); + p = xdr_encode_hyper(p, bex->soff); + *p++ = cpu_to_be32(bex->es); + } + + return nfs_ok; } static int diff --git a/fs/nfsd/blocklayoutxdr.h b/fs/nfsd/blocklayoutxdr.h index 7d25ef689671..2e0c6c7d2b42 100644 --- a/fs/nfsd/blocklayoutxdr.h +++ b/fs/nfsd/blocklayoutxdr.h @@ -8,6 +8,15 @@ struct iomap; struct xdr_stream; +/* On the wire size of the layout4 struct with zero number of extents */ +#define PNFS_BLOCK_LAYOUT4_SIZE \ + (sizeof(__be32) * 2 + /* offset4 */ \ + sizeof(__be32) * 2 + /* length4 */ \ + sizeof(__be32) + /* layoutiomode4 */ \ + sizeof(__be32) + /* layouttype4 */ \ + sizeof(__be32) + /* number of bytes */ \ + sizeof(__be32)) /* number of extents */ + struct pnfs_block_extent { struct nfsd4_deviceid vol_id; u64 foff; @@ -21,6 +30,11 @@ struct pnfs_block_range { u64 len; }; +struct pnfs_block_layout { + u32 nr_extents; + struct pnfs_block_extent extents[] __counted_by(nr_extents); +}; + /* * Random upper cap for the uuid length to avoid unbounded allocation. * Not actually limited by the protocol. diff --git a/fs/nfsd/debugfs.c b/fs/nfsd/debugfs.c index ed2b9e066206..7f44689e0a53 100644 --- a/fs/nfsd/debugfs.c +++ b/fs/nfsd/debugfs.c @@ -44,6 +44,7 @@ DEFINE_DEBUGFS_ATTRIBUTE(nfsd_dsr_fops, nfsd_dsr_get, nfsd_dsr_set, "%llu\n"); * Contents: * %0: NFS READ will use buffered IO * %1: NFS READ will use dontcache (buffered IO w/ dropbehind) + * %2: NFS READ will use direct IO * * This setting takes immediate effect for all NFS versions, * all exports, and in all NFSD net namespaces. @@ -64,6 +65,7 @@ static int nfsd_io_cache_read_set(void *data, u64 val) nfsd_io_cache_read = NFSD_IO_BUFFERED; break; case NFSD_IO_DONTCACHE: + case NFSD_IO_DIRECT: /* * Must disable splice_read when enabling * NFSD_IO_DONTCACHE. @@ -106,6 +108,7 @@ static int nfsd_io_cache_write_set(void *data, u64 val) switch (val) { case NFSD_IO_BUFFERED: case NFSD_IO_DONTCACHE: + case NFSD_IO_DIRECT: nfsd_io_cache_write = val; break; default: diff --git a/fs/nfsd/nfs4recover.c b/fs/nfsd/nfs4recover.c index b39d4cbdfd35..441dfbfe2d2b 100644 --- a/fs/nfsd/nfs4recover.c +++ b/fs/nfsd/nfs4recover.c @@ -32,7 +32,7 @@ * */ -#include <crypto/hash.h> +#include <crypto/md5.h> #include <crypto/sha2.h> #include <linux/file.h> #include <linux/slab.h> @@ -92,79 +92,29 @@ nfs4_reset_creds(const struct cred *original) put_cred(revert_creds(original)); } -static int +static void nfs4_make_rec_clidname(char dname[HEXDIR_LEN], const struct xdr_netobj *clname) { u8 digest[MD5_DIGEST_SIZE]; - struct crypto_shash *tfm; - int status; dprintk("NFSD: nfs4_make_rec_clidname for %.*s\n", clname->len, clname->data); - tfm = crypto_alloc_shash("md5", 0, 0); - if (IS_ERR(tfm)) { - status = PTR_ERR(tfm); - goto out_no_tfm; - } - status = crypto_shash_tfm_digest(tfm, clname->data, clname->len, - digest); - if (status) - goto out; + md5(clname->data, clname->len, digest); static_assert(HEXDIR_LEN == 2 * MD5_DIGEST_SIZE + 1); sprintf(dname, "%*phN", MD5_DIGEST_SIZE, digest); - - status = 0; -out: - crypto_free_shash(tfm); -out_no_tfm: - return status; -} - -/* - * If we had an error generating the recdir name for the legacy tracker - * then warn the admin. If the error doesn't appear to be transient, - * then disable recovery tracking. - */ -static void -legacy_recdir_name_error(struct nfs4_client *clp, int error) -{ - printk(KERN_ERR "NFSD: unable to generate recoverydir " - "name (%d).\n", error); - - /* - * if the algorithm just doesn't exist, then disable the recovery - * tracker altogether. The crypto libs will generally return this if - * FIPS is enabled as well. - */ - if (error == -ENOENT) { - printk(KERN_ERR "NFSD: disabling legacy clientid tracking. " - "Reboot recovery will not function correctly!\n"); - nfsd4_client_tracking_exit(clp->net); - } } static void __nfsd4_create_reclaim_record_grace(struct nfs4_client *clp, - const char *dname, int len, struct nfsd_net *nn) + char *dname, struct nfsd_net *nn) { - struct xdr_netobj name; + struct xdr_netobj name = { .len = strlen(dname), .data = dname }; struct xdr_netobj princhash = { .len = 0, .data = NULL }; struct nfs4_client_reclaim *crp; - name.data = kmemdup(dname, len, GFP_KERNEL); - if (!name.data) { - dprintk("%s: failed to allocate memory for name.data!\n", - __func__); - return; - } - name.len = len; crp = nfs4_client_to_reclaim(name, princhash, nn); - if (!crp) { - kfree(name.data); - return; - } crp->cr_clp = clp; } @@ -182,9 +132,7 @@ nfsd4_create_clid_dir(struct nfs4_client *clp) if (!nn->rec_file) return; - status = nfs4_make_rec_clidname(dname, &clp->cl_name); - if (status) - return legacy_recdir_name_error(clp, status); + nfs4_make_rec_clidname(dname, &clp->cl_name); status = nfs4_save_creds(&original_cred); if (status < 0) @@ -219,8 +167,7 @@ out_end: out: if (status == 0) { if (nn->in_grace) - __nfsd4_create_reclaim_record_grace(clp, dname, - HEXDIR_LEN, nn); + __nfsd4_create_reclaim_record_grace(clp, dname, nn); vfs_fsync(nn->rec_file, 0); } else { printk(KERN_ERR "NFSD: failed to write recovery record" @@ -233,7 +180,7 @@ out_creds: nfs4_reset_creds(original_cred); } -typedef int (recdir_func)(struct dentry *, struct dentry *, struct nfsd_net *); +typedef int (recdir_func)(struct dentry *, char *, struct nfsd_net *); struct name_list { char name[HEXDIR_LEN]; @@ -287,24 +234,14 @@ nfsd4_list_rec_dir(recdir_func *f, struct nfsd_net *nn) } status = iterate_dir(nn->rec_file, &ctx.ctx); - inode_lock_nested(d_inode(dir), I_MUTEX_PARENT); list_for_each_entry_safe(entry, tmp, &ctx.names, list) { - if (!status) { - struct dentry *dentry; - dentry = lookup_one(&nop_mnt_idmap, - &QSTR(entry->name), dir); - if (IS_ERR(dentry)) { - status = PTR_ERR(dentry); - break; - } - status = f(dir, dentry, nn); - dput(dentry); - } + if (!status) + status = f(dir, entry->name, nn); + list_del(&entry->list); kfree(entry); } - inode_unlock(d_inode(dir)); nfs4_reset_creds(original_cred); list_for_each_entry_safe(entry, tmp, &ctx.names, list) { @@ -364,9 +301,7 @@ nfsd4_remove_clid_dir(struct nfs4_client *clp) if (!nn->rec_file || !test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) return; - status = nfs4_make_rec_clidname(dname, &clp->cl_name); - if (status) - return legacy_recdir_name_error(clp, status); + nfs4_make_rec_clidname(dname, &clp->cl_name); status = mnt_want_write_file(nn->rec_file); if (status) @@ -394,18 +329,19 @@ out: } static int -purge_old(struct dentry *parent, struct dentry *child, struct nfsd_net *nn) +purge_old(struct dentry *parent, char *cname, struct nfsd_net *nn) { int status; + struct dentry *child; struct xdr_netobj name; - if (child->d_name.len != HEXDIR_LEN - 1) { - printk("%s: illegal name %pd in recovery directory\n", - __func__, child); + if (strlen(cname) != HEXDIR_LEN - 1) { + printk("%s: illegal name %s in recovery directory\n", + __func__, cname); /* Keep trying; maybe the others are OK: */ return 0; } - name.data = kmemdup_nul(child->d_name.name, child->d_name.len, GFP_KERNEL); + name.data = kstrdup(cname, GFP_KERNEL); if (!name.data) { dprintk("%s: failed to allocate memory for name.data!\n", __func__); @@ -415,10 +351,17 @@ purge_old(struct dentry *parent, struct dentry *child, struct nfsd_net *nn) if (nfs4_has_reclaimed_state(name, nn)) goto out_free; - status = vfs_rmdir(&nop_mnt_idmap, d_inode(parent), child, NULL); - if (status) - printk("failed to remove client recovery directory %pd\n", - child); + inode_lock_nested(d_inode(parent), I_MUTEX_PARENT); + child = lookup_one(&nop_mnt_idmap, &QSTR(cname), parent); + if (!IS_ERR(child)) { + status = vfs_rmdir(&nop_mnt_idmap, d_inode(parent), child, NULL); + if (status) + printk("failed to remove client recovery directory %pd\n", + child); + dput(child); + } + inode_unlock(d_inode(parent)); + out_free: kfree(name.data); out: @@ -449,27 +392,18 @@ out: } static int -load_recdir(struct dentry *parent, struct dentry *child, struct nfsd_net *nn) +load_recdir(struct dentry *parent, char *cname, struct nfsd_net *nn) { - struct xdr_netobj name; + struct xdr_netobj name = { .len = HEXDIR_LEN, .data = cname }; struct xdr_netobj princhash = { .len = 0, .data = NULL }; - if (child->d_name.len != HEXDIR_LEN - 1) { - printk("%s: illegal name %pd in recovery directory\n", - __func__, child); + if (strlen(cname) != HEXDIR_LEN - 1) { + printk("%s: illegal name %s in recovery directory\n", + __func__, cname); /* Keep trying; maybe the others are OK: */ return 0; } - name.data = kmemdup_nul(child->d_name.name, child->d_name.len, GFP_KERNEL); - if (!name.data) { - dprintk("%s: failed to allocate memory for name.data!\n", - __func__); - goto out; - } - name.len = HEXDIR_LEN; - if (!nfs4_client_to_reclaim(name, princhash, nn)) - kfree(name.data); -out: + nfs4_client_to_reclaim(name, princhash, nn); return 0; } @@ -647,7 +581,6 @@ nfs4_recoverydir(void) static int nfsd4_check_legacy_client(struct nfs4_client *clp) { - int status; char dname[HEXDIR_LEN]; struct nfs4_client_reclaim *crp; struct nfsd_net *nn = net_generic(clp->net, nfsd_net_id); @@ -657,11 +590,7 @@ nfsd4_check_legacy_client(struct nfs4_client *clp) if (test_bit(NFSD4_CLIENT_STABLE, &clp->cl_flags)) return 0; - status = nfs4_make_rec_clidname(dname, &clp->cl_name); - if (status) { - legacy_recdir_name_error(clp, status); - return status; - } + nfs4_make_rec_clidname(dname, &clp->cl_name); /* look for it in the reclaim hashtable otherwise */ name.data = kmemdup(dname, HEXDIR_LEN, GFP_KERNEL); @@ -767,6 +696,8 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, { uint8_t cmd, princhashlen; struct xdr_netobj name, princhash = { .len = 0, .data = NULL }; + char *namecopy __free(kfree) = NULL; + char *princhashcopy __free(kfree) = NULL; uint16_t namelen; if (get_user(cmd, &cmsg->cm_cmd)) { @@ -784,19 +715,19 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, dprintk("%s: invalid namelen (%u)", __func__, namelen); return -EINVAL; } - name.data = memdup_user(&ci->cc_name.cn_id, namelen); - if (IS_ERR(name.data)) - return PTR_ERR(name.data); + namecopy = memdup_user(&ci->cc_name.cn_id, namelen); + if (IS_ERR(namecopy)) + return PTR_ERR(namecopy); + name.data = namecopy; name.len = namelen; get_user(princhashlen, &ci->cc_princhash.cp_len); if (princhashlen > 0) { - princhash.data = memdup_user( - &ci->cc_princhash.cp_data, - princhashlen); - if (IS_ERR(princhash.data)) { - kfree(name.data); - return PTR_ERR(princhash.data); - } + princhashcopy = memdup_user( + &ci->cc_princhash.cp_data, + princhashlen); + if (IS_ERR(princhashcopy)) + return PTR_ERR(princhashcopy); + princhash.data = princhashcopy; princhash.len = princhashlen; } else princhash.len = 0; @@ -810,9 +741,10 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, dprintk("%s: invalid namelen (%u)", __func__, namelen); return -EINVAL; } - name.data = memdup_user(&cnm->cn_id, namelen); - if (IS_ERR(name.data)) - return PTR_ERR(name.data); + namecopy = memdup_user(&cnm->cn_id, namelen); + if (IS_ERR(namecopy)) + return PTR_ERR(namecopy); + name.data = namecopy; name.len = namelen; } #ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING @@ -820,15 +752,12 @@ __cld_pipe_inprogress_downcall(const struct cld_msg_v2 __user *cmsg, struct cld_net *cn = nn->cld_net; name.len = name.len - 5; - memmove(name.data, name.data + 5, name.len); + name.data = name.data + 5; cn->cn_has_legacy = true; } #endif - if (!nfs4_client_to_reclaim(name, princhash, nn)) { - kfree(name.data); - kfree(princhash.data); + if (!nfs4_client_to_reclaim(name, princhash, nn)) return -EFAULT; - } return nn->client_tracking_ops->msglen; } return -EFAULT; @@ -1254,13 +1183,10 @@ nfsd4_cld_check(struct nfs4_client *clp) #ifdef CONFIG_NFSD_LEGACY_CLIENT_TRACKING if (nn->cld_net->cn_has_legacy) { - int status; char dname[HEXDIR_LEN]; struct xdr_netobj name; - status = nfs4_make_rec_clidname(dname, &clp->cl_name); - if (status) - return -ENOENT; + nfs4_make_rec_clidname(dname, &clp->cl_name); name.data = kmemdup(dname, HEXDIR_LEN, GFP_KERNEL); if (!name.data) { @@ -1305,11 +1231,8 @@ nfsd4_cld_check_v2(struct nfs4_client *clp) if (cn->cn_has_legacy) { struct xdr_netobj name; char dname[HEXDIR_LEN]; - int status; - status = nfs4_make_rec_clidname(dname, &clp->cl_name); - if (status) - return -ENOENT; + nfs4_make_rec_clidname(dname, &clp->cl_name); name.data = kmemdup(dname, HEXDIR_LEN, GFP_KERNEL); if (!name.data) { @@ -1682,11 +1605,7 @@ nfsd4_cltrack_legacy_recdir(const struct xdr_netobj *name) return NULL; } - copied = nfs4_make_rec_clidname(result + copied, name); - if (copied) { - kfree(result); - return NULL; - } + nfs4_make_rec_clidname(result + copied, name); return result; } diff --git a/fs/nfsd/nfs4state.c b/fs/nfsd/nfs4state.c index 6791fc239dbd..808c24fb5c9a 100644 --- a/fs/nfsd/nfs4state.c +++ b/fs/nfsd/nfs4state.c @@ -3508,7 +3508,7 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp) free_svc_cred(&slot->sl_cred); copy_cred(&slot->sl_cred, &resp->rqstp->rq_cred); - if (!nfsd4_cache_this(resp)) { + if (!(resp->cstate.slot->sl_flags & NFSD4_SLOT_CACHETHIS)) { slot->sl_flags &= ~NFSD4_SLOT_CACHED; return; } @@ -3523,41 +3523,6 @@ nfsd4_store_cache_entry(struct nfsd4_compoundres *resp) } /* - * Encode the replay sequence operation from the slot values. - * If cachethis is FALSE encode the uncached rep error on the next - * operation which sets resp->p and increments resp->opcnt for - * nfs4svc_encode_compoundres. - * - */ -static __be32 -nfsd4_enc_sequence_replay(struct nfsd4_compoundargs *args, - struct nfsd4_compoundres *resp) -{ - struct nfsd4_op *op; - struct nfsd4_slot *slot = resp->cstate.slot; - - /* Encode the replayed sequence operation */ - op = &args->ops[resp->opcnt - 1]; - nfsd4_encode_operation(resp, op); - - if (slot->sl_flags & NFSD4_SLOT_CACHED) - return op->status; - if (args->opcnt == 1) { - /* - * The original operation wasn't a solo sequence--we - * always cache those--so this retry must not match the - * original: - */ - op->status = nfserr_seq_false_retry; - } else { - op = &args->ops[resp->opcnt++]; - op->status = nfserr_retry_uncached_rep; - nfsd4_encode_operation(resp, op); - } - return op->status; -} - -/* * The sequence operation is not cached because we can use the slot and * session values. */ @@ -3565,17 +3530,30 @@ static __be32 nfsd4_replay_cache_entry(struct nfsd4_compoundres *resp, struct nfsd4_sequence *seq) { + struct nfsd4_compoundargs *args = resp->rqstp->rq_argp; struct nfsd4_slot *slot = resp->cstate.slot; struct xdr_stream *xdr = resp->xdr; __be32 *p; - __be32 status; dprintk("--> %s slot %p\n", __func__, slot); - status = nfsd4_enc_sequence_replay(resp->rqstp->rq_argp, resp); - if (status) - return status; + /* Always encode the SEQUENCE response. */ + nfsd4_encode_operation(resp, &args->ops[0]); + if (args->opcnt == 1) + /* A solo SEQUENCE - nothing was cached */ + return args->ops[0].status; + if (!(slot->sl_flags & NFSD4_SLOT_CACHED)) { + /* We weren't asked to cache this. */ + struct nfsd4_op *op; + + op = &args->ops[resp->opcnt++]; + op->status = nfserr_retry_uncached_rep; + nfsd4_encode_operation(resp, op); + return op->status; + } + + /* return reply from cache */ p = xdr_reserve_space(xdr, slot->sl_datalen); if (!p) { WARN_ON_ONCE(1); @@ -6362,11 +6340,6 @@ nfs4_open_delegation(struct svc_rqst *rqstp, struct nfsd4_open *open, return; out_no_deleg: open->op_delegate_type = OPEN_DELEGATE_NONE; - if (open->op_claim_type == NFS4_OPEN_CLAIM_PREVIOUS && - open->op_delegate_type != OPEN_DELEGATE_NONE) { - dprintk("NFSD: WARNING: refusing delegation reclaim\n"); - open->op_recall = true; - } /* 4.1 client asking for a delegation? */ if (open->op_deleg_want) @@ -8802,9 +8775,6 @@ nfs4_has_reclaimed_state(struct xdr_netobj name, struct nfsd_net *nn) /* * failure => all reset bets are off, nfserr_no_grace... - * - * The caller is responsible for freeing name.data if NULL is returned (it - * will be freed in nfs4_remove_reclaim_record in the normal case). */ struct nfs4_client_reclaim * nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, @@ -8813,6 +8783,22 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, unsigned int strhashval; struct nfs4_client_reclaim *crp; + name.data = kmemdup(name.data, name.len, GFP_KERNEL); + if (!name.data) { + dprintk("%s: failed to allocate memory for name.data!\n", + __func__); + return NULL; + } + if (princhash.len) { + princhash.data = kmemdup(princhash.data, princhash.len, GFP_KERNEL); + if (!princhash.data) { + dprintk("%s: failed to allocate memory for princhash.data!\n", + __func__); + kfree(name.data); + return NULL; + } + } else + princhash.data = NULL; crp = alloc_reclaim(); if (crp) { strhashval = clientstr_hashval(name); @@ -8824,6 +8810,9 @@ nfs4_client_to_reclaim(struct xdr_netobj name, struct xdr_netobj princhash, crp->cr_princhash.len = princhash.len; crp->cr_clp = NULL; nn->reclaim_str_hashtbl_size++; + } else { + kfree(name.data); + kfree(princhash.data); } return crp; } diff --git a/fs/nfsd/nfs4xdr.c b/fs/nfsd/nfs4xdr.c index 67bb9c0b9fcb..30ce5851fe4c 100644 --- a/fs/nfsd/nfs4xdr.c +++ b/fs/nfsd/nfs4xdr.c @@ -4472,7 +4472,7 @@ out_err: static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp, struct nfsd4_read *read, - struct file *file, unsigned long maxcount) + unsigned long maxcount) { struct xdr_stream *xdr = resp->xdr; unsigned int base = xdr->buf->page_len & ~PAGE_MASK; @@ -4480,18 +4480,30 @@ static __be32 nfsd4_encode_readv(struct nfsd4_compoundres *resp, __be32 zero = xdr_zero; __be32 nfserr; - if (xdr_reserve_space_vec(xdr, maxcount) < 0) - return nfserr_resource; - - nfserr = nfsd_iter_read(resp->rqstp, read->rd_fhp, file, + nfserr = nfsd_iter_read(resp->rqstp, read->rd_fhp, read->rd_nf, read->rd_offset, &maxcount, base, &read->rd_eof); read->rd_length = maxcount; if (nfserr) return nfserr; + + /* + * svcxdr_encode_opaque_pages() is not used here because + * we don't want to encode subsequent results in this + * COMPOUND into the xdr->buf's tail, but rather those + * results should follow the NFS READ payload in the + * buf's pages. + */ + if (xdr_reserve_space_vec(xdr, maxcount) < 0) + return nfserr_resource; + + /* + * Mark the buffer location of the NFS READ payload so that + * direct placement-capable transports send only the + * payload bytes out-of-band. + */ if (svc_encode_result_payload(resp->rqstp, starting_len, maxcount)) return nfserr_io; - xdr_truncate_encode(xdr, starting_len + xdr_align_size(maxcount)); write_bytes_to_xdr_buf(xdr->buf, starting_len + maxcount, &zero, xdr_pad_size(maxcount)); @@ -4530,7 +4542,7 @@ nfsd4_encode_read(struct nfsd4_compoundres *resp, __be32 nfserr, if (file->f_op->splice_read && splice_ok) nfserr = nfsd4_encode_splice_read(resp, read, file, maxcount); else - nfserr = nfsd4_encode_readv(resp, read, file, maxcount); + nfserr = nfsd4_encode_readv(resp, read, maxcount); if (nfserr) { xdr_truncate_encode(xdr, eof_offset); return nfserr; @@ -5426,7 +5438,7 @@ nfsd4_encode_read_plus_data(struct nfsd4_compoundres *resp, if (file->f_op->splice_read && splice_ok) nfserr = nfsd4_encode_splice_read(resp, read, file, maxcount); else - nfserr = nfsd4_encode_readv(resp, read, file, maxcount); + nfserr = nfsd4_encode_readv(resp, read, maxcount); if (nfserr) return nfserr; diff --git a/fs/nfsd/nfsd.h b/fs/nfsd/nfsd.h index b752433c3c2c..e4263326ca4a 100644 --- a/fs/nfsd/nfsd.h +++ b/fs/nfsd/nfsd.h @@ -160,6 +160,7 @@ enum { /* Any new NFSD_IO enum value must be added at the end */ NFSD_IO_BUFFERED, NFSD_IO_DONTCACHE, + NFSD_IO_DIRECT, }; extern u64 nfsd_io_cache_read __read_mostly; @@ -397,14 +398,13 @@ enum { #define NFSD_CB_GETATTR_TIMEOUT NFSD_DELEGRETURN_TIMEOUT /* - * The following attributes are currently not supported by the NFSv4 server: + * The following attributes are not implemented by NFSD: * ARCHIVE (deprecated anyway) * HIDDEN (unlikely to be supported any time soon) * MIMETYPE (unlikely to be supported any time soon) * QUOTA_* (will be supported in a forthcoming patch) * SYSTEM (unlikely to be supported any time soon) * TIME_BACKUP (unlikely to be supported any time soon) - * TIME_CREATE (unlikely to be supported any time soon) */ #define NFSD4_SUPPORTED_ATTRS_WORD0 \ (FATTR4_WORD0_SUPPORTED_ATTRS | FATTR4_WORD0_TYPE | FATTR4_WORD0_FH_EXPIRE_TYPE \ diff --git a/fs/nfsd/nfssvc.c b/fs/nfsd/nfssvc.c index 7057ddd7a0a8..b08ae85d53ef 100644 --- a/fs/nfsd/nfssvc.c +++ b/fs/nfsd/nfssvc.c @@ -249,27 +249,6 @@ int nfsd_nrthreads(struct net *net) return rv; } -static int nfsd_init_socks(struct net *net, const struct cred *cred) -{ - int error; - struct nfsd_net *nn = net_generic(net, nfsd_net_id); - - if (!list_empty(&nn->nfsd_serv->sv_permsocks)) - return 0; - - error = svc_xprt_create(nn->nfsd_serv, "udp", net, PF_INET, NFS_PORT, - SVC_SOCK_DEFAULTS, cred); - if (error < 0) - return error; - - error = svc_xprt_create(nn->nfsd_serv, "tcp", net, PF_INET, NFS_PORT, - SVC_SOCK_DEFAULTS, cred); - if (error < 0) - return error; - - return 0; -} - static int nfsd_users = 0; static int nfsd_startup_generic(void) @@ -377,9 +356,12 @@ static int nfsd_startup_net(struct net *net, const struct cred *cred) ret = nfsd_startup_generic(); if (ret) return ret; - ret = nfsd_init_socks(net, cred); - if (ret) + + if (list_empty(&nn->nfsd_serv->sv_permsocks)) { + pr_warn("NFSD: Failed to start, no listeners configured.\n"); + ret = -EIO; goto out_socks; + } if (nfsd_needs_lockd(nn) && !nn->lockd_up) { ret = lockd_up(net, cred); diff --git a/fs/nfsd/trace.h b/fs/nfsd/trace.h index 6e2c8e2aab10..5ae2a611e57f 100644 --- a/fs/nfsd/trace.h +++ b/fs/nfsd/trace.h @@ -464,10 +464,13 @@ DEFINE_EVENT(nfsd_io_class, nfsd_##name, \ DEFINE_NFSD_IO_EVENT(read_start); DEFINE_NFSD_IO_EVENT(read_splice); DEFINE_NFSD_IO_EVENT(read_vector); +DEFINE_NFSD_IO_EVENT(read_direct); DEFINE_NFSD_IO_EVENT(read_io_done); DEFINE_NFSD_IO_EVENT(read_done); DEFINE_NFSD_IO_EVENT(write_start); DEFINE_NFSD_IO_EVENT(write_opened); +DEFINE_NFSD_IO_EVENT(write_direct); +DEFINE_NFSD_IO_EVENT(write_vector); DEFINE_NFSD_IO_EVENT(write_io_done); DEFINE_NFSD_IO_EVENT(write_done); DEFINE_NFSD_IO_EVENT(commit_start); @@ -2613,6 +2616,44 @@ DEFINE_EVENT(nfsd_vfs_getattr_class, __name, \ DEFINE_NFSD_VFS_GETATTR_EVENT(nfsd_vfs_getattr); DEFINE_NFSD_VFS_GETATTR_EVENT(nfsd_vfs_statfs); +DECLARE_EVENT_CLASS(nfsd_pnfs_class, + TP_PROTO( + const struct nfs4_client *clp, + const char *dev, + int error + ), + TP_ARGS(clp, dev, error), + TP_STRUCT__entry( + __sockaddr(addr, sizeof(struct sockaddr_in6)) + __field(unsigned int, netns_ino) + __string(dev, dev) + __field(int, error) + ), + TP_fast_assign( + __assign_sockaddr(addr, &clp->cl_addr, + sizeof(struct sockaddr_in6)); + __entry->netns_ino = clp->net->ns.inum; + __assign_str(dev); + __entry->error = error; + ), + TP_printk("client=%pISpc nn=%d dev=%s error=%d", + __get_sockaddr(addr), + __entry->netns_ino, + __get_str(dev), + __entry->error + ) +); + +#define DEFINE_NFSD_PNFS_ERR_EVENT(name) \ +DEFINE_EVENT(nfsd_pnfs_class, nfsd_pnfs_##name, \ + TP_PROTO( \ + const struct nfs4_client *clp, \ + const char *dev, \ + int error \ + ), \ + TP_ARGS(clp, dev, error)) + +DEFINE_NFSD_PNFS_ERR_EVENT(fence); #endif /* _NFSD_TRACE_H */ #undef TRACE_INCLUDE_PATH diff --git a/fs/nfsd/vfs.c b/fs/nfsd/vfs.c index 31cbf46b47b1..964cf922ad83 100644 --- a/fs/nfsd/vfs.c +++ b/fs/nfsd/vfs.c @@ -1075,11 +1075,88 @@ __be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp, return nfsd_finish_read(rqstp, fhp, file, offset, count, eof, host_err); } +/* + * The byte range of the client's READ request is expanded on both ends + * until it meets the underlying file system's direct I/O alignment + * requirements. After the internal read is complete, the byte range of + * the NFS READ payload is reduced to the byte range that was originally + * requested. + * + * Note that a direct read can be done only when the xdr_buf containing + * the NFS READ reply does not already have contents in its .pages array. + * This is due to potentially restrictive alignment requirements on the + * read buffer. When .page_len and @base are zero, the .pages array is + * guaranteed to be page-aligned. + */ +static noinline_for_stack __be32 +nfsd_direct_read(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file *nf, loff_t offset, unsigned long *count, + u32 *eof) +{ + u64 dio_start, dio_end; + unsigned long v, total; + struct iov_iter iter; + struct kiocb kiocb; + ssize_t host_err; + size_t len; + + init_sync_kiocb(&kiocb, nf->nf_file); + kiocb.ki_flags |= IOCB_DIRECT; + + /* Read a properly-aligned region of bytes into rq_bvec */ + dio_start = round_down(offset, nf->nf_dio_read_offset_align); + dio_end = round_up((u64)offset + *count, nf->nf_dio_read_offset_align); + + kiocb.ki_pos = dio_start; + + v = 0; + total = dio_end - dio_start; + while (total && v < rqstp->rq_maxpages && + rqstp->rq_next_page < rqstp->rq_page_end) { + len = min_t(size_t, total, PAGE_SIZE); + bvec_set_page(&rqstp->rq_bvec[v], *rqstp->rq_next_page, + len, 0); + + total -= len; + ++rqstp->rq_next_page; + ++v; + } + + trace_nfsd_read_direct(rqstp, fhp, offset, *count - total); + iov_iter_bvec(&iter, ITER_DEST, rqstp->rq_bvec, v, + dio_end - dio_start - total); + + host_err = vfs_iocb_iter_read(nf->nf_file, &kiocb, &iter); + if (host_err >= 0) { + unsigned int pad = offset - dio_start; + + /* The returned payload starts after the pad */ + rqstp->rq_res.page_base = pad; + + /* Compute the count of bytes to be returned */ + if (host_err > pad + *count) + host_err = *count; + else if (host_err > pad) + host_err -= pad; + else + host_err = 0; + } else if (unlikely(host_err == -EINVAL)) { + struct inode *inode = d_inode(fhp->fh_dentry); + + pr_info_ratelimited("nfsd: Direct I/O alignment failure on %s/%ld\n", + inode->i_sb->s_id, inode->i_ino); + host_err = -ESERVERFAULT; + } + + return nfsd_finish_read(rqstp, fhp, nf->nf_file, offset, count, + eof, host_err); +} + /** * nfsd_iter_read - Perform a VFS read using an iterator * @rqstp: RPC transaction context * @fhp: file handle of file to be read - * @file: opened struct file of file to be read + * @nf: opened struct nfsd_file of file to be read * @offset: starting byte offset * @count: IN: requested number of bytes; OUT: number of bytes read * @base: offset in first page of read buffer @@ -1092,9 +1169,10 @@ __be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp, * returned. */ __be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp, - struct file *file, loff_t offset, unsigned long *count, + struct nfsd_file *nf, loff_t offset, unsigned long *count, unsigned int base, u32 *eof) { + struct file *file = nf->nf_file; unsigned long v, total; struct iov_iter iter; struct kiocb kiocb; @@ -1106,6 +1184,12 @@ __be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp, switch (nfsd_io_cache_read) { case NFSD_IO_BUFFERED: break; + case NFSD_IO_DIRECT: + /* When dio_read_offset_align is zero, dio is not supported */ + if (nf->nf_dio_read_offset_align && !rqstp->rq_res.page_len) + return nfsd_direct_read(rqstp, fhp, nf, offset, + count, eof); + fallthrough; case NFSD_IO_DONTCACHE: if (file->f_op->fop_flags & FOP_DONTCACHE) kiocb.ki_flags = IOCB_DONTCACHE; @@ -1116,18 +1200,20 @@ __be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp, v = 0; total = *count; - while (total) { + while (total && v < rqstp->rq_maxpages && + rqstp->rq_next_page < rqstp->rq_page_end) { len = min_t(size_t, total, PAGE_SIZE - base); - bvec_set_page(&rqstp->rq_bvec[v], *(rqstp->rq_next_page++), + bvec_set_page(&rqstp->rq_bvec[v], *rqstp->rq_next_page, len, base); + total -= len; + ++rqstp->rq_next_page; ++v; base = 0; } - WARN_ON_ONCE(v > rqstp->rq_maxpages); - trace_nfsd_read_vector(rqstp, fhp, offset, *count); - iov_iter_bvec(&iter, ITER_DEST, rqstp->rq_bvec, v, *count); + trace_nfsd_read_vector(rqstp, fhp, offset, *count - total); + iov_iter_bvec(&iter, ITER_DEST, rqstp->rq_bvec, v, *count - total); host_err = vfs_iocb_iter_read(file, &kiocb, &iter); return nfsd_finish_read(rqstp, fhp, file, offset, count, eof, host_err); } @@ -1169,6 +1255,136 @@ static int wait_for_concurrent_writes(struct file *file) return err; } +struct nfsd_write_dio_seg { + struct iov_iter iter; + int flags; +}; + +static unsigned long +iov_iter_bvec_offset(const struct iov_iter *iter) +{ + return (unsigned long)(iter->bvec->bv_offset + iter->iov_offset); +} + +static void +nfsd_write_dio_seg_init(struct nfsd_write_dio_seg *segment, + struct bio_vec *bvec, unsigned int nvecs, + unsigned long total, size_t start, size_t len, + struct kiocb *iocb) +{ + iov_iter_bvec(&segment->iter, ITER_SOURCE, bvec, nvecs, total); + if (start) + iov_iter_advance(&segment->iter, start); + iov_iter_truncate(&segment->iter, len); + segment->flags = iocb->ki_flags; +} + +static unsigned int +nfsd_write_dio_iters_init(struct nfsd_file *nf, struct bio_vec *bvec, + unsigned int nvecs, struct kiocb *iocb, + unsigned long total, + struct nfsd_write_dio_seg segments[3]) +{ + u32 offset_align = nf->nf_dio_offset_align; + loff_t prefix_end, orig_end, middle_end; + u32 mem_align = nf->nf_dio_mem_align; + size_t prefix, middle, suffix; + loff_t offset = iocb->ki_pos; + unsigned int nsegs = 0; + + /* + * Check if direct I/O is feasible for this write request. + * If alignments are not available, the write is too small, + * or no alignment can be found, fall back to buffered I/O. + */ + if (unlikely(!mem_align || !offset_align) || + unlikely(total < max(offset_align, mem_align))) + goto no_dio; + + prefix_end = round_up(offset, offset_align); + orig_end = offset + total; + middle_end = round_down(orig_end, offset_align); + + prefix = prefix_end - offset; + middle = middle_end - prefix_end; + suffix = orig_end - middle_end; + + if (!middle) + goto no_dio; + + if (prefix) + nfsd_write_dio_seg_init(&segments[nsegs++], bvec, + nvecs, total, 0, prefix, iocb); + + nfsd_write_dio_seg_init(&segments[nsegs], bvec, nvecs, + total, prefix, middle, iocb); + + /* + * Check if the bvec iterator is aligned for direct I/O. + * + * bvecs generated from RPC receive buffers are contiguous: After + * the first bvec, all subsequent bvecs start at bv_offset zero + * (page-aligned). Therefore, only the first bvec is checked. + */ + if (iov_iter_bvec_offset(&segments[nsegs].iter) & (mem_align - 1)) + goto no_dio; + segments[nsegs].flags |= IOCB_DIRECT; + nsegs++; + + if (suffix) + nfsd_write_dio_seg_init(&segments[nsegs++], bvec, nvecs, total, + prefix + middle, suffix, iocb); + + return nsegs; + +no_dio: + /* No DIO alignment possible - pack into single non-DIO segment. */ + nfsd_write_dio_seg_init(&segments[0], bvec, nvecs, total, 0, + total, iocb); + return 1; +} + +static noinline_for_stack int +nfsd_direct_write(struct svc_rqst *rqstp, struct svc_fh *fhp, + struct nfsd_file *nf, unsigned int nvecs, + unsigned long *cnt, struct kiocb *kiocb) +{ + struct nfsd_write_dio_seg segments[3]; + struct file *file = nf->nf_file; + unsigned int nsegs, i; + ssize_t host_err; + + nsegs = nfsd_write_dio_iters_init(nf, rqstp->rq_bvec, nvecs, + kiocb, *cnt, segments); + + *cnt = 0; + for (i = 0; i < nsegs; i++) { + kiocb->ki_flags = segments[i].flags; + if (kiocb->ki_flags & IOCB_DIRECT) + trace_nfsd_write_direct(rqstp, fhp, kiocb->ki_pos, + segments[i].iter.count); + else { + trace_nfsd_write_vector(rqstp, fhp, kiocb->ki_pos, + segments[i].iter.count); + /* + * Mark the I/O buffer as evict-able to reduce + * memory contention. + */ + if (nf->nf_file->f_op->fop_flags & FOP_DONTCACHE) + kiocb->ki_flags |= IOCB_DONTCACHE; + } + + host_err = vfs_iocb_iter_write(file, kiocb, &segments[i].iter); + if (host_err < 0) + return host_err; + *cnt += host_err; + if (host_err < segments[i].iter.count) + break; /* partial write */ + } + + return 0; +} + /** * nfsd_vfs_write - write data to an already-open file * @rqstp: RPC execution context @@ -1229,29 +1445,46 @@ nfsd_vfs_write(struct svc_rqst *rqstp, struct svc_fh *fhp, stable = NFS_UNSTABLE; init_sync_kiocb(&kiocb, file); kiocb.ki_pos = offset; - if (stable && !fhp->fh_use_wgather) - kiocb.ki_flags |= IOCB_DSYNC; + if (likely(!fhp->fh_use_wgather)) { + switch (stable) { + case NFS_FILE_SYNC: + /* persist data and timestamps */ + kiocb.ki_flags |= IOCB_DSYNC | IOCB_SYNC; + break; + case NFS_DATA_SYNC: + /* persist data only */ + kiocb.ki_flags |= IOCB_DSYNC; + break; + } + } nvecs = xdr_buf_to_bvec(rqstp->rq_bvec, rqstp->rq_maxpages, payload); - iov_iter_bvec(&iter, ITER_SOURCE, rqstp->rq_bvec, nvecs, *cnt); + since = READ_ONCE(file->f_wb_err); if (verf) nfsd_copy_write_verifier(verf, nn); switch (nfsd_io_cache_write) { - case NFSD_IO_BUFFERED: + case NFSD_IO_DIRECT: + host_err = nfsd_direct_write(rqstp, fhp, nf, nvecs, + cnt, &kiocb); break; case NFSD_IO_DONTCACHE: if (file->f_op->fop_flags & FOP_DONTCACHE) kiocb.ki_flags |= IOCB_DONTCACHE; + fallthrough; + case NFSD_IO_BUFFERED: + iov_iter_bvec(&iter, ITER_SOURCE, rqstp->rq_bvec, nvecs, *cnt); + host_err = vfs_iocb_iter_write(file, &kiocb, &iter); + if (host_err < 0) + break; + *cnt = host_err; break; } - host_err = vfs_iocb_iter_write(file, &kiocb, &iter); if (host_err < 0) { commit_reset_write_verifier(nn, rqstp, host_err); goto out_nfserr; } - *cnt = host_err; nfsd_stats_io_write_add(nn, exp, *cnt); fsnotify_modify(file); host_err = filemap_check_wb_err(file->f_mapping, since); @@ -1335,7 +1568,7 @@ __be32 nfsd_read(struct svc_rqst *rqstp, struct svc_fh *fhp, if (file->f_op->splice_read && nfsd_read_splice_ok(rqstp)) err = nfsd_splice_read(rqstp, fhp, file, offset, count, eof); else - err = nfsd_iter_read(rqstp, fhp, file, offset, count, 0, eof); + err = nfsd_iter_read(rqstp, fhp, nf, offset, count, 0, eof); nfsd_file_put(nf); trace_nfsd_read_done(rqstp, fhp, offset, *count); diff --git a/fs/nfsd/vfs.h b/fs/nfsd/vfs.h index 09de48c50cbe..ded2900d423f 100644 --- a/fs/nfsd/vfs.h +++ b/fs/nfsd/vfs.h @@ -121,7 +121,7 @@ __be32 nfsd_splice_read(struct svc_rqst *rqstp, struct svc_fh *fhp, unsigned long *count, u32 *eof); __be32 nfsd_iter_read(struct svc_rqst *rqstp, struct svc_fh *fhp, - struct file *file, loff_t offset, + struct nfsd_file *nf, loff_t offset, unsigned long *count, unsigned int base, u32 *eof); bool nfsd_read_splice_ok(struct svc_rqst *rqstp); diff --git a/fs/nfsd/xdr4.h b/fs/nfsd/xdr4.h index 1ce8e12ae335..ae75846b3cd7 100644 --- a/fs/nfsd/xdr4.h +++ b/fs/nfsd/xdr4.h @@ -924,27 +924,6 @@ struct nfsd4_compoundres { struct nfsd4_compound_state cstate; }; -static inline bool nfsd4_is_solo_sequence(struct nfsd4_compoundres *resp) -{ - struct nfsd4_compoundargs *args = resp->rqstp->rq_argp; - return resp->opcnt == 1 && args->ops[0].opnum == OP_SEQUENCE; -} - -/* - * The session reply cache only needs to cache replies that the client - * actually asked us to. But it's almost free for us to cache compounds - * consisting of only a SEQUENCE op, so we may as well cache those too. - * Also, the protocol doesn't give us a convenient response in the case - * of a replay of a solo SEQUENCE op that wasn't cached - * (RETRY_UNCACHED_REP can only be returned in the second op of a - * compound). - */ -static inline bool nfsd4_cache_this(struct nfsd4_compoundres *resp) -{ - return (resp->cstate.slot->sl_flags & NFSD4_SLOT_CACHETHIS) - || nfsd4_is_solo_sequence(resp); -} - static inline bool nfsd4_last_compound_op(struct svc_rqst *rqstp) { struct nfsd4_compoundres *resp = rqstp->rq_resp; diff --git a/fs/nilfs2/ioctl.c b/fs/nilfs2/ioctl.c index 3288c3b4be9e..e17b8da66491 100644 --- a/fs/nilfs2/ioctl.c +++ b/fs/nilfs2/ioctl.c @@ -49,7 +49,7 @@ static int nilfs_ioctl_wrap_copy(struct the_nilfs *nilfs, void *, size_t, size_t)) { void *buf; - void __user *base = (void __user *)(unsigned long)argv->v_base; + void __user *base = u64_to_user_ptr(argv->v_base); size_t maxmembs, total, n; ssize_t nr; int ret, i; @@ -836,7 +836,6 @@ static int nilfs_ioctl_clean_segments(struct inode *inode, struct file *filp, sizeof(struct nilfs_bdesc), sizeof(__u64), }; - void __user *base; void *kbufs[5]; struct the_nilfs *nilfs; size_t len, nsegs; @@ -863,7 +862,7 @@ static int nilfs_ioctl_clean_segments(struct inode *inode, struct file *filp, * use kmalloc() for its buffer because the memory used for the * segment numbers is small enough. */ - kbufs[4] = memdup_array_user((void __user *)(unsigned long)argv[4].v_base, + kbufs[4] = memdup_array_user(u64_to_user_ptr(argv[4].v_base), nsegs, sizeof(__u64)); if (IS_ERR(kbufs[4])) { ret = PTR_ERR(kbufs[4]); @@ -883,20 +882,14 @@ static int nilfs_ioctl_clean_segments(struct inode *inode, struct file *filp, goto out_free; len = argv[n].v_size * argv[n].v_nmembs; - base = (void __user *)(unsigned long)argv[n].v_base; if (len == 0) { kbufs[n] = NULL; continue; } - kbufs[n] = vmalloc(len); - if (!kbufs[n]) { - ret = -ENOMEM; - goto out_free; - } - if (copy_from_user(kbufs[n], base, len)) { - ret = -EFAULT; - vfree(kbufs[n]); + kbufs[n] = vmemdup_user(u64_to_user_ptr(argv[n].v_base), len); + if (IS_ERR(kbufs[n])) { + ret = PTR_ERR(kbufs[n]); goto out_free; } } @@ -928,7 +921,7 @@ static int nilfs_ioctl_clean_segments(struct inode *inode, struct file *filp, out_free: while (--n >= 0) - vfree(kbufs[n]); + kvfree(kbufs[n]); kfree(kbufs[4]); out: mnt_drop_write_file(filp); @@ -1181,7 +1174,6 @@ static int nilfs_ioctl_set_suinfo(struct inode *inode, struct file *filp, struct nilfs_transaction_info ti; struct nilfs_argv argv; size_t len; - void __user *base; void *kbuf; int ret; @@ -1212,18 +1204,12 @@ static int nilfs_ioctl_set_suinfo(struct inode *inode, struct file *filp, goto out; } - base = (void __user *)(unsigned long)argv.v_base; - kbuf = vmalloc(len); - if (!kbuf) { - ret = -ENOMEM; + kbuf = vmemdup_user(u64_to_user_ptr(argv.v_base), len); + if (IS_ERR(kbuf)) { + ret = PTR_ERR(kbuf); goto out; } - if (copy_from_user(kbuf, base, len)) { - ret = -EFAULT; - goto out_free; - } - nilfs_transaction_begin(inode->i_sb, &ti, 0); ret = nilfs_sufile_set_suinfo(nilfs->ns_sufile, kbuf, argv.v_size, argv.v_nmembs); @@ -1232,8 +1218,7 @@ static int nilfs_ioctl_set_suinfo(struct inode *inode, struct file *filp, else nilfs_transaction_commit(inode->i_sb); /* never fails */ -out_free: - vfree(kbuf); + kvfree(kbuf); out: mnt_drop_write_file(filp); return ret; diff --git a/fs/ocfs2/dir.c b/fs/ocfs2/dir.c index 8c9c4825f984..2785ff245e79 100644 --- a/fs/ocfs2/dir.c +++ b/fs/ocfs2/dir.c @@ -302,8 +302,21 @@ static int ocfs2_check_dir_entry(struct inode *dir, unsigned long offset) { const char *error_msg = NULL; - const int rlen = le16_to_cpu(de->rec_len); - const unsigned long next_offset = ((char *) de - buf) + rlen; + unsigned long next_offset; + int rlen; + + if (offset > size - OCFS2_DIR_REC_LEN(1)) { + /* Dirent is (maybe partially) beyond the buffer + * boundaries so touching 'de' members is unsafe. + */ + mlog(ML_ERROR, "directory entry (#%llu: offset=%lu) " + "too close to end or out-of-bounds", + (unsigned long long)OCFS2_I(dir)->ip_blkno, offset); + return 0; + } + + rlen = le16_to_cpu(de->rec_len); + next_offset = ((char *) de - buf) + rlen; if (unlikely(rlen < OCFS2_DIR_REC_LEN(1))) error_msg = "rec_len is smaller than minimal"; @@ -778,6 +791,14 @@ static int ocfs2_dx_dir_lookup_rec(struct inode *inode, struct ocfs2_extent_block *eb; struct ocfs2_extent_rec *rec = NULL; + if (le16_to_cpu(el->l_count) != + ocfs2_extent_recs_per_dx_root(inode->i_sb)) { + ret = ocfs2_error(inode->i_sb, + "Inode %lu has invalid extent list length %u\n", + inode->i_ino, le16_to_cpu(el->l_count)); + goto out; + } + if (el->l_tree_depth) { ret = ocfs2_find_leaf(INODE_CACHE(inode), el, major_hash, &eb_bh); @@ -3423,6 +3444,14 @@ static int ocfs2_find_dir_space_id(struct inode *dir, struct buffer_head *di_bh, offset += le16_to_cpu(de->rec_len); } + if (!last_de) { + ret = ocfs2_error(sb, "Directory entry (#%llu: size=%lld) " + "is unexpectedly short", + (unsigned long long)OCFS2_I(dir)->ip_blkno, + i_size_read(dir)); + goto out; + } + /* * We're going to require expansion of the directory - figure * out how many blocks we'll need so that a place for the @@ -4104,10 +4133,15 @@ static int ocfs2_expand_inline_dx_root(struct inode *dir, } dx_root->dr_flags &= ~OCFS2_DX_FLAG_INLINE; - memset(&dx_root->dr_list, 0, osb->sb->s_blocksize - - offsetof(struct ocfs2_dx_root_block, dr_list)); + + dx_root->dr_list.l_tree_depth = 0; dx_root->dr_list.l_count = cpu_to_le16(ocfs2_extent_recs_per_dx_root(osb->sb)); + dx_root->dr_list.l_next_free_rec = 0; + memset(&dx_root->dr_list.l_recs, 0, + osb->sb->s_blocksize - + (offsetof(struct ocfs2_dx_root_block, dr_list) + + offsetof(struct ocfs2_extent_list, l_recs))); /* This should never fail considering we start with an empty * dx_root. */ diff --git a/fs/ocfs2/inode.c b/fs/ocfs2/inode.c index 78f81950c9ee..8340525e5589 100644 --- a/fs/ocfs2/inode.c +++ b/fs/ocfs2/inode.c @@ -201,13 +201,15 @@ bail: static int ocfs2_dinode_has_extents(struct ocfs2_dinode *di) { /* inodes flagged with other stuff in id2 */ - if (di->i_flags & (OCFS2_SUPER_BLOCK_FL | OCFS2_LOCAL_ALLOC_FL | - OCFS2_CHAIN_FL | OCFS2_DEALLOC_FL)) + if (le32_to_cpu(di->i_flags) & + (OCFS2_SUPER_BLOCK_FL | OCFS2_LOCAL_ALLOC_FL | OCFS2_CHAIN_FL | + OCFS2_DEALLOC_FL)) return 0; /* i_flags doesn't indicate when id2 is a fast symlink */ - if (S_ISLNK(di->i_mode) && di->i_size && di->i_clusters == 0) + if (S_ISLNK(le16_to_cpu(di->i_mode)) && le64_to_cpu(di->i_size) && + !le32_to_cpu(di->i_clusters)) return 0; - if (di->i_dyn_features & OCFS2_INLINE_DATA_FL) + if (le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) return 0; return 1; @@ -1460,7 +1462,7 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } - if (!(di->i_flags & cpu_to_le32(OCFS2_VALID_FL))) { + if (!(le32_to_cpu(di->i_flags) & OCFS2_VALID_FL)) { rc = ocfs2_error(sb, "Invalid dinode #%llu: OCFS2_VALID_FL not set\n", (unsigned long long)bh->b_blocknr); @@ -1484,6 +1486,41 @@ int ocfs2_validate_inode_block(struct super_block *sb, goto bail; } + if ((le16_to_cpu(di->i_dyn_features) & OCFS2_INLINE_DATA_FL) && + le32_to_cpu(di->i_clusters)) { + rc = ocfs2_error(sb, "Invalid dinode %llu: %u clusters\n", + (unsigned long long)bh->b_blocknr, + le32_to_cpu(di->i_clusters)); + goto bail; + } + + if (le32_to_cpu(di->i_flags) & OCFS2_CHAIN_FL) { + struct ocfs2_chain_list *cl = &di->id2.i_chain; + u16 bpc = 1 << (OCFS2_SB(sb)->s_clustersize_bits - + sb->s_blocksize_bits); + + if (le16_to_cpu(cl->cl_count) != ocfs2_chain_recs_per_inode(sb)) { + rc = ocfs2_error(sb, "Invalid dinode %llu: chain list count %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(cl->cl_count)); + goto bail; + } + if (le16_to_cpu(cl->cl_next_free_rec) > le16_to_cpu(cl->cl_count)) { + rc = ocfs2_error(sb, "Invalid dinode %llu: chain list index %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(cl->cl_next_free_rec)); + goto bail; + } + if (OCFS2_SB(sb)->bitmap_blkno && + OCFS2_SB(sb)->bitmap_blkno != le64_to_cpu(di->i_blkno) && + le16_to_cpu(cl->cl_bpc) != bpc) { + rc = ocfs2_error(sb, "Invalid dinode %llu: bits per cluster %u\n", + (unsigned long long)bh->b_blocknr, + le16_to_cpu(cl->cl_bpc)); + goto bail; + } + } + rc = 0; bail: @@ -1671,6 +1708,8 @@ int ocfs2_read_inode_block_full(struct inode *inode, struct buffer_head **bh, rc = ocfs2_read_blocks(INODE_CACHE(inode), OCFS2_I(inode)->ip_blkno, 1, &tmp, flags, ocfs2_validate_inode_block); + if (rc < 0) + make_bad_inode(inode); /* If ocfs2_read_blocks() got us a new bh, pass it up. */ if (!rc && !*bh) *bh = tmp; diff --git a/fs/ocfs2/move_extents.c b/fs/ocfs2/move_extents.c index 10923bf7c8b8..ce978a2497d9 100644 --- a/fs/ocfs2/move_extents.c +++ b/fs/ocfs2/move_extents.c @@ -98,7 +98,13 @@ static int __ocfs2_move_extent(handle_t *handle, rec = &el->l_recs[index]; - BUG_ON(ext_flags != rec->e_flags); + if (ext_flags != rec->e_flags) { + ret = ocfs2_error(inode->i_sb, + "Inode %llu has corrupted extent %d with flags 0x%x at cpos %u\n", + (unsigned long long)ino, index, rec->e_flags, cpos); + goto out; + } + /* * after moving/defraging to new location, the extent is not going * to be refcounted anymore. @@ -1036,6 +1042,12 @@ int ocfs2_ioctl_move_extents(struct file *filp, void __user *argp) if (range.me_threshold > i_size_read(inode)) range.me_threshold = i_size_read(inode); + if (range.me_flags & ~(OCFS2_MOVE_EXT_FL_AUTO_DEFRAG | + OCFS2_MOVE_EXT_FL_PART_DEFRAG)) { + status = -EINVAL; + goto out_free; + } + if (range.me_flags & OCFS2_MOVE_EXT_FL_AUTO_DEFRAG) { context->auto_defrag = 1; diff --git a/fs/ocfs2/ocfs2_fs.h b/fs/ocfs2/ocfs2_fs.h index ae0e44e5f2ad..f7763da5c4a2 100644 --- a/fs/ocfs2/ocfs2_fs.h +++ b/fs/ocfs2/ocfs2_fs.h @@ -468,7 +468,8 @@ struct ocfs2_extent_list { __le16 l_reserved1; __le64 l_reserved2; /* Pad to sizeof(ocfs2_extent_rec) */ -/*10*/ struct ocfs2_extent_rec l_recs[]; /* Extent records */ + /* Extent records */ +/*10*/ struct ocfs2_extent_rec l_recs[] __counted_by_le(l_count); }; /* @@ -482,7 +483,8 @@ struct ocfs2_chain_list { __le16 cl_count; /* Total chains in this list */ __le16 cl_next_free_rec; /* Next unused chain slot */ __le64 cl_reserved1; -/*10*/ struct ocfs2_chain_rec cl_recs[]; /* Chain records */ + /* Chain records */ +/*10*/ struct ocfs2_chain_rec cl_recs[] __counted_by_le(cl_count); }; /* @@ -494,7 +496,8 @@ struct ocfs2_truncate_log { /*00*/ __le16 tl_count; /* Total records in this log */ __le16 tl_used; /* Number of records in use */ __le32 tl_reserved1; -/*08*/ struct ocfs2_truncate_rec tl_recs[]; /* Truncate records */ + /* Truncate records */ +/*08*/ struct ocfs2_truncate_rec tl_recs[] __counted_by_le(tl_count); }; /* @@ -796,9 +799,10 @@ struct ocfs2_dx_entry_list { * possible in de_entries */ __le16 de_num_used; /* Current number of * de_entries entries */ - struct ocfs2_dx_entry de_entries[]; /* Indexed dir entries - * in a packed array of - * length de_num_used */ + /* Indexed dir entries in a packed + * array of length de_num_used. + */ + struct ocfs2_dx_entry de_entries[] __counted_by_le(de_count); }; #define OCFS2_DX_FLAG_INLINE 0x01 @@ -934,7 +938,8 @@ struct ocfs2_refcount_list { __le16 rl_used; /* Current number of used records */ __le32 rl_reserved2; __le64 rl_reserved1; /* Pad to sizeof(ocfs2_refcount_record) */ -/*10*/ struct ocfs2_refcount_rec rl_recs[]; /* Refcount records */ + /* Refcount records */ +/*10*/ struct ocfs2_refcount_rec rl_recs[] __counted_by_le(rl_count); }; @@ -1020,7 +1025,8 @@ struct ocfs2_xattr_header { buckets. A block uses xb_check and sets this field to zero.) */ - struct ocfs2_xattr_entry xh_entries[]; /* xattr entry list. */ + /* xattr entry list. */ + struct ocfs2_xattr_entry xh_entries[] __counted_by_le(xh_count); }; /* diff --git a/fs/ocfs2/refcounttree.c b/fs/ocfs2/refcounttree.c index 267b50e8e42e..c92e0ea85bca 100644 --- a/fs/ocfs2/refcounttree.c +++ b/fs/ocfs2/refcounttree.c @@ -34,6 +34,7 @@ #include <linux/pagevec.h> #include <linux/swap.h> #include <linux/security.h> +#include <linux/string.h> #include <linux/fsnotify.h> #include <linux/quotaops.h> #include <linux/namei.h> @@ -621,7 +622,7 @@ static int ocfs2_create_refcount_tree(struct inode *inode, /* Initialize ocfs2_refcount_block. */ rb = (struct ocfs2_refcount_block *)new_bh->b_data; memset(rb, 0, inode->i_sb->s_blocksize); - strcpy((void *)rb, OCFS2_REFCOUNT_BLOCK_SIGNATURE); + strscpy(rb->rf_signature, OCFS2_REFCOUNT_BLOCK_SIGNATURE); rb->rf_suballoc_slot = cpu_to_le16(meta_ac->ac_alloc_slot); rb->rf_suballoc_loc = cpu_to_le64(suballoc_loc); rb->rf_suballoc_bit = cpu_to_le16(suballoc_bit_start); @@ -1562,7 +1563,7 @@ static int ocfs2_new_leaf_refcount_block(handle_t *handle, /* Initialize ocfs2_refcount_block. */ new_rb = (struct ocfs2_refcount_block *)new_bh->b_data; memset(new_rb, 0, sb->s_blocksize); - strcpy((void *)new_rb, OCFS2_REFCOUNT_BLOCK_SIGNATURE); + strscpy(new_rb->rf_signature, OCFS2_REFCOUNT_BLOCK_SIGNATURE); new_rb->rf_suballoc_slot = cpu_to_le16(meta_ac->ac_alloc_slot); new_rb->rf_suballoc_loc = cpu_to_le64(suballoc_loc); new_rb->rf_suballoc_bit = cpu_to_le16(suballoc_bit_start); diff --git a/fs/ocfs2/xattr.c b/fs/ocfs2/xattr.c index d70a20d29e3e..dc1761e84814 100644 --- a/fs/ocfs2/xattr.c +++ b/fs/ocfs2/xattr.c @@ -2908,7 +2908,7 @@ static int ocfs2_create_xattr_block(struct inode *inode, /* Initialize ocfs2_xattr_block */ xblk = (struct ocfs2_xattr_block *)new_bh->b_data; memset(xblk, 0, inode->i_sb->s_blocksize); - strcpy((void *)xblk, OCFS2_XATTR_BLOCK_SIGNATURE); + strscpy(xblk->xb_signature, OCFS2_XATTR_BLOCK_SIGNATURE); xblk->xb_suballoc_slot = cpu_to_le16(ctxt->meta_ac->ac_alloc_slot); xblk->xb_suballoc_loc = cpu_to_le64(suballoc_loc); xblk->xb_suballoc_bit = cpu_to_le16(suballoc_bit_start); @@ -6351,7 +6351,7 @@ static int ocfs2_reflink_xattr_header(handle_t *handle, trace_ocfs2_reflink_xattr_header((unsigned long long)old_bh->b_blocknr, le16_to_cpu(xh->xh_count)); - last = &new_xh->xh_entries[le16_to_cpu(new_xh->xh_count)]; + last = &new_xh->xh_entries[le16_to_cpu(new_xh->xh_count)] - 1; for (i = 0, j = 0; i < le16_to_cpu(xh->xh_count); i++, j++) { xe = &xh->xh_entries[i]; diff --git a/fs/proc/array.c b/fs/proc/array.c index cbd4bc4a58e4..42932f88141a 100644 --- a/fs/proc/array.c +++ b/fs/proc/array.c @@ -537,27 +537,27 @@ static int do_task_stat(struct seq_file *m, struct pid_namespace *ns, if (permitted && (!whole || num_threads < 2)) wchan = !task_is_running(task); - scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) { - cmin_flt = sig->cmin_flt; - cmaj_flt = sig->cmaj_flt; - cutime = sig->cutime; - cstime = sig->cstime; - cgtime = sig->cgtime; - - if (whole) { - struct task_struct *t; - - min_flt = sig->min_flt; - maj_flt = sig->maj_flt; - gtime = sig->gtime; - - rcu_read_lock(); - __for_each_thread(sig, t) { - min_flt += t->min_flt; - maj_flt += t->maj_flt; - gtime += task_gtime(t); + scoped_guard(rcu) { + scoped_seqlock_read (&sig->stats_lock, ss_lock_irqsave) { + cmin_flt = sig->cmin_flt; + cmaj_flt = sig->cmaj_flt; + cutime = sig->cutime; + cstime = sig->cstime; + cgtime = sig->cgtime; + + if (whole) { + struct task_struct *t; + + min_flt = sig->min_flt; + maj_flt = sig->maj_flt; + gtime = sig->gtime; + + __for_each_thread(sig, t) { + min_flt += t->min_flt; + maj_flt += t->maj_flt; + gtime += task_gtime(t); + } } - rcu_read_unlock(); } } diff --git a/fs/proc/page.c b/fs/proc/page.c index fc64f23e05e5..f9b2c2c906cd 100644 --- a/fs/proc/page.c +++ b/fs/proc/page.c @@ -20,7 +20,6 @@ #define KPMSIZE sizeof(u64) #define KPMMASK (KPMSIZE - 1) -#define KPMBITS (KPMSIZE * BITS_PER_BYTE) enum kpage_operation { KPAGE_FLAGS, diff --git a/include/dt-bindings/watchdog/aspeed-wdt.h b/include/dt-bindings/watchdog/aspeed-wdt.h index 7ae6d84b2bd9..89fa31ffce2d 100644 --- a/include/dt-bindings/watchdog/aspeed-wdt.h +++ b/include/dt-bindings/watchdog/aspeed-wdt.h @@ -89,4 +89,142 @@ #define AST2600_WDT_RESET2_DEFAULT 0x03fffff1 +#define AST2700_WDT_RESET1_CPU (1 << 0) +#define AST2700_WDT_RESET1_DRAM (1 << 1) +#define AST2700_WDT_RESET1_SLI0 (1 << 2) +#define AST2700_WDT_RESET1_EHCI (1 << 3) +#define AST2700_WDT_RESET1_HACE (1 << 4) +#define AST2700_WDT_RESET1_SOC_MISC0 (1 << 5) +#define AST2700_WDT_RESET1_VIDEO (1 << 6) +#define AST2700_WDT_RESET1_2D_GRAPHIC (1 << 7) +#define AST2700_WDT_RESET1_RAVS0 (1 << 8) +#define AST2700_WDT_RESET1_RAVS1 (1 << 9) +#define AST2700_WDT_RESET1_GPIO0 (1 << 10) +#define AST2700_WDT_RESET1_SSP (1 << 11) +#define AST2700_WDT_RESET1_TSP (1 << 12) +#define AST2700_WDT_RESET1_CRT (1 << 13) +#define AST2700_WDT_RESET1_USB20_HOST (1 << 14) +#define AST2700_WDT_RESET1_USB11_HOST (1 << 15) +#define AST2700_WDT_RESET1_UFS (1 << 16) +#define AST2700_WDT_RESET1_EMMC (1 << 17) +#define AST2700_WDT_RESET1_AHB_TO_PCIE1 (1 << 18) +#define AST2700_WDT_RESET1_XDMA0 (1 << 22) +#define AST2700_WDT_RESET1_MCTP1 (1 << 23) +#define AST2700_WDT_RESET1_MCTP0 (1 << 24) +#define AST2700_WDT_RESET1_JTAG0 (1 << 25) +#define AST2700_WDT_RESET1_ECC (1 << 26) +#define AST2700_WDT_RESET1_XDMA1 (1 << 27) +#define AST2700_WDT_RESET1_DP (1 << 28) +#define AST2700_WDT_RESET1_DP_MCU (1 << 29) +#define AST2700_WDT_RESET1_AHB_TO_PCIE0 (1 << 31) + +#define AST2700_WDT_RESET1_DEFAULT 0x8207ff71 + +#define AST2700_WDT_RESET2_USB3_A_HOST (1 << 0) +#define AST2700_WDT_RESET2_USB3_A_VHUB3 (1 << 1) +#define AST2700_WDT_RESET2_USB3_A_VHUB2 (1 << 2) +#define AST2700_WDT_RESET2_USB3_B_HOST (1 << 3) +#define AST2700_WDT_RESET2_USB3_B_VHUB3 (1 << 4) +#define AST2700_WDT_RESET2_USB3_B_VHUB2 (1 << 5) +#define AST2700_WDT_RESET2_SM3 (1 << 6) +#define AST2700_WDT_RESET2_SM4 (1 << 7) +#define AST2700_WDT_RESET2_SHA3 (1 << 8) +#define AST2700_WDT_RESET2_RSA (1 << 9) + +#define AST2700_WDT_RESET2_DEFAULT 0x000003f6 + +#define AST2700_WDT_RESET3_LPC0 (1 << 0) +#define AST2700_WDT_RESET3_LPC1 (1 << 1) +#define AST2700_WDT_RESET3_MDIO (1 << 2) +#define AST2700_WDT_RESET3_PECI (1 << 3) +#define AST2700_WDT_RESET3_PWM (1 << 4) +#define AST2700_WDT_RESET3_MAC0 (1 << 5) +#define AST2700_WDT_RESET3_MAC1 (1 << 6) +#define AST2700_WDT_RESET3_MAC2 (1 << 7) +#define AST2700_WDT_RESET3_ADC (1 << 8) +#define AST2700_WDT_RESET3_SDC (1 << 9) +#define AST2700_WDT_RESET3_ESPI0 (1 << 10) +#define AST2700_WDT_RESET3_ESPI1 (1 << 11) +#define AST2700_WDT_RESET3_JTAG1 (1 << 12) +#define AST2700_WDT_RESET3_SPI0 (1 << 13) +#define AST2700_WDT_RESET3_SPI1 (1 << 14) +#define AST2700_WDT_RESET3_SPI2 (1 << 15) +#define AST2700_WDT_RESET3_I3C0 (1 << 16) +#define AST2700_WDT_RESET3_I3C1 (1 << 17) +#define AST2700_WDT_RESET3_I3C2 (1 << 18) +#define AST2700_WDT_RESET3_I3C3 (1 << 19) +#define AST2700_WDT_RESET3_I3C4 (1 << 20) +#define AST2700_WDT_RESET3_I3C5 (1 << 21) +#define AST2700_WDT_RESET3_I3C6 (1 << 22) +#define AST2700_WDT_RESET3_I3C7 (1 << 23) +#define AST2700_WDT_RESET3_I3C8 (1 << 24) +#define AST2700_WDT_RESET3_I3C9 (1 << 25) +#define AST2700_WDT_RESET3_I3C10 (1 << 26) +#define AST2700_WDT_RESET3_I3C11 (1 << 27) +#define AST2700_WDT_RESET3_I3C12 (1 << 28) +#define AST2700_WDT_RESET3_I3C13 (1 << 29) +#define AST2700_WDT_RESET3_I3C14 (1 << 30) +#define AST2700_WDT_RESET3_I3C15 (1 << 31) + +#define AST2700_WDT_RESET3_DEFAULT 0x000093ec + +#define AST2700_WDT_RESET4_FMC (1 << 0) +#define AST2700_WDT_RESET4_SOC_MISC1 (1 << 1) +#define AST2700_WDT_RESET4_AHB (1 << 2) +#define AST2700_WDT_RESET4_SLI1 (1 << 3) +#define AST2700_WDT_RESET4_UART0 (1 << 4) +#define AST2700_WDT_RESET4_UART1 (1 << 5) +#define AST2700_WDT_RESET4_UART2 (1 << 6) +#define AST2700_WDT_RESET4_UART3 (1 << 7) +#define AST2700_WDT_RESET4_I2C_MONITOR (1 << 8) +#define AST2700_WDT_RESET4_HOST_TO_SPI1 (1 << 9) +#define AST2700_WDT_RESET4_HOST_TO_SPI2 (1 << 10) +#define AST2700_WDT_RESET4_GPIO1 (1 << 11) +#define AST2700_WDT_RESET4_FSI (1 << 12) +#define AST2700_WDT_RESET4_CANBUS (1 << 13) +#define AST2700_WDT_RESET4_MCTP (1 << 14) +#define AST2700_WDT_RESET4_XDMA (1 << 15) +#define AST2700_WDT_RESET4_UART5 (1 << 16) +#define AST2700_WDT_RESET4_UART6 (1 << 17) +#define AST2700_WDT_RESET4_UART7 (1 << 18) +#define AST2700_WDT_RESET4_UART8 (1 << 19) +#define AST2700_WDT_RESET4_BOOT_MCU (1 << 20) +#define AST2700_WDT_RESET4_IO_MCU (1 << 21) +#define AST2700_WDT_RESET4_LTPI0 (1 << 22) +#define AST2700_WDT_RESET4_VGA_LINK (1 << 23) +#define AST2700_WDT_RESET4_LTPI1 (1 << 24) +#define AST2700_WDT_RESET4_LTPI_PHY (1 << 25) +#define AST2700_WDT_RESET4_ACE (1 << 26) +#define AST2700_WDT_RESET4_LTPI_GPIO0 (1 << 28) +#define AST2700_WDT_RESET4_LTPI_GPIO1 (1 << 29) +#define AST2700_WDT_RESET4_AHB_TO_PCIE1 (1 << 30) +#define AST2700_WDT_RESET4_I3C_DMA (1 << 31) + +#define AST2700_WDT_RESET4_DEFAULT 0x40303803 + +#define AST2700_WDT_RESET5_I2C_GLOBAL (1 << 0) +#define AST2700_WDT_RESET5_I2C0 (1 << 1) +#define AST2700_WDT_RESET5_I2C1 (1 << 2) +#define AST2700_WDT_RESET5_I2C2 (1 << 3) +#define AST2700_WDT_RESET5_I2C3 (1 << 4) +#define AST2700_WDT_RESET5_I2C4 (1 << 5) +#define AST2700_WDT_RESET5_I2C5 (1 << 6) +#define AST2700_WDT_RESET5_I2C6 (1 << 7) +#define AST2700_WDT_RESET5_I2C7 (1 << 8) +#define AST2700_WDT_RESET5_I2C8 (1 << 9) +#define AST2700_WDT_RESET5_I2C9 (1 << 10) +#define AST2700_WDT_RESET5_I2C10 (1 << 11) +#define AST2700_WDT_RESET5_I2C11 (1 << 12) +#define AST2700_WDT_RESET5_I2C12 (1 << 13) +#define AST2700_WDT_RESET5_I2C13 (1 << 14) +#define AST2700_WDT_RESET5_I2C14 (1 << 15) +#define AST2700_WDT_RESET5_I2C15 (1 << 16) +#define AST2700_WDT_RESET5_UHCI (1 << 17) +#define AST2700_WDT_RESET5_USB2_C_UART (1 << 18) +#define AST2700_WDT_RESET5_USB2_C (1 << 19) +#define AST2700_WDT_RESET5_USB2_D_UART (1 << 20) +#define AST2700_WDT_RESET5_USB2_D (1 << 21) + +#define AST2700_WDT_RESET5_DEFAULT 0x00320000 + #endif diff --git a/include/linux/amd-iommu.h b/include/linux/amd-iommu.h index 8cced632ecd0..edcee9f5335a 100644 --- a/include/linux/amd-iommu.h +++ b/include/linux/amd-iommu.h @@ -70,8 +70,10 @@ struct amd_iommu *get_amd_iommu(unsigned int idx); #ifdef CONFIG_KVM_AMD_SEV int amd_iommu_snp_disable(void); +extern bool amd_iommu_sev_tio_supported(void); #else static inline int amd_iommu_snp_disable(void) { return 0; } +static inline bool amd_iommu_sev_tio_supported(void) { return false; } #endif #endif /* _ASM_X86_AMD_IOMMU_H */ diff --git a/include/linux/annotate.h b/include/linux/annotate.h index 7c10d34d198c..2f1599c9e573 100644 --- a/include/linux/annotate.h +++ b/include/linux/annotate.h @@ -6,41 +6,34 @@ #ifdef CONFIG_OBJTOOL -#ifndef __ASSEMBLY__ - #define __ASM_ANNOTATE(section, label, type) \ - ".pushsection " section ",\"M\", @progbits, 8\n\t" \ - ".long " __stringify(label) " - .\n\t" \ - ".long " __stringify(type) "\n\t" \ - ".popsection\n\t" + .pushsection section, "M", @progbits, 8; \ + .long label - ., type; \ + .popsection + +#ifndef __ASSEMBLY__ #define ASM_ANNOTATE_LABEL(label, type) \ - __ASM_ANNOTATE(".discard.annotate_insn", label, type) + __stringify(__ASM_ANNOTATE(.discard.annotate_insn, label, type)) #define ASM_ANNOTATE(type) \ - "911:\n\t" \ - ASM_ANNOTATE_LABEL(911b, type) + "911: " \ + __stringify(__ASM_ANNOTATE(.discard.annotate_insn, 911b, type)) #define ASM_ANNOTATE_DATA(type) \ - "912:\n\t" \ - __ASM_ANNOTATE(".discard.annotate_data", 912b, type) + "912: " \ + __stringify(__ASM_ANNOTATE(.discard.annotate_data, 912b, type)) #else /* __ASSEMBLY__ */ -.macro __ANNOTATE section, type -.Lhere_\@: - .pushsection \section, "M", @progbits, 8 - .long .Lhere_\@ - . - .long \type - .popsection -.endm - .macro ANNOTATE type - __ANNOTATE ".discard.annotate_insn", \type +.Lhere_\@: + __ASM_ANNOTATE(.discard.annotate_insn, .Lhere_\@, \type) .endm .macro ANNOTATE_DATA type - __ANNOTATE ".discard.annotate_data", \type +.Lhere_\@: + __ASM_ANNOTATE(.discard.annotate_data, .Lhere_\@, \type) .endm #endif /* __ASSEMBLY__ */ diff --git a/include/linux/base64.h b/include/linux/base64.h index 660d4cb1ef31..a2c6c9222da3 100644 --- a/include/linux/base64.h +++ b/include/linux/base64.h @@ -8,9 +8,15 @@ #include <linux/types.h> +enum base64_variant { + BASE64_STD, /* RFC 4648 (standard) */ + BASE64_URLSAFE, /* RFC 4648 (base64url) */ + BASE64_IMAP, /* RFC 3501 */ +}; + #define BASE64_CHARS(nbytes) DIV_ROUND_UP((nbytes) * 4, 3) -int base64_encode(const u8 *src, int len, char *dst); -int base64_decode(const char *src, int len, u8 *dst); +int base64_encode(const u8 *src, int len, char *dst, bool padding, enum base64_variant variant); +int base64_decode(const char *src, int len, u8 *dst, bool padding, enum base64_variant variant); #endif /* _LINUX_BASE64_H */ diff --git a/include/linux/bitfield.h b/include/linux/bitfield.h index 5355f8f806a9..126dc5b380af 100644 --- a/include/linux/bitfield.h +++ b/include/linux/bitfield.h @@ -17,6 +17,7 @@ * FIELD_{GET,PREP} macros take as first parameter shifted mask * from which they extract the base mask and shift amount. * Mask must be a compilation time constant. + * field_{get,prep} are variants that take a non-const mask. * * Example: * @@ -60,7 +61,7 @@ #define __bf_cast_unsigned(type, x) ((__unsigned_scalar_typeof(type))(x)) -#define __BF_FIELD_CHECK(_mask, _reg, _val, _pfx) \ +#define __BF_FIELD_CHECK_MASK(_mask, _val, _pfx) \ ({ \ BUILD_BUG_ON_MSG(!__builtin_constant_p(_mask), \ _pfx "mask is not constant"); \ @@ -69,13 +70,33 @@ ~((_mask) >> __bf_shf(_mask)) & \ (0 + (_val)) : 0, \ _pfx "value too large for the field"); \ - BUILD_BUG_ON_MSG(__bf_cast_unsigned(_mask, _mask) > \ - __bf_cast_unsigned(_reg, ~0ull), \ - _pfx "type of reg too small for mask"); \ __BUILD_BUG_ON_NOT_POWER_OF_2((_mask) + \ (1ULL << __bf_shf(_mask))); \ }) +#define __BF_FIELD_CHECK_REG(mask, reg, pfx) \ + BUILD_BUG_ON_MSG(__bf_cast_unsigned(mask, mask) > \ + __bf_cast_unsigned(reg, ~0ull), \ + pfx "type of reg too small for mask") + +#define __BF_FIELD_CHECK(mask, reg, val, pfx) \ + ({ \ + __BF_FIELD_CHECK_MASK(mask, val, pfx); \ + __BF_FIELD_CHECK_REG(mask, reg, pfx); \ + }) + +#define __FIELD_PREP(mask, val, pfx) \ + ({ \ + __BF_FIELD_CHECK_MASK(mask, val, pfx); \ + ((typeof(mask))(val) << __bf_shf(mask)) & (mask); \ + }) + +#define __FIELD_GET(mask, reg, pfx) \ + ({ \ + __BF_FIELD_CHECK_MASK(mask, 0U, pfx); \ + (typeof(mask))(((reg) & (mask)) >> __bf_shf(mask)); \ + }) + /** * FIELD_MAX() - produce the maximum value representable by a field * @_mask: shifted mask defining the field's length and position @@ -112,8 +133,8 @@ */ #define FIELD_PREP(_mask, _val) \ ({ \ - __BF_FIELD_CHECK(_mask, 0ULL, _val, "FIELD_PREP: "); \ - ((typeof(_mask))(_val) << __bf_shf(_mask)) & (_mask); \ + __BF_FIELD_CHECK_REG(_mask, 0ULL, "FIELD_PREP: "); \ + __FIELD_PREP(_mask, _val, "FIELD_PREP: "); \ }) #define __BF_CHECK_POW2(n) BUILD_BUG_ON_ZERO(((n) & ((n) - 1)) != 0) @@ -152,8 +173,8 @@ */ #define FIELD_GET(_mask, _reg) \ ({ \ - __BF_FIELD_CHECK(_mask, _reg, 0U, "FIELD_GET: "); \ - (typeof(_mask))(((_reg) & (_mask)) >> __bf_shf(_mask)); \ + __BF_FIELD_CHECK_REG(_mask, _reg, "FIELD_GET: "); \ + __FIELD_GET(_mask, _reg, "FIELD_GET: "); \ }) /** @@ -220,4 +241,62 @@ __MAKE_OP(64) #undef __MAKE_OP #undef ____MAKE_OP +#define __field_prep(mask, val) \ + ({ \ + __auto_type __mask = (mask); \ + typeof(__mask) __val = (val); \ + unsigned int __shift = BITS_PER_TYPE(__mask) <= 32 ? \ + __ffs(__mask) : __ffs64(__mask); \ + (__val << __shift) & __mask; \ + }) + +#define __field_get(mask, reg) \ + ({ \ + __auto_type __mask = (mask); \ + typeof(__mask) __reg = (reg); \ + unsigned int __shift = BITS_PER_TYPE(__mask) <= 32 ? \ + __ffs(__mask) : __ffs64(__mask); \ + (__reg & __mask) >> __shift; \ + }) + +/** + * field_prep() - prepare a bitfield element + * @mask: shifted mask defining the field's length and position, must be + * non-zero + * @val: value to put in the field + * + * Return: field value masked and shifted to its final destination + * + * field_prep() masks and shifts up the value. The result should be + * combined with other fields of the bitfield using logical OR. + * Unlike FIELD_PREP(), @mask is not limited to a compile-time constant. + * Typical usage patterns are a value stored in a table, or calculated by + * shifting a constant by a variable number of bits. + * If you want to ensure that @mask is a compile-time constant, please use + * FIELD_PREP() directly instead. + */ +#define field_prep(mask, val) \ + (__builtin_constant_p(mask) ? __FIELD_PREP(mask, val, "field_prep: ") \ + : __field_prep(mask, val)) + +/** + * field_get() - extract a bitfield element + * @mask: shifted mask defining the field's length and position, must be + * non-zero + * @reg: value of entire bitfield + * + * Return: extracted field value + * + * field_get() extracts the field specified by @mask from the + * bitfield passed in as @reg by masking and shifting it down. + * Unlike FIELD_GET(), @mask is not limited to a compile-time constant. + * Typical usage patterns are a value stored in a table, or calculated by + * shifting a constant by a variable number of bits. + * If you want to ensure that @mask is a compile-time constant, please use + * FIELD_GET() directly instead. + */ +#define field_get(mask, reg) \ + (__builtin_constant_p(mask) ? __FIELD_GET(mask, reg, "field_get: ") \ + : __field_get(mask, reg)) + #endif diff --git a/include/linux/compiler.h b/include/linux/compiler.h index ab181d87d71d..ff71bebe56f5 100644 --- a/include/linux/compiler.h +++ b/include/linux/compiler.h @@ -273,12 +273,6 @@ static inline void *offset_to_ptr(const int *off) #endif /* __ASSEMBLY__ */ -#ifdef CONFIG_64BIT -#define ARCH_SEL(a,b) a -#else -#define ARCH_SEL(a,b) b -#endif - /* * Force the compiler to emit 'sym' as a symbol, so that we can reference * it from inline assembler. Necessary in case 'sym' could be inlined diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index afedfd5bea07..80211900f373 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -7,14 +7,16 @@ * set of CPUs in a system, one bit position per CPU number. In general, * only nr_cpu_ids (<= NR_CPUS) bits are valid. */ -#include <linux/cleanup.h> -#include <linux/kernel.h> +#include <linux/atomic.h> #include <linux/bitmap.h> +#include <linux/cleanup.h> #include <linux/cpumask_types.h> -#include <linux/atomic.h> -#include <linux/bug.h> #include <linux/gfp_types.h> #include <linux/numa.h> +#include <linux/threads.h> +#include <linux/types.h> + +#include <asm/bug.h> /** * cpumask_pr_args - printf args to output a cpumask diff --git a/include/linux/crash_reserve.h b/include/linux/crash_reserve.h index 7b44b41d0a20..f0dc03d94ca2 100644 --- a/include/linux/crash_reserve.h +++ b/include/linux/crash_reserve.h @@ -32,6 +32,12 @@ int __init parse_crashkernel(char *cmdline, unsigned long long system_ram, void __init reserve_crashkernel_cma(unsigned long long cma_size); #ifdef CONFIG_ARCH_HAS_GENERIC_CRASHKERNEL_RESERVATION +#ifndef arch_add_crash_res_to_iomem +static inline bool arch_add_crash_res_to_iomem(void) +{ + return true; +} +#endif #ifndef DEFAULT_CRASH_KERNEL_LOW_SIZE #define DEFAULT_CRASH_KERNEL_LOW_SIZE (128UL << 20) #endif diff --git a/include/linux/device/bus.h b/include/linux/device/bus.h index f5a56efd2bd6..99b1002b3e31 100644 --- a/include/linux/device/bus.h +++ b/include/linux/device/bus.h @@ -150,6 +150,9 @@ int bus_for_each_dev(const struct bus_type *bus, struct device *start, void *data, device_iter_t fn); struct device *bus_find_device(const struct bus_type *bus, struct device *start, const void *data, device_match_t match); +struct device *bus_find_device_reverse(const struct bus_type *bus, + struct device *start, const void *data, + device_match_t match); /** * bus_find_device_by_name - device iterator for locating a particular device * of a specific name. diff --git a/include/linux/dma-map-ops.h b/include/linux/dma-map-ops.h index 10882d00cb17..4809204c674c 100644 --- a/include/linux/dma-map-ops.h +++ b/include/linux/dma-map-ops.h @@ -31,10 +31,10 @@ struct dma_map_ops { void *cpu_addr, dma_addr_t dma_addr, size_t size, unsigned long attrs); - dma_addr_t (*map_page)(struct device *dev, struct page *page, - unsigned long offset, size_t size, - enum dma_data_direction dir, unsigned long attrs); - void (*unmap_page)(struct device *dev, dma_addr_t dma_handle, + dma_addr_t (*map_phys)(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, + unsigned long attrs); + void (*unmap_phys)(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir, unsigned long attrs); /* @@ -46,12 +46,6 @@ struct dma_map_ops { enum dma_data_direction dir, unsigned long attrs); void (*unmap_sg)(struct device *dev, struct scatterlist *sg, int nents, enum dma_data_direction dir, unsigned long attrs); - dma_addr_t (*map_resource)(struct device *dev, phys_addr_t phys_addr, - size_t size, enum dma_data_direction dir, - unsigned long attrs); - void (*unmap_resource)(struct device *dev, dma_addr_t dma_handle, - size_t size, enum dma_data_direction dir, - unsigned long attrs); void (*sync_single_for_cpu)(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir); void (*sync_single_for_device)(struct device *dev, diff --git a/include/linux/dynamic_debug.h b/include/linux/dynamic_debug.h index ff44ec346162..05743900a116 100644 --- a/include/linux/dynamic_debug.h +++ b/include/linux/dynamic_debug.h @@ -38,11 +38,12 @@ struct _ddebug { #define _DPRINTK_FLAGS_INCL_LINENO (1<<3) #define _DPRINTK_FLAGS_INCL_TID (1<<4) #define _DPRINTK_FLAGS_INCL_SOURCENAME (1<<5) +#define _DPRINTK_FLAGS_INCL_STACK (1<<6) #define _DPRINTK_FLAGS_INCL_ANY \ (_DPRINTK_FLAGS_INCL_MODNAME | _DPRINTK_FLAGS_INCL_FUNCNAME |\ _DPRINTK_FLAGS_INCL_LINENO | _DPRINTK_FLAGS_INCL_TID |\ - _DPRINTK_FLAGS_INCL_SOURCENAME) + _DPRINTK_FLAGS_INCL_SOURCENAME | _DPRINTK_FLAGS_INCL_STACK) #if defined DEBUG #define _DPRINTK_FLAGS_DEFAULT _DPRINTK_FLAGS_PRINT @@ -160,6 +161,12 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, const struct ib_device *ibdev, const char *fmt, ...); +#define __dynamic_dump_stack(desc) \ +{ \ + if (desc.flags & _DPRINTK_FLAGS_INCL_STACK) \ + dump_stack(); \ +} + #define DEFINE_DYNAMIC_DEBUG_METADATA_CLS(name, cls, fmt) \ static struct _ddebug __aligned(8) \ __section("__dyndbg") name = { \ @@ -220,8 +227,10 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, */ #define __dynamic_func_call_cls(id, cls, fmt, func, ...) do { \ DEFINE_DYNAMIC_DEBUG_METADATA_CLS(id, cls, fmt); \ - if (DYNAMIC_DEBUG_BRANCH(id)) \ + if (DYNAMIC_DEBUG_BRANCH(id)) { \ func(&id, ##__VA_ARGS__); \ + __dynamic_dump_stack(id); \ + } \ } while (0) #define __dynamic_func_call(id, fmt, func, ...) \ __dynamic_func_call_cls(id, _DPRINTK_CLASS_DFLT, fmt, \ @@ -229,8 +238,10 @@ void __dynamic_ibdev_dbg(struct _ddebug *descriptor, #define __dynamic_func_call_cls_no_desc(id, cls, fmt, func, ...) do { \ DEFINE_DYNAMIC_DEBUG_METADATA_CLS(id, cls, fmt); \ - if (DYNAMIC_DEBUG_BRANCH(id)) \ + if (DYNAMIC_DEBUG_BRANCH(id)) { \ func(__VA_ARGS__); \ + __dynamic_dump_stack(id); \ + } \ } while (0) #define __dynamic_func_call_no_desc(id, fmt, func, ...) \ __dynamic_func_call_cls_no_desc(id, _DPRINTK_CLASS_DFLT, \ diff --git a/include/linux/font.h b/include/linux/font.h index 81caffd51bb4..fd8625cd76b2 100644 --- a/include/linux/font.h +++ b/include/linux/font.h @@ -35,6 +35,7 @@ struct font_desc { #define FONT6x10_IDX 10 #define TER16x32_IDX 11 #define FONT6x8_IDX 12 +#define TER10x18_IDX 13 extern const struct font_desc font_vga_8x8, font_vga_8x16, @@ -48,7 +49,8 @@ extern const struct font_desc font_vga_8x8, font_mini_4x6, font_6x10, font_ter_16x32, - font_6x8; + font_6x8, + font_ter_10x18; /* Find a font with a specific name */ diff --git a/include/linux/ioport.h b/include/linux/ioport.h index e8b2d6aa4013..9afa30f9346f 100644 --- a/include/linux/ioport.h +++ b/include/linux/ioport.h @@ -334,6 +334,15 @@ static inline bool resource_union(const struct resource *r1, const struct resour return true; } +/* + * Check if this resource is added to a resource tree or detached. Caller is + * responsible for not racing assignment. + */ +static inline bool resource_assigned(struct resource *res) +{ + return res->parent; +} + int find_resource_space(struct resource *root, struct resource *new, resource_size_t size, struct resource_constraint *constraint); diff --git a/include/linux/kexec_handover.h b/include/linux/kexec_handover.h index 25042c1d8d54..5f7b9de97e8d 100644 --- a/include/linux/kexec_handover.h +++ b/include/linux/kexec_handover.h @@ -2,22 +2,16 @@ #ifndef LINUX_KEXEC_HANDOVER_H #define LINUX_KEXEC_HANDOVER_H -#include <linux/types.h> +#include <linux/err.h> #include <linux/errno.h> +#include <linux/types.h> struct kho_scratch { phys_addr_t addr; phys_addr_t size; }; -/* KHO Notifier index */ -enum kho_event { - KEXEC_KHO_FINALIZE = 0, - KEXEC_KHO_ABORT = 1, -}; - struct folio; -struct notifier_block; struct page; #define DECLARE_KHOSER_PTR(name, type) \ @@ -37,8 +31,6 @@ struct page; (typeof((s).ptr))((s).phys ? phys_to_virt((s).phys) : NULL); \ }) -struct kho_serialization; - struct kho_vmalloc_chunk; struct kho_vmalloc { DECLARE_KHOSER_PTR(first, struct kho_vmalloc_chunk *); @@ -52,17 +44,21 @@ bool kho_is_enabled(void); bool is_kho_boot(void); int kho_preserve_folio(struct folio *folio); +void kho_unpreserve_folio(struct folio *folio); int kho_preserve_pages(struct page *page, unsigned int nr_pages); +void kho_unpreserve_pages(struct page *page, unsigned int nr_pages); int kho_preserve_vmalloc(void *ptr, struct kho_vmalloc *preservation); +void kho_unpreserve_vmalloc(struct kho_vmalloc *preservation); +void *kho_alloc_preserve(size_t size); +void kho_unpreserve_free(void *mem); +void kho_restore_free(void *mem); struct folio *kho_restore_folio(phys_addr_t phys); struct page *kho_restore_pages(phys_addr_t phys, unsigned int nr_pages); void *kho_restore_vmalloc(const struct kho_vmalloc *preservation); -int kho_add_subtree(struct kho_serialization *ser, const char *name, void *fdt); +int kho_add_subtree(const char *name, void *fdt); +void kho_remove_subtree(void *fdt); int kho_retrieve_subtree(const char *name, phys_addr_t *phys); -int register_kho_notifier(struct notifier_block *nb); -int unregister_kho_notifier(struct notifier_block *nb); - void kho_memory_init(void); void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t scratch_phys, @@ -83,17 +79,31 @@ static inline int kho_preserve_folio(struct folio *folio) return -EOPNOTSUPP; } +static inline void kho_unpreserve_folio(struct folio *folio) { } + static inline int kho_preserve_pages(struct page *page, unsigned int nr_pages) { return -EOPNOTSUPP; } +static inline void kho_unpreserve_pages(struct page *page, unsigned int nr_pages) { } + static inline int kho_preserve_vmalloc(void *ptr, struct kho_vmalloc *preservation) { return -EOPNOTSUPP; } +static inline void kho_unpreserve_vmalloc(struct kho_vmalloc *preservation) { } + +static inline void *kho_alloc_preserve(size_t size) +{ + return ERR_PTR(-EOPNOTSUPP); +} + +static inline void kho_unpreserve_free(void *mem) { } +static inline void kho_restore_free(void *mem) { } + static inline struct folio *kho_restore_folio(phys_addr_t phys) { return NULL; @@ -110,30 +120,19 @@ static inline void *kho_restore_vmalloc(const struct kho_vmalloc *preservation) return NULL; } -static inline int kho_add_subtree(struct kho_serialization *ser, - const char *name, void *fdt) -{ - return -EOPNOTSUPP; -} - -static inline int kho_retrieve_subtree(const char *name, phys_addr_t *phys) +static inline int kho_add_subtree(const char *name, void *fdt) { return -EOPNOTSUPP; } -static inline int register_kho_notifier(struct notifier_block *nb) -{ - return -EOPNOTSUPP; -} +static inline void kho_remove_subtree(void *fdt) { } -static inline int unregister_kho_notifier(struct notifier_block *nb) +static inline int kho_retrieve_subtree(const char *name, phys_addr_t *phys) { return -EOPNOTSUPP; } -static inline void kho_memory_init(void) -{ -} +static inline void kho_memory_init(void) { } static inline void kho_populate(phys_addr_t fdt_phys, u64 fdt_len, phys_addr_t scratch_phys, u64 scratch_len) diff --git a/include/linux/kho/abi/luo.h b/include/linux/kho/abi/luo.h new file mode 100644 index 000000000000..bb099c92e469 --- /dev/null +++ b/include/linux/kho/abi/luo.h @@ -0,0 +1,166 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +/** + * DOC: Live Update Orchestrator ABI + * + * This header defines the stable Application Binary Interface used by the + * Live Update Orchestrator to pass state from a pre-update kernel to a + * post-update kernel. The ABI is built upon the Kexec HandOver framework + * and uses a Flattened Device Tree to describe the preserved data. + * + * This interface is a contract. Any modification to the FDT structure, node + * properties, compatible strings, or the layout of the `__packed` serialization + * structures defined here constitutes a breaking change. Such changes require + * incrementing the version number in the relevant `_COMPATIBLE` string to + * prevent a new kernel from misinterpreting data from an old kernel. + * + * Changes are allowed provided the compatibility version is incremented; + * however, backward/forward compatibility is only guaranteed for kernels + * supporting the same ABI version. + * + * FDT Structure Overview: + * The entire LUO state is encapsulated within a single KHO entry named "LUO". + * This entry contains an FDT with the following layout: + * + * .. code-block:: none + * + * / { + * compatible = "luo-v1"; + * liveupdate-number = <...>; + * + * luo-session { + * compatible = "luo-session-v1"; + * luo-session-header = <phys_addr_of_session_header_ser>; + * }; + * }; + * + * Main LUO Node (/): + * + * - compatible: "luo-v1" + * Identifies the overall LUO ABI version. + * - liveupdate-number: u64 + * A counter tracking the number of successful live updates performed. + * + * Session Node (luo-session): + * This node describes all preserved user-space sessions. + * + * - compatible: "luo-session-v1" + * Identifies the session ABI version. + * - luo-session-header: u64 + * The physical address of a `struct luo_session_header_ser`. This structure + * is the header for a contiguous block of memory containing an array of + * `struct luo_session_ser`, one for each preserved session. + * + * Serialization Structures: + * The FDT properties point to memory regions containing arrays of simple, + * `__packed` structures. These structures contain the actual preserved state. + * + * - struct luo_session_header_ser: + * Header for the session array. Contains the total page count of the + * preserved memory block and the number of `struct luo_session_ser` + * entries that follow. + * + * - struct luo_session_ser: + * Metadata for a single session, including its name and a physical pointer + * to another preserved memory block containing an array of + * `struct luo_file_ser` for all files in that session. + * + * - struct luo_file_ser: + * Metadata for a single preserved file. Contains the `compatible` string to + * find the correct handler in the new kernel, a user-provided `token` for + * identification, and an opaque `data` handle for the handler to use. + */ + +#ifndef _LINUX_KHO_ABI_LUO_H +#define _LINUX_KHO_ABI_LUO_H + +#include <uapi/linux/liveupdate.h> + +/* + * The LUO FDT hooks all LUO state for sessions, fds, etc. + * In the root it also carries "liveupdate-number" 64-bit property that + * corresponds to the number of live-updates performed on this machine. + */ +#define LUO_FDT_SIZE PAGE_SIZE +#define LUO_FDT_KHO_ENTRY_NAME "LUO" +#define LUO_FDT_COMPATIBLE "luo-v1" +#define LUO_FDT_LIVEUPDATE_NUM "liveupdate-number" + +#define LIVEUPDATE_HNDL_COMPAT_LENGTH 48 + +/** + * struct luo_file_ser - Represents the serialized preserves files. + * @compatible: File handler compatible string. + * @data: Private data + * @token: User provided token for this file + * + * If this structure is modified, LUO_SESSION_COMPATIBLE must be updated. + */ +struct luo_file_ser { + char compatible[LIVEUPDATE_HNDL_COMPAT_LENGTH]; + u64 data; + u64 token; +} __packed; + +/** + * struct luo_file_set_ser - Represents the serialized metadata for file set + * @files: The physical address of a contiguous memory block that holds + * the serialized state of files (array of luo_file_ser) in this file + * set. + * @count: The total number of files that were part of this session during + * serialization. Used for iteration and validation during + * restoration. + */ +struct luo_file_set_ser { + u64 files; + u64 count; +} __packed; + +/* + * LUO FDT session node + * LUO_FDT_SESSION_HEADER: is a u64 physical address of struct + * luo_session_header_ser + */ +#define LUO_FDT_SESSION_NODE_NAME "luo-session" +#define LUO_FDT_SESSION_COMPATIBLE "luo-session-v2" +#define LUO_FDT_SESSION_HEADER "luo-session-header" + +/** + * struct luo_session_header_ser - Header for the serialized session data block. + * @count: The number of `struct luo_session_ser` entries that immediately + * follow this header in the memory block. + * + * This structure is located at the beginning of a contiguous block of + * physical memory preserved across the kexec. It provides the necessary + * metadata to interpret the array of session entries that follow. + * + * If this structure is modified, `LUO_FDT_SESSION_COMPATIBLE` must be updated. + */ +struct luo_session_header_ser { + u64 count; +} __packed; + +/** + * struct luo_session_ser - Represents the serialized metadata for a LUO session. + * @name: The unique name of the session, provided by the userspace at + * the time of session creation. + * @file_set_ser: Serialized files belonging to this session, + * + * This structure is used to package session-specific metadata for transfer + * between kernels via Kexec Handover. An array of these structures (one per + * session) is created and passed to the new kernel, allowing it to reconstruct + * the session context. + * + * If this structure is modified, `LUO_FDT_SESSION_COMPATIBLE` must be updated. + */ +struct luo_session_ser { + char name[LIVEUPDATE_SESSION_NAME_LENGTH]; + struct luo_file_set_ser file_set_ser; +} __packed; + +#endif /* _LINUX_KHO_ABI_LUO_H */ diff --git a/include/linux/kho/abi/memfd.h b/include/linux/kho/abi/memfd.h new file mode 100644 index 000000000000..da7d063474a1 --- /dev/null +++ b/include/linux/kho/abi/memfd.h @@ -0,0 +1,77 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + * + * Copyright (C) 2025 Amazon.com Inc. or its affiliates. + * Pratyush Yadav <ptyadav@amazon.de> + */ + +#ifndef _LINUX_KHO_ABI_MEMFD_H +#define _LINUX_KHO_ABI_MEMFD_H + +#include <linux/types.h> +#include <linux/kexec_handover.h> + +/** + * DOC: memfd Live Update ABI + * + * This header defines the ABI for preserving the state of a memfd across a + * kexec reboot using the LUO. + * + * The state is serialized into a packed structure `struct memfd_luo_ser` + * which is handed over to the next kernel via the KHO mechanism. + * + * This interface is a contract. Any modification to the structure layout + * constitutes a breaking change. Such changes require incrementing the + * version number in the MEMFD_LUO_FH_COMPATIBLE string. + */ + +/** + * MEMFD_LUO_FOLIO_DIRTY - The folio is dirty. + * + * This flag indicates the folio contains data from user. A non-dirty folio is + * one that was allocated (say using fallocate(2)) but not written to. + */ +#define MEMFD_LUO_FOLIO_DIRTY BIT(0) + +/** + * MEMFD_LUO_FOLIO_UPTODATE - The folio is up-to-date. + * + * An up-to-date folio has been zeroed out. shmem zeroes out folios on first + * use. This flag tracks which folios need zeroing. + */ +#define MEMFD_LUO_FOLIO_UPTODATE BIT(1) + +/** + * struct memfd_luo_folio_ser - Serialized state of a single folio. + * @pfn: The page frame number of the folio. + * @flags: Flags to describe the state of the folio. + * @index: The page offset (pgoff_t) of the folio within the original file. + */ +struct memfd_luo_folio_ser { + u64 pfn:52; + u64 flags:12; + u64 index; +} __packed; + +/** + * struct memfd_luo_ser - Main serialization structure for a memfd. + * @pos: The file's current position (f_pos). + * @size: The total size of the file in bytes (i_size). + * @nr_folios: Number of folios in the folios array. + * @folios: KHO vmalloc descriptor pointing to the array of + * struct memfd_luo_folio_ser. + */ +struct memfd_luo_ser { + u64 pos; + u64 size; + u64 nr_folios; + struct kho_vmalloc folios; +} __packed; + +/* The compatibility string for memfd file handler */ +#define MEMFD_LUO_FH_COMPATIBLE "memfd-v1" + +#endif /* _LINUX_KHO_ABI_MEMFD_H */ diff --git a/include/linux/liveupdate.h b/include/linux/liveupdate.h new file mode 100644 index 000000000000..a7f6ee5b6771 --- /dev/null +++ b/include/linux/liveupdate.h @@ -0,0 +1,138 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ +#ifndef _LINUX_LIVEUPDATE_H +#define _LINUX_LIVEUPDATE_H + +#include <linux/bug.h> +#include <linux/compiler.h> +#include <linux/kho/abi/luo.h> +#include <linux/list.h> +#include <linux/types.h> +#include <uapi/linux/liveupdate.h> + +struct liveupdate_file_handler; +struct file; + +/** + * struct liveupdate_file_op_args - Arguments for file operation callbacks. + * @handler: The file handler being called. + * @retrieved: The retrieve status for the 'can_finish / finish' + * operation. + * @file: The file object. For retrieve: [OUT] The callback sets + * this to the new file. For other ops: [IN] The caller sets + * this to the file being operated on. + * @serialized_data: The opaque u64 handle, preserve/prepare/freeze may update + * this field. + * @private_data: Private data for the file used to hold runtime state that + * is not preserved. Set by the handler's .preserve() + * callback, and must be freed in the handler's + * .unpreserve() callback. + * + * This structure bundles all parameters for the file operation callbacks. + * The 'data' and 'file' fields are used for both input and output. + */ +struct liveupdate_file_op_args { + struct liveupdate_file_handler *handler; + bool retrieved; + struct file *file; + u64 serialized_data; + void *private_data; +}; + +/** + * struct liveupdate_file_ops - Callbacks for live-updatable files. + * @can_preserve: Required. Lightweight check to see if this handler is + * compatible with the given file. + * @preserve: Required. Performs state-saving for the file. + * @unpreserve: Required. Cleans up any resources allocated by @preserve. + * @freeze: Optional. Final actions just before kernel transition. + * @unfreeze: Optional. Undo freeze operations. + * @retrieve: Required. Restores the file in the new kernel. + * @can_finish: Optional. Check if this FD can finish, i.e. all restoration + * pre-requirements for this FD are satisfied. Called prior to + * finish, in order to do successful finish calls for all + * resources in the session. + * @finish: Required. Final cleanup in the new kernel. + * @owner: Module reference + * + * All operations (except can_preserve) receive a pointer to a + * 'struct liveupdate_file_op_args' containing the necessary context. + */ +struct liveupdate_file_ops { + bool (*can_preserve)(struct liveupdate_file_handler *handler, + struct file *file); + int (*preserve)(struct liveupdate_file_op_args *args); + void (*unpreserve)(struct liveupdate_file_op_args *args); + int (*freeze)(struct liveupdate_file_op_args *args); + void (*unfreeze)(struct liveupdate_file_op_args *args); + int (*retrieve)(struct liveupdate_file_op_args *args); + bool (*can_finish)(struct liveupdate_file_op_args *args); + void (*finish)(struct liveupdate_file_op_args *args); + struct module *owner; +}; + +/** + * struct liveupdate_file_handler - Represents a handler for a live-updatable file type. + * @ops: Callback functions + * @compatible: The compatibility string (e.g., "memfd-v1", "vfiofd-v1") + * that uniquely identifies the file type this handler + * supports. This is matched against the compatible string + * associated with individual &struct file instances. + * + * Modules that want to support live update for specific file types should + * register an instance of this structure. LUO uses this registration to + * determine if a given file can be preserved and to find the appropriate + * operations to manage its state across the update. + */ +struct liveupdate_file_handler { + const struct liveupdate_file_ops *ops; + const char compatible[LIVEUPDATE_HNDL_COMPAT_LENGTH]; + + /* private: */ + + /* + * Used for linking this handler instance into a global list of + * registered file handlers. + */ + struct list_head __private list; +}; + +#ifdef CONFIG_LIVEUPDATE + +/* Return true if live update orchestrator is enabled */ +bool liveupdate_enabled(void); + +/* Called during kexec to tell LUO that entered into reboot */ +int liveupdate_reboot(void); + +int liveupdate_register_file_handler(struct liveupdate_file_handler *fh); +int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh); + +#else /* CONFIG_LIVEUPDATE */ + +static inline bool liveupdate_enabled(void) +{ + return false; +} + +static inline int liveupdate_reboot(void) +{ + return 0; +} + +static inline int liveupdate_register_file_handler(struct liveupdate_file_handler *fh) +{ + return -EOPNOTSUPP; +} + +static inline int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh) +{ + return -EOPNOTSUPP; +} + +#endif /* CONFIG_LIVEUPDATE */ +#endif /* _LINUX_LIVEUPDATE_H */ diff --git a/include/linux/lockd/lockd.h b/include/linux/lockd/lockd.h index c8f0f9458f2c..330e38776bb2 100644 --- a/include/linux/lockd/lockd.h +++ b/include/linux/lockd/lockd.h @@ -12,6 +12,7 @@ /* XXX: a lot of this should really be under fs/lockd. */ +#include <linux/exportfs.h> #include <linux/in.h> #include <linux/in6.h> #include <net/ipv6.h> @@ -307,7 +308,7 @@ void nlmsvc_invalidate_all(void); int nlmsvc_unlock_all_by_sb(struct super_block *sb); int nlmsvc_unlock_all_by_ip(struct sockaddr *server_addr); -static inline struct file *nlmsvc_file_file(struct nlm_file *file) +static inline struct file *nlmsvc_file_file(const struct nlm_file *file) { return file->f_file[O_RDONLY] ? file->f_file[O_RDONLY] : file->f_file[O_WRONLY]; @@ -318,6 +319,12 @@ static inline struct inode *nlmsvc_file_inode(struct nlm_file *file) return file_inode(nlmsvc_file_file(file)); } +static inline bool +nlmsvc_file_cannot_lock(const struct nlm_file *file) +{ + return exportfs_cannot_lock(nlmsvc_file_file(file)->f_path.dentry->d_sb->s_export_op); +} + static inline int __nlm_privileged_request4(const struct sockaddr *sap) { const struct sockaddr_in *sin = (struct sockaddr_in *)sap; diff --git a/include/linux/math.h b/include/linux/math.h index 0198c92cbe3e..6dc1d1d32fbc 100644 --- a/include/linux/math.h +++ b/include/linux/math.h @@ -148,11 +148,16 @@ __STRUCT_FRACT(u32) /** * abs - return absolute value of an argument - * @x: the value. If it is unsigned type, it is converted to signed type first. - * char is treated as if it was signed (regardless of whether it really is) - * but the macro's return type is preserved as char. + * @x: the value. * - * Return: an absolute value of x. + * If it is unsigned type, @x is converted to signed type first. + * char is treated as if it was signed (regardless of whether it really is) + * but the macro's return type is preserved as char. + * + * NOTE, for signed type if @x is the minimum, the returned result is undefined + * as there is not enough bits to represent it as a positive number. + * + * Return: an absolute value of @x. */ #define abs(x) __abs_choose_expr(x, long long, \ __abs_choose_expr(x, long, \ diff --git a/include/linux/math64.h b/include/linux/math64.h index 6aaccc1626ab..cc305206d89f 100644 --- a/include/linux/math64.h +++ b/include/linux/math64.h @@ -158,6 +158,17 @@ static inline u64 mul_u32_u32(u32 a, u32 b) } #endif +#ifndef add_u64_u32 +/* + * Many a GCC version also messes this up. + * Zero extending b and then spilling everything to stack. + */ +static inline u64 add_u64_u32(u64 a, u32 b) +{ + return a + b; +} +#endif + #if defined(CONFIG_ARCH_SUPPORTS_INT128) && defined(__SIZEOF_INT128__) #ifndef mul_u64_u32_shr @@ -282,7 +293,53 @@ static inline u64 mul_u64_u32_div(u64 a, u32 mul, u32 divisor) } #endif /* mul_u64_u32_div */ -u64 mul_u64_u64_div_u64(u64 a, u64 mul, u64 div); +/** + * mul_u64_add_u64_div_u64 - unsigned 64bit multiply, add, and divide + * @a: first unsigned 64bit multiplicand + * @b: second unsigned 64bit multiplicand + * @c: unsigned 64bit addend + * @d: unsigned 64bit divisor + * + * Multiply two 64bit values together to generate a 128bit product + * add a third value and then divide by a fourth. + * The Generic code divides by 0 if @d is zero and returns ~0 on overflow. + * Architecture specific code may trap on zero or overflow. + * + * Return: (@a * @b + @c) / @d + */ +u64 mul_u64_add_u64_div_u64(u64 a, u64 b, u64 c, u64 d); + +/** + * mul_u64_u64_div_u64 - unsigned 64bit multiply and divide + * @a: first unsigned 64bit multiplicand + * @b: second unsigned 64bit multiplicand + * @d: unsigned 64bit divisor + * + * Multiply two 64bit values together to generate a 128bit product + * and then divide by a third value. + * The Generic code divides by 0 if @d is zero and returns ~0 on overflow. + * Architecture specific code may trap on zero or overflow. + * + * Return: @a * @b / @d + */ +#define mul_u64_u64_div_u64(a, b, d) mul_u64_add_u64_div_u64(a, b, 0, d) + +/** + * mul_u64_u64_div_u64_roundup - unsigned 64bit multiply and divide rounded up + * @a: first unsigned 64bit multiplicand + * @b: second unsigned 64bit multiplicand + * @d: unsigned 64bit divisor + * + * Multiply two 64bit values together to generate a 128bit product + * and then divide and round up. + * The Generic code divides by 0 if @d is zero and returns ~0 on overflow. + * Architecture specific code may trap on zero or overflow. + * + * Return: (@a * @b + @d - 1) / @d + */ +#define mul_u64_u64_div_u64_roundup(a, b, d) \ + ({ u64 _tmp = (d); mul_u64_add_u64_div_u64(a, b, _tmp - 1, _tmp); }) + /** * DIV64_U64_ROUND_UP - unsigned 64bit divide with 64bit divisor rounded up diff --git a/include/linux/nodemask.h b/include/linux/nodemask.h index 7ad1f5c7407e..bd38648c998d 100644 --- a/include/linux/nodemask.h +++ b/include/linux/nodemask.h @@ -245,18 +245,18 @@ static __always_inline int __nodes_weight(const nodemask_t *srcp, unsigned int n } /* FIXME: better would be to fix all architectures to never return - > MAX_NUMNODES, then the silly min_ts could be dropped. */ + > MAX_NUMNODES, then the silly min()s could be dropped. */ #define first_node(src) __first_node(&(src)) static __always_inline unsigned int __first_node(const nodemask_t *srcp) { - return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES)); + return min(MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES)); } #define next_node(n, src) __next_node((n), &(src)) static __always_inline unsigned int __next_node(int n, const nodemask_t *srcp) { - return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1)); + return min(MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1)); } /* @@ -293,8 +293,7 @@ static __always_inline void init_nodemask_of_node(nodemask_t *mask, int node) #define first_unset_node(mask) __first_unset_node(&(mask)) static __always_inline unsigned int __first_unset_node(const nodemask_t *maskp) { - return min_t(unsigned int, MAX_NUMNODES, - find_first_zero_bit(maskp->bits, MAX_NUMNODES)); + return min(MAX_NUMNODES, find_first_zero_bit(maskp->bits, MAX_NUMNODES)); } #define NODE_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(MAX_NUMNODES) diff --git a/include/linux/objtool.h b/include/linux/objtool.h index b18ab53561c9..9a00e701454c 100644 --- a/include/linux/objtool.h +++ b/include/linux/objtool.h @@ -12,7 +12,7 @@ #define UNWIND_HINT(type, sp_reg, sp_offset, signal) \ "987: \n\t" \ ".pushsection .discard.unwind_hints\n\t" \ - ANNOTATE_DATA_SPECIAL \ + ANNOTATE_DATA_SPECIAL "\n\t" \ /* struct unwind_hint */ \ ".long 987b - .\n\t" \ ".short " __stringify(sp_offset) "\n\t" \ diff --git a/include/linux/once_lite.h b/include/linux/once_lite.h index 27de7bc32a06..236592c4eeb1 100644 --- a/include/linux/once_lite.h +++ b/include/linux/once_lite.h @@ -16,7 +16,7 @@ bool __ret_cond = !!(condition); \ bool __ret_once = false; \ \ - if (unlikely(__ret_cond && !__already_done)) { \ + if (unlikely(__ret_cond) && unlikely(!__already_done)) {\ __already_done = true; \ __ret_once = true; \ } \ diff --git a/include/linux/panic.h b/include/linux/panic.h index 6f972a66c13e..a00bc0937698 100644 --- a/include/linux/panic.h +++ b/include/linux/panic.h @@ -86,7 +86,6 @@ static inline void set_arch_panic_timeout(int timeout, int arch_default_timeout) struct taint_flag { char c_true; /* character printed when tainted */ char c_false; /* character printed when not tainted */ - bool module; /* also show as a per-module taint flag */ const char *desc; /* verbose description of the set taint flag */ }; diff --git a/include/linux/pci-doe.h b/include/linux/pci-doe.h index 1f14aed4354b..bd4346a7c4e7 100644 --- a/include/linux/pci-doe.h +++ b/include/linux/pci-doe.h @@ -15,6 +15,10 @@ struct pci_doe_mb; +#define PCI_DOE_FEATURE_DISCOVERY 0 +#define PCI_DOE_FEATURE_CMA 1 +#define PCI_DOE_FEATURE_SSESSION 2 + struct pci_doe_mb *pci_find_doe_mailbox(struct pci_dev *pdev, u16 vendor, u8 type); diff --git a/include/linux/pci-ide.h b/include/linux/pci-ide.h new file mode 100644 index 000000000000..37a1ad9501b0 --- /dev/null +++ b/include/linux/pci-ide.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Common helpers for drivers (e.g. low-level PCI/TSM drivers) implementing the + * IDE key management protocol (IDE_KM) as defined by: + * PCIe r7.0 section 6.33 Integrity & Data Encryption (IDE) + * + * Copyright(c) 2024-2025 Intel Corporation. All rights reserved. + */ + +#ifndef __PCI_IDE_H__ +#define __PCI_IDE_H__ + +enum pci_ide_partner_select { + PCI_IDE_EP, + PCI_IDE_RP, + PCI_IDE_PARTNER_MAX, + /* + * In addition to the resources in each partner port the + * platform / host-bridge additionally has a Stream ID pool that + * it shares across root ports. Let pci_ide_stream_alloc() use + * the alloc_stream_index() helper as endpoints and root ports. + */ + PCI_IDE_HB = PCI_IDE_PARTNER_MAX, +}; + +/** + * struct pci_ide_partner - Per port pair Selective IDE Stream settings + * @rid_start: Partner Port Requester ID range start + * @rid_end: Partner Port Requester ID range end + * @stream_index: Selective IDE Stream Register Block selection + * @mem_assoc: PCI bus memory address association for targeting peer partner + * @pref_assoc: PCI bus prefetchable memory address association for + * targeting peer partner + * @default_stream: Endpoint uses this stream for all upstream TLPs regardless of + * address and RID association registers + * @setup: flag to track whether to run pci_ide_stream_teardown() for this + * partner slot + * @enable: flag whether to run pci_ide_stream_disable() for this partner slot + * + * By default, pci_ide_stream_alloc() initializes @mem_assoc and @pref_assoc + * with the immediate ancestor downstream port memory ranges (i.e. Type 1 + * Configuration Space Header values). Caller may zero size ({0, -1}) the range + * to drop it from consideration at pci_ide_stream_setup() time. + */ +struct pci_ide_partner { + u16 rid_start; + u16 rid_end; + u8 stream_index; + struct pci_bus_region mem_assoc; + struct pci_bus_region pref_assoc; + unsigned int default_stream:1; + unsigned int setup:1; + unsigned int enable:1; +}; + +/** + * struct pci_ide_regs - Hardware register association settings for Selective + * IDE Streams + * @rid1: IDE RID Association Register 1 + * @rid2: IDE RID Association Register 2 + * @addr: Up to two address association blocks (IDE Address Association Register + * 1 through 3) for MMIO and prefetchable MMIO + * @nr_addr: Number of address association blocks initialized + * + * See pci_ide_stream_to_regs() + */ +struct pci_ide_regs { + u32 rid1; + u32 rid2; + struct { + u32 assoc1; + u32 assoc2; + u32 assoc3; + } addr[2]; + int nr_addr; +}; + +/** + * struct pci_ide - PCIe Selective IDE Stream descriptor + * @pdev: PCIe Endpoint in the pci_ide_partner pair + * @partner: per-partner settings + * @host_bridge_stream: allocated from host bridge @ide_stream_ida pool + * @stream_id: unique Stream ID (within Partner Port pairing) + * @name: name of the established Selective IDE Stream in sysfs + * @tsm_dev: For TSM established IDE, the TSM device context + * + * Negative @stream_id values indicate "uninitialized" on the + * expectation that with TSM established IDE the TSM owns the stream_id + * allocation. + */ +struct pci_ide { + struct pci_dev *pdev; + struct pci_ide_partner partner[PCI_IDE_PARTNER_MAX]; + u8 host_bridge_stream; + int stream_id; + const char *name; + struct tsm_dev *tsm_dev; +}; + +/* + * Some devices need help with aliased stream-ids even for idle streams. Use + * this id as the "never enabled" place holder. + */ +#define PCI_IDE_RESERVED_STREAM_ID 255 + +void pci_ide_set_nr_streams(struct pci_host_bridge *hb, u16 nr); +struct pci_ide_partner *pci_ide_to_settings(struct pci_dev *pdev, + struct pci_ide *ide); +struct pci_ide *pci_ide_stream_alloc(struct pci_dev *pdev); +void pci_ide_stream_free(struct pci_ide *ide); +int pci_ide_stream_register(struct pci_ide *ide); +void pci_ide_stream_unregister(struct pci_ide *ide); +void pci_ide_stream_setup(struct pci_dev *pdev, struct pci_ide *ide); +void pci_ide_stream_teardown(struct pci_dev *pdev, struct pci_ide *ide); +int pci_ide_stream_enable(struct pci_dev *pdev, struct pci_ide *ide); +void pci_ide_stream_disable(struct pci_dev *pdev, struct pci_ide *ide); +void pci_ide_stream_release(struct pci_ide *ide); +DEFINE_FREE(pci_ide_stream_release, struct pci_ide *, if (_T) pci_ide_stream_release(_T)) +#endif /* __PCI_IDE_H__ */ diff --git a/include/linux/pci-tsm.h b/include/linux/pci-tsm.h new file mode 100644 index 000000000000..a6435aba03f9 --- /dev/null +++ b/include/linux/pci-tsm.h @@ -0,0 +1,243 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PCI_TSM_H +#define __PCI_TSM_H +#include <linux/mutex.h> +#include <linux/pci.h> +#include <linux/sockptr.h> + +struct pci_tsm; +struct tsm_dev; +struct kvm; +enum pci_tsm_req_scope; + +/* + * struct pci_tsm_ops - manage confidential links and security state + * @link_ops: Coordinate PCIe SPDM and IDE establishment via a platform TSM. + * Provide a secure session transport for TDISP state management + * (typically bare metal physical function operations). + * @devsec_ops: Lock, unlock, and interrogate the security state of the + * function via the platform TSM (typically virtual function + * operations). + * + * This operations are mutually exclusive either a tsm_dev instance + * manages physical link properties or it manages function security + * states like TDISP lock/unlock. + */ +struct pci_tsm_ops { + /* + * struct pci_tsm_link_ops - Manage physical link and the TSM/DSM session + * @probe: establish context with the TSM (allocate / wrap 'struct + * pci_tsm') for follow-on link operations + * @remove: destroy link operations context + * @connect: establish / validate a secure connection (e.g. IDE) + * with the device + * @disconnect: teardown the secure link + * @bind: bind a TDI in preparation for it to be accepted by a TVM + * @unbind: remove a TDI from secure operation with a TVM + * @guest_req: marshal TVM information and state change requests + * + * Context: @probe, @remove, @connect, and @disconnect run under + * pci_tsm_rwsem held for write to sync with TSM unregistration and + * mutual exclusion of @connect and @disconnect. @connect and + * @disconnect additionally run under the DSM lock (struct + * pci_tsm_pf0::lock) as well as @probe and @remove of the subfunctions. + * @bind, @unbind, and @guest_req run under pci_tsm_rwsem held for read + * and the DSM lock. + */ + struct_group_tagged(pci_tsm_link_ops, link_ops, + struct pci_tsm *(*probe)(struct tsm_dev *tsm_dev, + struct pci_dev *pdev); + void (*remove)(struct pci_tsm *tsm); + int (*connect)(struct pci_dev *pdev); + void (*disconnect)(struct pci_dev *pdev); + struct pci_tdi *(*bind)(struct pci_dev *pdev, + struct kvm *kvm, u32 tdi_id); + void (*unbind)(struct pci_tdi *tdi); + ssize_t (*guest_req)(struct pci_tdi *tdi, + enum pci_tsm_req_scope scope, + sockptr_t req_in, size_t in_len, + sockptr_t req_out, size_t out_len, + u64 *tsm_code); + ); + + /* + * struct pci_tsm_devsec_ops - Manage the security state of the function + * @lock: establish context with the TSM (allocate / wrap 'struct + * pci_tsm') for follow-on security state transitions from the + * LOCKED state + * @unlock: destroy TSM context and return device to UNLOCKED state + * + * Context: @lock and @unlock run under pci_tsm_rwsem held for write to + * sync with TSM unregistration and each other + */ + struct_group_tagged(pci_tsm_devsec_ops, devsec_ops, + struct pci_tsm *(*lock)(struct tsm_dev *tsm_dev, + struct pci_dev *pdev); + void (*unlock)(struct pci_tsm *tsm); + ); +}; + +/** + * struct pci_tdi - Core TEE I/O Device Interface (TDI) context + * @pdev: host side representation of guest-side TDI + * @kvm: TEE VM context of bound TDI + * @tdi_id: Identifier (virtual BDF) for the TDI as referenced by the TSM and DSM + */ +struct pci_tdi { + struct pci_dev *pdev; + struct kvm *kvm; + u32 tdi_id; +}; + +/** + * struct pci_tsm - Core TSM context for a given PCIe endpoint + * @pdev: Back ref to device function, distinguishes type of pci_tsm context + * @dsm_dev: PCI Device Security Manager for link operations on @pdev + * @tsm_dev: PCI TEE Security Manager device for Link Confidentiality or Device + * Function Security operations + * @tdi: TDI context established by the @bind link operation + * + * This structure is wrapped by low level TSM driver data and returned by + * probe()/lock(), it is freed by the corresponding remove()/unlock(). + * + * For link operations it serves to cache the association between a Device + * Security Manager (DSM) and the functions that manager can assign to a TVM. + * That can be "self", for assigning function0 of a TEE I/O device, a + * sub-function (SR-IOV virtual function, or non-function0 + * multifunction-device), or a downstream endpoint (PCIe upstream switch-port as + * DSM). + */ +struct pci_tsm { + struct pci_dev *pdev; + struct pci_dev *dsm_dev; + struct tsm_dev *tsm_dev; + struct pci_tdi *tdi; +}; + +/** + * struct pci_tsm_pf0 - Physical Function 0 TDISP link context + * @base_tsm: generic core "tsm" context + * @lock: mutual exclustion for pci_tsm_ops invocation + * @doe_mb: PCIe Data Object Exchange mailbox + */ +struct pci_tsm_pf0 { + struct pci_tsm base_tsm; + struct mutex lock; + struct pci_doe_mb *doe_mb; +}; + +/* physical function0 and capable of 'connect' */ +static inline bool is_pci_tsm_pf0(struct pci_dev *pdev) +{ + if (!pdev) + return false; + + if (!pci_is_pcie(pdev)) + return false; + + if (pdev->is_virtfn) + return false; + + /* + * Allow for a Device Security Manager (DSM) associated with function0 + * of an Endpoint to coordinate TDISP requests for other functions + * (physical or virtual) of the device, or allow for an Upstream Port + * DSM to accept TDISP requests for the Endpoints downstream of the + * switch. + */ + switch (pci_pcie_type(pdev)) { + case PCI_EXP_TYPE_ENDPOINT: + case PCI_EXP_TYPE_UPSTREAM: + case PCI_EXP_TYPE_RC_END: + if (pdev->ide_cap || (pdev->devcap & PCI_EXP_DEVCAP_TEE)) + break; + fallthrough; + default: + return false; + } + + return PCI_FUNC(pdev->devfn) == 0; +} + +/** + * enum pci_tsm_req_scope - Scope of guest requests to be validated by TSM + * + * Guest requests are a transport for a TVM to communicate with a TSM + DSM for + * a given TDI. A TSM driver is responsible for maintaining the kernel security + * model and limit commands that may affect the host, or are otherwise outside + * the typical TDISP operational model. + */ +enum pci_tsm_req_scope { + /** + * @PCI_TSM_REQ_INFO: Read-only, without side effects, request for + * typical TDISP collateral information like Device Interface Reports. + * No device secrets are permitted, and no device state is changed. + */ + PCI_TSM_REQ_INFO = 0, + /** + * @PCI_TSM_REQ_STATE_CHANGE: Request to change the TDISP state from + * UNLOCKED->LOCKED, LOCKED->RUN, or other architecture specific state + * changes to support those transitions for a TDI. No other (unrelated + * to TDISP) device / host state, configuration, or data change is + * permitted. + */ + PCI_TSM_REQ_STATE_CHANGE = 1, + /** + * @PCI_TSM_REQ_DEBUG_READ: Read-only request for debug information + * + * A method to facilitate TVM information retrieval outside of typical + * TDISP operational requirements. No device secrets are permitted. + */ + PCI_TSM_REQ_DEBUG_READ = 2, + /** + * @PCI_TSM_REQ_DEBUG_WRITE: Device state changes for debug purposes + * + * The request may affect the operational state of the device outside of + * the TDISP operational model. If allowed, requires CAP_SYS_RAW_IO, and + * will taint the kernel. + */ + PCI_TSM_REQ_DEBUG_WRITE = 3, +}; + +#ifdef CONFIG_PCI_TSM +int pci_tsm_register(struct tsm_dev *tsm_dev); +void pci_tsm_unregister(struct tsm_dev *tsm_dev); +int pci_tsm_link_constructor(struct pci_dev *pdev, struct pci_tsm *tsm, + struct tsm_dev *tsm_dev); +int pci_tsm_pf0_constructor(struct pci_dev *pdev, struct pci_tsm_pf0 *tsm, + struct tsm_dev *tsm_dev); +void pci_tsm_pf0_destructor(struct pci_tsm_pf0 *tsm); +int pci_tsm_doe_transfer(struct pci_dev *pdev, u8 type, const void *req, + size_t req_sz, void *resp, size_t resp_sz); +int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u32 tdi_id); +void pci_tsm_unbind(struct pci_dev *pdev); +void pci_tsm_tdi_constructor(struct pci_dev *pdev, struct pci_tdi *tdi, + struct kvm *kvm, u32 tdi_id); +ssize_t pci_tsm_guest_req(struct pci_dev *pdev, enum pci_tsm_req_scope scope, + sockptr_t req_in, size_t in_len, sockptr_t req_out, + size_t out_len, u64 *tsm_code); +#else +static inline int pci_tsm_register(struct tsm_dev *tsm_dev) +{ + return 0; +} +static inline void pci_tsm_unregister(struct tsm_dev *tsm_dev) +{ +} +static inline int pci_tsm_bind(struct pci_dev *pdev, struct kvm *kvm, u64 tdi_id) +{ + return -ENXIO; +} +static inline void pci_tsm_unbind(struct pci_dev *pdev) +{ +} +static inline ssize_t pci_tsm_guest_req(struct pci_dev *pdev, + enum pci_tsm_req_scope scope, + sockptr_t req_in, size_t in_len, + sockptr_t req_out, size_t out_len, + u64 *tsm_code) +{ + return -ENXIO; +} +#endif +#endif /*__PCI_TSM_H */ diff --git a/include/linux/pci.h b/include/linux/pci.h index b16127c6a7b4..864775651c6f 100644 --- a/include/linux/pci.h +++ b/include/linux/pci.h @@ -452,6 +452,7 @@ struct pci_dev { unsigned int pasid_enabled:1; /* Process Address Space ID */ unsigned int pri_enabled:1; /* Page Request Interface */ unsigned int tph_enabled:1; /* TLP Processing Hints */ + unsigned int fm_enabled:1; /* Flit Mode (segment captured) */ unsigned int is_managed:1; /* Managed via devres */ unsigned int is_msi_managed:1; /* MSI release via devres installed */ unsigned int needs_freset:1; /* Requires fundamental reset */ @@ -544,6 +545,18 @@ struct pci_dev { #ifdef CONFIG_PCI_NPEM struct npem *npem; /* Native PCIe Enclosure Management */ #endif +#ifdef CONFIG_PCI_IDE + u16 ide_cap; /* Link Integrity & Data Encryption */ + u8 nr_ide_mem; /* Address association resources for streams */ + u8 nr_link_ide; /* Link Stream count (Selective Stream offset) */ + u16 nr_sel_ide; /* Selective Stream count (register block allocator) */ + struct ida ide_stream_ida; + unsigned int ide_cfg:1; /* Config cycles over IDE */ + unsigned int ide_tee_limit:1; /* Disallow T=0 traffic over IDE */ +#endif +#ifdef CONFIG_PCI_TSM + struct pci_tsm *tsm; /* TSM operation state */ +#endif u16 acs_cap; /* ACS Capability offset */ u8 supported_speeds; /* Supported Link Speeds Vector */ phys_addr_t rom; /* Physical address if not from BAR */ @@ -579,6 +592,8 @@ struct pci_dev *pci_alloc_dev(struct pci_bus *bus); #define to_pci_dev(n) container_of(n, struct pci_dev, dev) #define for_each_pci_dev(d) while ((d = pci_get_device(PCI_ANY_ID, PCI_ANY_ID, d)) != NULL) +#define for_each_pci_dev_reverse(d) \ + while ((d = pci_get_device_reverse(PCI_ANY_ID, PCI_ANY_ID, d)) != NULL) static inline int pci_channel_offline(struct pci_dev *pdev) { @@ -605,6 +620,11 @@ struct pci_host_bridge { int domain_nr; struct list_head windows; /* resource_entry */ struct list_head dma_ranges; /* dma ranges resource list */ +#ifdef CONFIG_PCI_IDE + u16 nr_ide_streams; /* Max streams possibly active in @ide_stream_ida */ + struct ida ide_stream_ida; + struct ida ide_stream_ids_ida; /* track unique ids per domain */ +#endif u8 (*swizzle_irq)(struct pci_dev *, u8 *); /* Platform IRQ swizzler */ int (*map_irq)(const struct pci_dev *, u8, u8); void (*release_fn)(struct pci_host_bridge *); @@ -857,6 +877,11 @@ struct pci_bus_region { pci_bus_addr_t end; }; +static inline pci_bus_addr_t pci_bus_region_size(const struct pci_bus_region *region) +{ + return region->end - region->start + 1; +} + struct pci_dynids { spinlock_t lock; /* Protects list, index */ struct list_head list; /* For IDs added at runtime */ @@ -1241,6 +1266,8 @@ u64 pci_get_dsn(struct pci_dev *dev); struct pci_dev *pci_get_device(unsigned int vendor, unsigned int device, struct pci_dev *from); +struct pci_dev *pci_get_device_reverse(unsigned int vendor, unsigned int device, + struct pci_dev *from); struct pci_dev *pci_get_subsys(unsigned int vendor, unsigned int device, unsigned int ss_vendor, unsigned int ss_device, struct pci_dev *from); @@ -1660,6 +1687,8 @@ int pci_scan_bridge(struct pci_bus *bus, struct pci_dev *dev, int max, void pci_walk_bus(struct pci_bus *top, int (*cb)(struct pci_dev *, void *), void *userdata); +void pci_walk_bus_reverse(struct pci_bus *top, + int (*cb)(struct pci_dev *, void *), void *userdata); int pci_cfg_space_size(struct pci_dev *dev); unsigned char pci_bus_max_busnr(struct pci_bus *bus); resource_size_t pcibios_window_alignment(struct pci_bus *bus, @@ -2055,6 +2084,11 @@ static inline struct pci_dev *pci_get_device(unsigned int vendor, struct pci_dev *from) { return NULL; } +static inline struct pci_dev *pci_get_device_reverse(unsigned int vendor, + unsigned int device, + struct pci_dev *from) +{ return NULL; } + static inline struct pci_dev *pci_get_subsys(unsigned int vendor, unsigned int device, unsigned int ss_vendor, diff --git a/include/linux/psp-sev.h b/include/linux/psp-sev.h index abcdee256c65..69ffa4b4d1fa 100644 --- a/include/linux/psp-sev.h +++ b/include/linux/psp-sev.h @@ -142,6 +142,13 @@ enum sev_cmd { SEV_CMD_SNP_VLEK_LOAD = 0x0CD, SEV_CMD_SNP_FEATURE_INFO = 0x0CE, + /* SEV-TIO commands */ + SEV_CMD_TIO_STATUS = 0x0D0, + SEV_CMD_TIO_INIT = 0x0D1, + SEV_CMD_TIO_DEV_CREATE = 0x0D2, + SEV_CMD_TIO_DEV_RECLAIM = 0x0D3, + SEV_CMD_TIO_DEV_CONNECT = 0x0D4, + SEV_CMD_TIO_DEV_DISCONNECT = 0x0D5, SEV_CMD_MAX, }; @@ -783,7 +790,8 @@ struct sev_data_snp_init_ex { u32 list_paddr_en:1; u32 rapl_dis:1; u32 ciphertext_hiding_en:1; - u32 rsvd:28; + u32 tio_en:1; + u32 rsvd:27; u32 rsvd1; u64 list_paddr; u16 max_snp_asid; @@ -882,11 +890,15 @@ struct snp_feature_info { u32 edx; } __packed; +/* Feature bits in ECX */ #define SNP_RAPL_DISABLE_SUPPORTED BIT(2) #define SNP_CIPHER_TEXT_HIDING_SUPPORTED BIT(3) #define SNP_AES_256_XTS_POLICY_SUPPORTED BIT(4) #define SNP_CXL_ALLOW_POLICY_SUPPORTED BIT(5) +/* Feature bits in EBX */ +#define SNP_SEV_TIO_SUPPORTED BIT(1) + #ifdef CONFIG_CRYPTO_DEV_SP_PSP /** @@ -1028,6 +1040,7 @@ int sev_do_cmd(int cmd, void *data, int *psp_ret); void *psp_copy_user_blob(u64 uaddr, u32 len); void *snp_alloc_firmware_page(gfp_t mask); +int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked); void snp_free_firmware_page(void *addr); void sev_platform_shutdown(void); bool sev_is_snp_ciphertext_hiding_supported(void); @@ -1064,6 +1077,11 @@ static inline void *snp_alloc_firmware_page(gfp_t mask) return NULL; } +static inline int snp_reclaim_pages(unsigned long paddr, unsigned int npages, bool locked) +{ + return -ENODEV; +} + static inline void snp_free_firmware_page(void *addr) { } static inline void sev_platform_shutdown(void) { } diff --git a/include/linux/rbtree.h b/include/linux/rbtree.h index 8d2ba3749866..4091e978aef2 100644 --- a/include/linux/rbtree.h +++ b/include/linux/rbtree.h @@ -43,8 +43,36 @@ extern void rb_erase(struct rb_node *, struct rb_root *); /* Find logical next and previous nodes in a tree */ extern struct rb_node *rb_next(const struct rb_node *); extern struct rb_node *rb_prev(const struct rb_node *); -extern struct rb_node *rb_first(const struct rb_root *); -extern struct rb_node *rb_last(const struct rb_root *); + +/* + * This function returns the first node (in sort order) of the tree. + */ +static inline struct rb_node *rb_first(const struct rb_root *root) +{ + struct rb_node *n; + + n = root->rb_node; + if (!n) + return NULL; + while (n->rb_left) + n = n->rb_left; + return n; +} + +/* + * This function returns the last node (in sort order) of the tree. + */ +static inline struct rb_node *rb_last(const struct rb_root *root) +{ + struct rb_node *n; + + n = root->rb_node; + if (!n) + return NULL; + while (n->rb_right) + n = n->rb_right; + return n; +} /* Postorder iteration - always visit the parent after its children */ extern struct rb_node *rb_first_postorder(const struct rb_root *); diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h index a8a8661839b6..221123660e71 100644 --- a/include/linux/seqlock.h +++ b/include/linux/seqlock.h @@ -1224,7 +1224,7 @@ struct ss_tmp { spinlock_t *lock_irqsave; }; -static inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) +static __always_inline void __scoped_seqlock_cleanup(struct ss_tmp *sst) { if (sst->lock) spin_unlock(sst->lock); @@ -1252,7 +1252,7 @@ static inline void __scoped_seqlock_bug(void) { } extern void __scoped_seqlock_bug(void); #endif -static inline void +static __always_inline void __scoped_seqlock_next(struct ss_tmp *sst, seqlock_t *lock, enum ss_state target) { switch (sst->state) { diff --git a/include/linux/shmem_fs.h b/include/linux/shmem_fs.h index 5e4b3c1ae5c2..e2069b3179c4 100644 --- a/include/linux/shmem_fs.h +++ b/include/linux/shmem_fs.h @@ -10,6 +10,7 @@ #include <linux/xattr.h> #include <linux/fs_parser.h> #include <linux/userfaultfd_k.h> +#include <linux/bits.h> struct swap_iocb; @@ -19,6 +20,19 @@ struct swap_iocb; #define SHMEM_MAXQUOTAS 2 #endif +/* Suppress pre-accounting of the entire object size. */ +#define SHMEM_F_NORESERVE BIT(0) +/* Disallow swapping. */ +#define SHMEM_F_LOCKED BIT(1) +/* + * Disallow growing, shrinking, or hole punching in the inode. Combined with + * folio pinning, makes sure the inode's mapping stays fixed. + * + * In some ways similar to F_SEAL_GROW | F_SEAL_SHRINK, but can be removed and + * isn't directly visible to userspace. + */ +#define SHMEM_F_MAPPING_FROZEN BIT(2) + struct shmem_inode_info { spinlock_t lock; unsigned int seals; /* shmem seals */ @@ -186,6 +200,15 @@ static inline bool shmem_file(struct file *file) return shmem_mapping(file->f_mapping); } +/* Must be called with inode lock taken exclusive. */ +static inline void shmem_freeze(struct inode *inode, bool freeze) +{ + if (freeze) + SHMEM_I(inode)->flags |= SHMEM_F_MAPPING_FROZEN; + else + SHMEM_I(inode)->flags &= ~SHMEM_F_MAPPING_FROZEN; +} + /* * If fallocate(FALLOC_FL_KEEP_SIZE) has been used, there may be pages * beyond i_size's notion of EOF, which fallocate has committed to reserving: diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h index 22704c2e5b9b..57f4fd94166a 100644 --- a/include/linux/sunrpc/svc_rdma.h +++ b/include/linux/sunrpc/svc_rdma.h @@ -131,7 +131,7 @@ static inline struct svcxprt_rdma *svc_rdma_rqst_rdma(struct svc_rqst *rqstp) */ enum { RPCRDMA_LISTEN_BACKLOG = 10, - RPCRDMA_MAX_REQUESTS = 64, + RPCRDMA_MAX_REQUESTS = 128, RPCRDMA_MAX_BC_REQUESTS = 2, }; diff --git a/include/linux/sunrpc/svcsock.h b/include/linux/sunrpc/svcsock.h index 963bbe251e52..de37069aba90 100644 --- a/include/linux/sunrpc/svcsock.h +++ b/include/linux/sunrpc/svcsock.h @@ -26,6 +26,9 @@ struct svc_sock { void (*sk_odata)(struct sock *); void (*sk_owspace)(struct sock *); + /* For sends (protected by xpt_mutex) */ + struct bio_vec *sk_bvec; + /* private TCP part */ /* On-the-wire fragment header: */ __be32 sk_marker; diff --git a/include/linux/sys_info.h b/include/linux/sys_info.h index 89d77dc4f2ed..a5bc3ea3d44b 100644 --- a/include/linux/sys_info.h +++ b/include/linux/sys_info.h @@ -14,7 +14,7 @@ #define SYS_INFO_LOCKS 0x00000008 #define SYS_INFO_FTRACE 0x00000010 #define SYS_INFO_PANIC_CONSOLE_REPLAY 0x00000020 -#define SYS_INFO_ALL_CPU_BT 0x00000040 +#define SYS_INFO_ALL_BT 0x00000040 #define SYS_INFO_BLOCKED_TASKS 0x00000080 void sys_info(unsigned long si_mask); diff --git a/include/linux/trace_events.h b/include/linux/trace_events.h index 04307a19cde3..3690221ba3d8 100644 --- a/include/linux/trace_events.h +++ b/include/linux/trace_events.h @@ -138,6 +138,7 @@ enum trace_iter_flags { TRACE_FILE_LAT_FMT = 1, TRACE_FILE_ANNOTATE = 2, TRACE_FILE_TIME_IN_NS = 4, + TRACE_FILE_PAUSE = 8, }; diff --git a/include/linux/tsm.h b/include/linux/tsm.h index 431054810dca..a3b7ab668eff 100644 --- a/include/linux/tsm.h +++ b/include/linux/tsm.h @@ -5,6 +5,7 @@ #include <linux/sizes.h> #include <linux/types.h> #include <linux/uuid.h> +#include <linux/device.h> #define TSM_REPORT_INBLOB_MAX 64 #define TSM_REPORT_OUTBLOB_MAX SZ_32K @@ -107,6 +108,22 @@ struct tsm_report_ops { bool (*report_bin_attr_visible)(int n); }; +struct pci_tsm_ops; +struct tsm_dev { + struct device dev; + int id; + const struct pci_tsm_ops *pci_ops; +}; + +DEFINE_FREE(put_tsm_dev, struct tsm_dev *, + if (!IS_ERR_OR_NULL(_T)) put_device(&_T->dev)) + int tsm_report_register(const struct tsm_report_ops *ops, void *priv); int tsm_report_unregister(const struct tsm_report_ops *ops); +struct tsm_dev *tsm_register(struct device *parent, struct pci_tsm_ops *ops); +void tsm_unregister(struct tsm_dev *tsm_dev); +struct tsm_dev *find_tsm_dev(int id); +struct pci_ide; +int tsm_ide_stream_register(struct pci_ide *ide); +void tsm_ide_stream_unregister(struct pci_ide *ide); #endif /* __TSM_H */ diff --git a/include/linux/uaccess.h b/include/linux/uaccess.h index be395f5f7ee3..1f3804245c06 100644 --- a/include/linux/uaccess.h +++ b/include/linux/uaccess.h @@ -161,8 +161,6 @@ __copy_to_user(void __user *to, const void *from, unsigned long n) * directly in the normal copy_to/from_user(), the other ones go * through an extern _copy_to/from_user(), which expands the same code * here. - * - * Rust code always uses the extern definition. */ static inline __must_check unsigned long _inline_copy_from_user(void *to, const void __user *from, unsigned long n) @@ -192,8 +190,10 @@ fail: memset(to + (n - res), 0, res); return res; } +#ifndef INLINE_COPY_FROM_USER extern __must_check unsigned long _copy_from_user(void *, const void __user *, unsigned long); +#endif static inline __must_check unsigned long _inline_copy_to_user(void __user *to, const void *from, unsigned long n) @@ -207,8 +207,10 @@ _inline_copy_to_user(void __user *to, const void *from, unsigned long n) } return n; } +#ifndef INLINE_COPY_TO_USER extern __must_check unsigned long _copy_to_user(void __user *, const void *, unsigned long); +#endif static __always_inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long n) diff --git a/include/linux/util_macros.h b/include/linux/util_macros.h index 9373962aade9..2eb528058d0d 100644 --- a/include/linux/util_macros.h +++ b/include/linux/util_macros.h @@ -136,10 +136,10 @@ #define PTR_IF(cond, ptr) ((cond) ? (ptr) : NULL) /** - * to_user_ptr - cast a pointer passed as u64 from user space to void __user * + * u64_to_user_ptr - cast a pointer passed as u64 from user space to void __user * * @x: The u64 value from user space, usually via IOCTL * - * to_user_ptr() simply casts a pointer passed as u64 from user space to void + * u64_to_user_ptr() simply casts a pointer passed as u64 from user space to void * __user * correctly. Using this lets us get rid of all the tiresome casts. */ #define u64_to_user_ptr(x) \ diff --git a/include/linux/vmcore_info.h b/include/linux/vmcore_info.h index 37e003ae5262..e71518caacdf 100644 --- a/include/linux/vmcore_info.h +++ b/include/linux/vmcore_info.h @@ -5,6 +5,7 @@ #include <linux/linkage.h> #include <linux/elfcore.h> #include <linux/elf.h> +#include <uapi/linux/vmcore.h> #define CRASH_CORE_NOTE_HEAD_BYTES ALIGN(sizeof(struct elf_note), 4) #define CRASH_CORE_NOTE_NAME_BYTES ALIGN(sizeof(NN_PRSTATUS), 4) @@ -77,4 +78,11 @@ extern u32 *vmcoreinfo_note; Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type, void *data, size_t data_len); void final_note(Elf_Word *buf); + +#ifdef CONFIG_VMCORE_INFO +void hwerr_log_error_type(enum hwerr_error_type src); +#else +static inline void hwerr_log_error_type(enum hwerr_error_type src) {}; +#endif + #endif /* LINUX_VMCORE_INFO_H */ diff --git a/include/linux/xxhash.h b/include/linux/xxhash.h index 27f57eca8cb1..587122e2c29c 100644 --- a/include/linux/xxhash.h +++ b/include/linux/xxhash.h @@ -141,21 +141,7 @@ static inline unsigned long xxhash(const void *input, size_t length, */ /** - * struct xxh32_state - private xxh32 state, do not use members directly - */ -struct xxh32_state { - uint32_t total_len_32; - uint32_t large_len; - uint32_t v1; - uint32_t v2; - uint32_t v3; - uint32_t v4; - uint32_t mem32[4]; - uint32_t memsize; -}; - -/** - * struct xxh32_state - private xxh64 state, do not use members directly + * struct xxh64_state - private xxh64 state, do not use members directly */ struct xxh64_state { uint64_t total_len; @@ -168,16 +154,6 @@ struct xxh64_state { }; /** - * xxh32_reset() - reset the xxh32 state to start a new hashing operation - * - * @state: The xxh32 state to reset. - * @seed: Initialize the hash state with this seed. - * - * Call this function on any xxh32_state to prepare for a new hashing operation. - */ -void xxh32_reset(struct xxh32_state *state, uint32_t seed); - -/** * xxh64_reset() - reset the xxh64 state to start a new hashing operation * * @state: The xxh64 state to reset. @@ -210,24 +186,4 @@ int xxh64_update(struct xxh64_state *state, const void *input, size_t length); */ uint64_t xxh64_digest(const struct xxh64_state *state); -/*-************************** - * Utils - ***************************/ - -/** - * xxh32_copy_state() - copy the source state into the destination state - * - * @src: The source xxh32 state. - * @dst: The destination xxh32 state. - */ -void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src); - -/** - * xxh64_copy_state() - copy the source state into the destination state - * - * @src: The source xxh64 state. - * @dst: The destination xxh64 state. - */ -void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src); - #endif /* XXHASH_H */ diff --git a/include/uapi/linux/liveupdate.h b/include/uapi/linux/liveupdate.h new file mode 100644 index 000000000000..30bc66ee9436 --- /dev/null +++ b/include/uapi/linux/liveupdate.h @@ -0,0 +1,216 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ + +/* + * Userspace interface for /dev/liveupdate + * Live Update Orchestrator + * + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +#ifndef _UAPI_LIVEUPDATE_H +#define _UAPI_LIVEUPDATE_H + +#include <linux/ioctl.h> +#include <linux/types.h> + +/** + * DOC: General ioctl format + * + * The ioctl interface follows a general format to allow for extensibility. Each + * ioctl is passed in a structure pointer as the argument providing the size of + * the structure in the first u32. The kernel checks that any structure space + * beyond what it understands is 0. This allows userspace to use the backward + * compatible portion while consistently using the newer, larger, structures. + * + * ioctls use a standard meaning for common errnos: + * + * - ENOTTY: The IOCTL number itself is not supported at all + * - E2BIG: The IOCTL number is supported, but the provided structure has + * non-zero in a part the kernel does not understand. + * - EOPNOTSUPP: The IOCTL number is supported, and the structure is + * understood, however a known field has a value the kernel does not + * understand or support. + * - EINVAL: Everything about the IOCTL was understood, but a field is not + * correct. + * - ENOENT: A provided token does not exist. + * - ENOMEM: Out of memory. + * - EOVERFLOW: Mathematics overflowed. + * + * As well as additional errnos, within specific ioctls. + */ + +/* The ioctl type, documented in ioctl-number.rst */ +#define LIVEUPDATE_IOCTL_TYPE 0xBA + +/* The maximum length of session name including null termination */ +#define LIVEUPDATE_SESSION_NAME_LENGTH 64 + +/* The /dev/liveupdate ioctl commands */ +enum { + LIVEUPDATE_CMD_BASE = 0x00, + LIVEUPDATE_CMD_CREATE_SESSION = LIVEUPDATE_CMD_BASE, + LIVEUPDATE_CMD_RETRIEVE_SESSION = 0x01, +}; + +/* ioctl commands for session file descriptors */ +enum { + LIVEUPDATE_CMD_SESSION_BASE = 0x40, + LIVEUPDATE_CMD_SESSION_PRESERVE_FD = LIVEUPDATE_CMD_SESSION_BASE, + LIVEUPDATE_CMD_SESSION_RETRIEVE_FD = 0x41, + LIVEUPDATE_CMD_SESSION_FINISH = 0x42, +}; + +/** + * struct liveupdate_ioctl_create_session - ioctl(LIVEUPDATE_IOCTL_CREATE_SESSION) + * @size: Input; sizeof(struct liveupdate_ioctl_create_session) + * @fd: Output; The new file descriptor for the created session. + * @name: Input; A null-terminated string for the session name, max + * length %LIVEUPDATE_SESSION_NAME_LENGTH including termination + * character. + * + * Creates a new live update session for managing preserved resources. + * This ioctl can only be called on the main /dev/liveupdate device. + * + * Return: 0 on success, negative error code on failure. + */ +struct liveupdate_ioctl_create_session { + __u32 size; + __s32 fd; + __u8 name[LIVEUPDATE_SESSION_NAME_LENGTH]; +}; + +#define LIVEUPDATE_IOCTL_CREATE_SESSION \ + _IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_CREATE_SESSION) + +/** + * struct liveupdate_ioctl_retrieve_session - ioctl(LIVEUPDATE_IOCTL_RETRIEVE_SESSION) + * @size: Input; sizeof(struct liveupdate_ioctl_retrieve_session) + * @fd: Output; The new file descriptor for the retrieved session. + * @name: Input; A null-terminated string identifying the session to retrieve. + * The name must exactly match the name used when the session was + * created in the previous kernel. + * + * Retrieves a handle (a new file descriptor) for a preserved session by its + * name. This is the primary mechanism for a userspace agent to regain control + * of its preserved resources after a live update. + * + * The userspace application provides the null-terminated `name` of a session + * it created before the live update. If a preserved session with a matching + * name is found, the kernel instantiates it and returns a new file descriptor + * in the `fd` field. This new session FD can then be used for all file-specific + * operations, such as restoring individual file descriptors with + * LIVEUPDATE_SESSION_RETRIEVE_FD. + * + * It is the responsibility of the userspace application to know the names of + * the sessions it needs to retrieve. If no session with the given name is + * found, the ioctl will fail with -ENOENT. + * + * This ioctl can only be called on the main /dev/liveupdate device when the + * system is in the LIVEUPDATE_STATE_UPDATED state. + */ +struct liveupdate_ioctl_retrieve_session { + __u32 size; + __s32 fd; + __u8 name[LIVEUPDATE_SESSION_NAME_LENGTH]; +}; + +#define LIVEUPDATE_IOCTL_RETRIEVE_SESSION \ + _IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_RETRIEVE_SESSION) + +/* Session specific IOCTLs */ + +/** + * struct liveupdate_session_preserve_fd - ioctl(LIVEUPDATE_SESSION_PRESERVE_FD) + * @size: Input; sizeof(struct liveupdate_session_preserve_fd) + * @fd: Input; The user-space file descriptor to be preserved. + * @token: Input; An opaque, unique token for preserved resource. + * + * Holds parameters for preserving a file descriptor. + * + * User sets the @fd field identifying the file descriptor to preserve + * (e.g., memfd, kvm, iommufd, VFIO). The kernel validates if this FD type + * and its dependencies are supported for preservation. If validation passes, + * the kernel marks the FD internally and *initiates the process* of preparing + * its state for saving. The actual snapshotting of the state typically occurs + * during the subsequent %LIVEUPDATE_IOCTL_PREPARE execution phase, though + * some finalization might occur during freeze. + * On successful validation and initiation, the kernel uses the @token + * field with an opaque identifier representing the resource being preserved. + * This token confirms the FD is targeted for preservation and is required for + * the subsequent %LIVEUPDATE_SESSION_RETRIEVE_FD call after the live update. + * + * Return: 0 on success (validation passed, preservation initiated), negative + * error code on failure (e.g., unsupported FD type, dependency issue, + * validation failed). + */ +struct liveupdate_session_preserve_fd { + __u32 size; + __s32 fd; + __aligned_u64 token; +}; + +#define LIVEUPDATE_SESSION_PRESERVE_FD \ + _IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_SESSION_PRESERVE_FD) + +/** + * struct liveupdate_session_retrieve_fd - ioctl(LIVEUPDATE_SESSION_RETRIEVE_FD) + * @size: Input; sizeof(struct liveupdate_session_retrieve_fd) + * @fd: Output; The new file descriptor representing the fully restored + * kernel resource. + * @token: Input; An opaque, token that was used to preserve the resource. + * + * Retrieve a previously preserved file descriptor. + * + * User sets the @token field to the value obtained from a successful + * %LIVEUPDATE_IOCTL_FD_PRESERVE call before the live update. On success, + * the kernel restores the state (saved during the PREPARE/FREEZE phases) + * associated with the token and populates the @fd field with a new file + * descriptor referencing the restored resource in the current (new) kernel. + * This operation must be performed *before* signaling completion via + * %LIVEUPDATE_IOCTL_FINISH. + * + * Return: 0 on success, negative error code on failure (e.g., invalid token). + */ +struct liveupdate_session_retrieve_fd { + __u32 size; + __s32 fd; + __aligned_u64 token; +}; + +#define LIVEUPDATE_SESSION_RETRIEVE_FD \ + _IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_SESSION_RETRIEVE_FD) + +/** + * struct liveupdate_session_finish - ioctl(LIVEUPDATE_SESSION_FINISH) + * @size: Input; sizeof(struct liveupdate_session_finish) + * @reserved: Input; Must be zero. Reserved for future use. + * + * Signals the completion of the restoration process for a retrieved session. + * This is the final operation that should be performed on a session file + * descriptor after a live update. + * + * This ioctl must be called once all required file descriptors for the session + * have been successfully retrieved (using %LIVEUPDATE_SESSION_RETRIEVE_FD) and + * are fully restored from the userspace and kernel perspective. + * + * Upon success, the kernel releases its ownership of the preserved resources + * associated with this session. This allows internal resources to be freed, + * typically by decrementing reference counts on the underlying preserved + * objects. + * + * If this operation fails, the resources remain preserved in memory. Userspace + * may attempt to call finish again. The resources will otherwise be reset + * during the next live update cycle. + * + * Return: 0 on success, negative error code on failure. + */ +struct liveupdate_session_finish { + __u32 size; + __u32 reserved; +}; + +#define LIVEUPDATE_SESSION_FINISH \ + _IO(LIVEUPDATE_IOCTL_TYPE, LIVEUPDATE_CMD_SESSION_FINISH) + +#endif /* _UAPI_LIVEUPDATE_H */ diff --git a/include/linux/map_benchmark.h b/include/uapi/linux/map_benchmark.h index 48e2ff95332f..c2d91088a40d 100644 --- a/include/linux/map_benchmark.h +++ b/include/uapi/linux/map_benchmark.h @@ -1,10 +1,12 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ +/* SPDX-License-Identifier: GPL-2.0-only WITH Linux-syscall-note */ /* - * Copyright (C) 2022 HiSilicon Limited. + * Copyright (C) 2022-2025 HiSilicon Limited. */ -#ifndef _KERNEL_DMA_BENCHMARK_H -#define _KERNEL_DMA_BENCHMARK_H +#ifndef _UAPI_DMA_BENCHMARK_H +#define _UAPI_DMA_BENCHMARK_H + +#include <linux/types.h> #define DMA_MAP_BENCHMARK _IOWR('d', 1, struct map_benchmark) #define DMA_MAP_MAX_THREADS 1024 @@ -29,4 +31,5 @@ struct map_benchmark { __u32 granule; /* how many PAGE_SIZE will do map/unmap once a time */ __u8 expansion[76]; /* For future use */ }; -#endif /* _KERNEL_DMA_BENCHMARK_H */ + +#endif /* _UAPI_DMA_BENCHMARK_H */ diff --git a/include/uapi/linux/pci_regs.h b/include/uapi/linux/pci_regs.h index 07e06aafec50..3add74ae2594 100644 --- a/include/uapi/linux/pci_regs.h +++ b/include/uapi/linux/pci_regs.h @@ -503,6 +503,7 @@ #define PCI_EXP_DEVCAP_PWR_VAL 0x03fc0000 /* Slot Power Limit Value */ #define PCI_EXP_DEVCAP_PWR_SCL 0x0c000000 /* Slot Power Limit Scale */ #define PCI_EXP_DEVCAP_FLR 0x10000000 /* Function Level Reset */ +#define PCI_EXP_DEVCAP_TEE 0x40000000 /* TEE I/O (TDISP) Support */ #define PCI_EXP_DEVCTL 0x08 /* Device Control */ #define PCI_EXP_DEVCTL_CERE 0x0001 /* Correctable Error Reporting En. */ #define PCI_EXP_DEVCTL_NFERE 0x0002 /* Non-Fatal Error Reporting Enable */ @@ -754,6 +755,8 @@ #define PCI_EXT_CAP_ID_NPEM 0x29 /* Native PCIe Enclosure Management */ #define PCI_EXT_CAP_ID_PL_32GT 0x2A /* Physical Layer 32.0 GT/s */ #define PCI_EXT_CAP_ID_DOE 0x2E /* Data Object Exchange */ +#define PCI_EXT_CAP_ID_DEV3 0x2F /* Device 3 Capability/Control/Status */ +#define PCI_EXT_CAP_ID_IDE 0x30 /* Integrity and Data Encryption */ #define PCI_EXT_CAP_ID_PL_64GT 0x31 /* Physical Layer 64.0 GT/s */ #define PCI_EXT_CAP_ID_MAX PCI_EXT_CAP_ID_PL_64GT @@ -1244,9 +1247,95 @@ /* Deprecated old name, replaced with PCI_DOE_DATA_OBJECT_DISC_RSP_3_TYPE */ #define PCI_DOE_DATA_OBJECT_DISC_RSP_3_PROTOCOL PCI_DOE_DATA_OBJECT_DISC_RSP_3_TYPE +/* Device 3 Extended Capability */ +#define PCI_DEV3_CAP 0x04 /* Device 3 Capabilities Register */ +#define PCI_DEV3_CTL 0x08 /* Device 3 Control Register */ +#define PCI_DEV3_STA 0x0c /* Device 3 Status Register */ +#define PCI_DEV3_STA_SEGMENT 0x8 /* Segment Captured (end-to-end flit-mode detected) */ + /* Compute Express Link (CXL r3.1, sec 8.1.5) */ #define PCI_DVSEC_CXL_PORT 3 #define PCI_DVSEC_CXL_PORT_CTL 0x0c #define PCI_DVSEC_CXL_PORT_CTL_UNMASK_SBR 0x00000001 +/* Integrity and Data Encryption Extended Capability */ +#define PCI_IDE_CAP 0x04 +#define PCI_IDE_CAP_LINK 0x1 /* Link IDE Stream Supported */ +#define PCI_IDE_CAP_SELECTIVE 0x2 /* Selective IDE Streams Supported */ +#define PCI_IDE_CAP_FLOWTHROUGH 0x4 /* Flow-Through IDE Stream Supported */ +#define PCI_IDE_CAP_PARTIAL_HEADER_ENC 0x8 /* Partial Header Encryption Supported */ +#define PCI_IDE_CAP_AGGREGATION 0x10 /* Aggregation Supported */ +#define PCI_IDE_CAP_PCRC 0x20 /* PCRC Supported */ +#define PCI_IDE_CAP_IDE_KM 0x40 /* IDE_KM Protocol Supported */ +#define PCI_IDE_CAP_SEL_CFG 0x80 /* Selective IDE for Config Request Support */ +#define PCI_IDE_CAP_ALG __GENMASK(12, 8) /* Supported Algorithms */ +#define PCI_IDE_CAP_ALG_AES_GCM_256 0 /* AES-GCM 256 key size, 96b MAC */ +#define PCI_IDE_CAP_LINK_TC_NUM __GENMASK(15, 13) /* Link IDE TCs */ +#define PCI_IDE_CAP_SEL_NUM __GENMASK(23, 16) /* Supported Selective IDE Streams */ +#define PCI_IDE_CAP_TEE_LIMITED 0x1000000 /* TEE-Limited Stream Supported */ +#define PCI_IDE_CTL 0x08 +#define PCI_IDE_CTL_FLOWTHROUGH_IDE 0x4 /* Flow-Through IDE Stream Enabled */ + +#define PCI_IDE_LINK_STREAM_0 0xc /* First Link Stream Register Block */ +#define PCI_IDE_LINK_BLOCK_SIZE 8 +/* Link IDE Stream block, up to PCI_IDE_CAP_LINK_TC_NUM */ +#define PCI_IDE_LINK_CTL_0 0x00 /* First Link Control Register Offset in block */ +#define PCI_IDE_LINK_CTL_EN 0x1 /* Link IDE Stream Enable */ +#define PCI_IDE_LINK_CTL_TX_AGGR_NPR __GENMASK(3, 2) /* Tx Aggregation Mode NPR */ +#define PCI_IDE_LINK_CTL_TX_AGGR_PR __GENMASK(5, 4) /* Tx Aggregation Mode PR */ +#define PCI_IDE_LINK_CTL_TX_AGGR_CPL __GENMASK(7, 6) /* Tx Aggregation Mode CPL */ +#define PCI_IDE_LINK_CTL_PCRC_EN 0x100 /* PCRC Enable */ +#define PCI_IDE_LINK_CTL_PART_ENC __GENMASK(13, 10) /* Partial Header Encryption Mode */ +#define PCI_IDE_LINK_CTL_ALG __GENMASK(18, 14) /* Selection from PCI_IDE_CAP_ALG */ +#define PCI_IDE_LINK_CTL_TC __GENMASK(21, 19) /* Traffic Class */ +#define PCI_IDE_LINK_CTL_ID __GENMASK(31, 24) /* Stream ID */ +#define PCI_IDE_LINK_STS_0 0x4 /* First Link Status Register Offset in block */ +#define PCI_IDE_LINK_STS_STATE __GENMASK(3, 0) /* Link IDE Stream State */ +#define PCI_IDE_LINK_STS_IDE_FAIL 0x80000000 /* IDE fail message received */ + +/* Selective IDE Stream block, up to PCI_IDE_CAP_SELECTIVE_STREAMS_NUM */ +/* Selective IDE Stream Capability Register */ +#define PCI_IDE_SEL_CAP 0x00 +#define PCI_IDE_SEL_CAP_ASSOC_NUM __GENMASK(3, 0) +/* Selective IDE Stream Control Register */ +#define PCI_IDE_SEL_CTL 0x04 +#define PCI_IDE_SEL_CTL_EN 0x1 /* Selective IDE Stream Enable */ +#define PCI_IDE_SEL_CTL_TX_AGGR_NPR __GENMASK(3, 2) /* Tx Aggregation Mode NPR */ +#define PCI_IDE_SEL_CTL_TX_AGGR_PR __GENMASK(5, 4) /* Tx Aggregation Mode PR */ +#define PCI_IDE_SEL_CTL_TX_AGGR_CPL __GENMASK(7, 6) /* Tx Aggregation Mode CPL */ +#define PCI_IDE_SEL_CTL_PCRC_EN 0x100 /* PCRC Enable */ +#define PCI_IDE_SEL_CTL_CFG_EN 0x200 /* Selective IDE for Configuration Requests */ +#define PCI_IDE_SEL_CTL_PART_ENC __GENMASK(13, 10) /* Partial Header Encryption Mode */ +#define PCI_IDE_SEL_CTL_ALG __GENMASK(18, 14) /* Selection from PCI_IDE_CAP_ALG */ +#define PCI_IDE_SEL_CTL_TC __GENMASK(21, 19) /* Traffic Class */ +#define PCI_IDE_SEL_CTL_DEFAULT 0x400000 /* Default Stream */ +#define PCI_IDE_SEL_CTL_TEE_LIMITED 0x800000 /* TEE-Limited Stream */ +#define PCI_IDE_SEL_CTL_ID __GENMASK(31, 24) /* Stream ID */ +#define PCI_IDE_SEL_CTL_ID_MAX 255 +/* Selective IDE Stream Status Register */ +#define PCI_IDE_SEL_STS 0x08 +#define PCI_IDE_SEL_STS_STATE __GENMASK(3, 0) /* Selective IDE Stream State */ +#define PCI_IDE_SEL_STS_STATE_INSECURE 0 +#define PCI_IDE_SEL_STS_STATE_SECURE 2 +#define PCI_IDE_SEL_STS_IDE_FAIL 0x80000000 /* IDE fail message received */ +/* IDE RID Association Register 1 */ +#define PCI_IDE_SEL_RID_1 0x0c +#define PCI_IDE_SEL_RID_1_LIMIT __GENMASK(23, 8) +/* IDE RID Association Register 2 */ +#define PCI_IDE_SEL_RID_2 0x10 +#define PCI_IDE_SEL_RID_2_VALID 0x1 +#define PCI_IDE_SEL_RID_2_BASE __GENMASK(23, 8) +#define PCI_IDE_SEL_RID_2_SEG __GENMASK(31, 24) +/* Selective IDE Address Association Register Block, up to PCI_IDE_SEL_CAP_ASSOC_NUM */ +#define PCI_IDE_SEL_ADDR_BLOCK_SIZE 12 +#define PCI_IDE_SEL_ADDR_1(x) (20 + (x) * PCI_IDE_SEL_ADDR_BLOCK_SIZE) +#define PCI_IDE_SEL_ADDR_1_VALID 0x1 +#define PCI_IDE_SEL_ADDR_1_BASE_LOW __GENMASK(19, 8) +#define PCI_IDE_SEL_ADDR_1_LIMIT_LOW __GENMASK(31, 20) +/* IDE Address Association Register 2 is "Memory Limit Upper" */ +#define PCI_IDE_SEL_ADDR_2(x) (24 + (x) * PCI_IDE_SEL_ADDR_BLOCK_SIZE) +/* IDE Address Association Register 3 is "Memory Base Upper" */ +#define PCI_IDE_SEL_ADDR_3(x) (28 + (x) * PCI_IDE_SEL_ADDR_BLOCK_SIZE) +#define PCI_IDE_SEL_BLOCK_SIZE(nr_assoc) (20 + PCI_IDE_SEL_ADDR_BLOCK_SIZE * (nr_assoc)) + #endif /* LINUX_PCI_REGS_H */ diff --git a/include/uapi/linux/psp-sev.h b/include/uapi/linux/psp-sev.h index c2fd324623c4..2b5b042eb73b 100644 --- a/include/uapi/linux/psp-sev.h +++ b/include/uapi/linux/psp-sev.h @@ -47,32 +47,32 @@ typedef enum { * with possible values from the specification. */ SEV_RET_NO_FW_CALL = -1, - SEV_RET_SUCCESS = 0, - SEV_RET_INVALID_PLATFORM_STATE, - SEV_RET_INVALID_GUEST_STATE, - SEV_RET_INAVLID_CONFIG, + SEV_RET_SUCCESS = 0, + SEV_RET_INVALID_PLATFORM_STATE = 0x0001, + SEV_RET_INVALID_GUEST_STATE = 0x0002, + SEV_RET_INAVLID_CONFIG = 0x0003, SEV_RET_INVALID_CONFIG = SEV_RET_INAVLID_CONFIG, - SEV_RET_INVALID_LEN, - SEV_RET_ALREADY_OWNED, - SEV_RET_INVALID_CERTIFICATE, - SEV_RET_POLICY_FAILURE, - SEV_RET_INACTIVE, - SEV_RET_INVALID_ADDRESS, - SEV_RET_BAD_SIGNATURE, - SEV_RET_BAD_MEASUREMENT, - SEV_RET_ASID_OWNED, - SEV_RET_INVALID_ASID, - SEV_RET_WBINVD_REQUIRED, - SEV_RET_DFFLUSH_REQUIRED, - SEV_RET_INVALID_GUEST, - SEV_RET_INVALID_COMMAND, - SEV_RET_ACTIVE, - SEV_RET_HWSEV_RET_PLATFORM, - SEV_RET_HWSEV_RET_UNSAFE, - SEV_RET_UNSUPPORTED, - SEV_RET_INVALID_PARAM, - SEV_RET_RESOURCE_LIMIT, - SEV_RET_SECURE_DATA_INVALID, + SEV_RET_INVALID_LEN = 0x0004, + SEV_RET_ALREADY_OWNED = 0x0005, + SEV_RET_INVALID_CERTIFICATE = 0x0006, + SEV_RET_POLICY_FAILURE = 0x0007, + SEV_RET_INACTIVE = 0x0008, + SEV_RET_INVALID_ADDRESS = 0x0009, + SEV_RET_BAD_SIGNATURE = 0x000A, + SEV_RET_BAD_MEASUREMENT = 0x000B, + SEV_RET_ASID_OWNED = 0x000C, + SEV_RET_INVALID_ASID = 0x000D, + SEV_RET_WBINVD_REQUIRED = 0x000E, + SEV_RET_DFFLUSH_REQUIRED = 0x000F, + SEV_RET_INVALID_GUEST = 0x0010, + SEV_RET_INVALID_COMMAND = 0x0011, + SEV_RET_ACTIVE = 0x0012, + SEV_RET_HWSEV_RET_PLATFORM = 0x0013, + SEV_RET_HWSEV_RET_UNSAFE = 0x0014, + SEV_RET_UNSUPPORTED = 0x0015, + SEV_RET_INVALID_PARAM = 0x0016, + SEV_RET_RESOURCE_LIMIT = 0x0017, + SEV_RET_SECURE_DATA_INVALID = 0x0018, SEV_RET_INVALID_PAGE_SIZE = 0x0019, SEV_RET_INVALID_PAGE_STATE = 0x001A, SEV_RET_INVALID_MDATA_ENTRY = 0x001B, @@ -87,6 +87,22 @@ typedef enum { SEV_RET_RESTORE_REQUIRED = 0x0025, SEV_RET_RMP_INITIALIZATION_FAILED = 0x0026, SEV_RET_INVALID_KEY = 0x0027, + SEV_RET_SHUTDOWN_INCOMPLETE = 0x0028, + SEV_RET_INCORRECT_BUFFER_LENGTH = 0x0030, + SEV_RET_EXPAND_BUFFER_LENGTH_REQUEST = 0x0031, + SEV_RET_SPDM_REQUEST = 0x0032, + SEV_RET_SPDM_ERROR = 0x0033, + SEV_RET_SEV_STATUS_ERR_IN_DEV_CONN = 0x0035, + SEV_RET_SEV_STATUS_INVALID_DEV_CTX = 0x0036, + SEV_RET_SEV_STATUS_INVALID_TDI_CTX = 0x0037, + SEV_RET_SEV_STATUS_INVALID_TDI = 0x0038, + SEV_RET_SEV_STATUS_RECLAIM_REQUIRED = 0x0039, + SEV_RET_IN_USE = 0x003A, + SEV_RET_SEV_STATUS_INVALID_DEV_STATE = 0x003B, + SEV_RET_SEV_STATUS_INVALID_TDI_STATE = 0x003C, + SEV_RET_SEV_STATUS_DEV_CERT_CHANGED = 0x003D, + SEV_RET_SEV_STATUS_RESYNC_REQ = 0x003E, + SEV_RET_SEV_STATUS_RESPONSE_TOO_LARGE = 0x003F, SEV_RET_MAX, } sev_ret_code; diff --git a/include/uapi/linux/vmcore.h b/include/uapi/linux/vmcore.h index 3e9da91866ff..2ba89fafa518 100644 --- a/include/uapi/linux/vmcore.h +++ b/include/uapi/linux/vmcore.h @@ -15,4 +15,13 @@ struct vmcoredd_header { __u8 dump_name[VMCOREDD_MAX_NAME_BYTES]; /* Device dump's name */ }; +enum hwerr_error_type { + HWERR_RECOV_CPU, + HWERR_RECOV_MEMORY, + HWERR_RECOV_PCI, + HWERR_RECOV_CXL, + HWERR_RECOV_OTHERS, + HWERR_RECOV_MAX, +}; + #endif /* _UAPI_VMCORE_H */ diff --git a/include/xen/xenbus.h b/include/xen/xenbus.h index 7dab04cf4a36..c94caf852aea 100644 --- a/include/xen/xenbus.h +++ b/include/xen/xenbus.h @@ -158,7 +158,7 @@ int xenbus_exists(struct xenbus_transaction t, const char *dir, const char *node); int xenbus_rm(struct xenbus_transaction t, const char *dir, const char *node); int xenbus_transaction_start(struct xenbus_transaction *t); -int xenbus_transaction_end(struct xenbus_transaction t, int abort); +int xenbus_transaction_end(struct xenbus_transaction t, bool abort); /* Single read and scanf: returns -errno or num scanned if > 0. */ __scanf(4, 5) diff --git a/init/Kconfig b/init/Kconfig index ad42558d842d..fa79feb8fe57 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -1519,6 +1519,24 @@ config BOOT_CONFIG_EMBED_FILE This bootconfig will be used if there is no initrd or no other bootconfig in the initrd. +config CMDLINE_LOG_WRAP_IDEAL_LEN + int "Length to try to wrap the cmdline when logged at boot" + default 1021 + range 0 1021 + help + At boot time, the kernel command line is logged to the console. + The log message will start with the prefix "Kernel command line: ". + The log message will attempt to be wrapped (split into multiple log + messages) at spaces based on CMDLINE_LOG_WRAP_IDEAL_LEN characters. + If wrapping happens, each log message will start with the prefix and + all but the last message will end with " \". Messages may exceed the + ideal length if a place to wrap isn't found before the specified + number of characters. + + A value of 0 disables wrapping, though be warned that the maximum + length of a log message (1021 characters) may cause the cmdline to + be truncated. + config INITRAMFS_PRESERVE_MTIME bool "Preserve cpio archive mtimes in initramfs" depends on BLK_DEV_INITRD @@ -2171,6 +2189,8 @@ config TRACEPOINTS source "kernel/Kconfig.kexec" +source "kernel/liveupdate/Kconfig" + endmenu # General setup source "arch/Kconfig" diff --git a/init/calibrate.c b/init/calibrate.c index f3831272f113..63be4c65bc52 100644 --- a/init/calibrate.c +++ b/init/calibrate.c @@ -5,19 +5,22 @@ * Copyright (C) 1991, 1992 Linus Torvalds */ -#include <linux/jiffies.h> #include <linux/delay.h> #include <linux/init.h> -#include <linux/timex.h> -#include <linux/smp.h> +#include <linux/jiffies.h> +#include <linux/kstrtox.h> #include <linux/percpu.h> +#include <linux/printk.h> +#include <linux/smp.h> +#include <linux/stddef.h> +#include <linux/timex.h> unsigned long lpj_fine; unsigned long preset_lpj; + static int __init lpj_setup(char *str) { - preset_lpj = simple_strtoul(str,NULL,0); - return 1; + return kstrtoul(str, 0, &preset_lpj) == 0; } __setup("lpj=", lpj_setup); diff --git a/init/main.c b/init/main.c index 07a3116811c5..b84818ad9685 100644 --- a/init/main.c +++ b/init/main.c @@ -906,6 +906,101 @@ static void __init early_numa_node_init(void) #endif } +#define KERNEL_CMDLINE_PREFIX "Kernel command line: " +#define KERNEL_CMDLINE_PREFIX_LEN (sizeof(KERNEL_CMDLINE_PREFIX) - 1) +#define KERNEL_CMDLINE_CONTINUATION " \\" +#define KERNEL_CMDLINE_CONTINUATION_LEN (sizeof(KERNEL_CMDLINE_CONTINUATION) - 1) + +#define MIN_CMDLINE_LOG_WRAP_IDEAL_LEN (KERNEL_CMDLINE_PREFIX_LEN + \ + KERNEL_CMDLINE_CONTINUATION_LEN) +#define CMDLINE_LOG_WRAP_IDEAL_LEN (CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN > \ + MIN_CMDLINE_LOG_WRAP_IDEAL_LEN ? \ + CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN : \ + MIN_CMDLINE_LOG_WRAP_IDEAL_LEN) + +#define IDEAL_CMDLINE_LEN (CMDLINE_LOG_WRAP_IDEAL_LEN - KERNEL_CMDLINE_PREFIX_LEN) +#define IDEAL_CMDLINE_SPLIT_LEN (IDEAL_CMDLINE_LEN - KERNEL_CMDLINE_CONTINUATION_LEN) + +/** + * print_kernel_cmdline() - Print the kernel cmdline with wrapping. + * @cmdline: The cmdline to print. + * + * Print the kernel command line, trying to wrap based on the Kconfig knob + * CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN. + * + * Wrapping is based on spaces, ignoring quotes. All lines are prefixed + * with "Kernel command line: " and lines that are not the last line have + * a " \" suffix added to them. The prefix and suffix count towards the + * line length for wrapping purposes. The ideal length will be exceeded + * if no appropriate place to wrap is found. + * + * Example output if CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN is 40: + * Kernel command line: loglevel=7 \ + * Kernel command line: init=/sbin/init \ + * Kernel command line: root=PARTUUID=8c3efc1a-768b-6642-8d0c-89eb782f19f0/PARTNROFF=1 \ + * Kernel command line: rootwait ro \ + * Kernel command line: my_quoted_arg="The \ + * Kernel command line: quick brown fox \ + * Kernel command line: jumps over the \ + * Kernel command line: lazy dog." + */ +static void __init print_kernel_cmdline(const char *cmdline) +{ + size_t len; + + /* Config option of 0 or anything longer than the max disables wrapping */ + if (CONFIG_CMDLINE_LOG_WRAP_IDEAL_LEN == 0 || + IDEAL_CMDLINE_LEN >= COMMAND_LINE_SIZE - 1) { + pr_notice("%s%s\n", KERNEL_CMDLINE_PREFIX, cmdline); + return; + } + + len = strlen(cmdline); + while (len > IDEAL_CMDLINE_LEN) { + const char *first_space; + const char *prev_cutoff; + const char *cutoff; + int to_print; + size_t used; + + /* Find the last ' ' that wouldn't make the line too long */ + prev_cutoff = NULL; + cutoff = cmdline; + while (true) { + cutoff = strchr(cutoff + 1, ' '); + if (!cutoff || cutoff - cmdline > IDEAL_CMDLINE_SPLIT_LEN) + break; + prev_cutoff = cutoff; + } + if (prev_cutoff) + cutoff = prev_cutoff; + else if (!cutoff) + break; + + /* Find the beginning and end of the string of spaces */ + first_space = cutoff; + while (first_space > cmdline && first_space[-1] == ' ') + first_space--; + to_print = first_space - cmdline; + while (*cutoff == ' ') + cutoff++; + used = cutoff - cmdline; + + /* If the whole string is used, break and do the final printout */ + if (len == used) + break; + + if (to_print) + pr_notice("%s%.*s%s\n", KERNEL_CMDLINE_PREFIX, + to_print, cmdline, KERNEL_CMDLINE_CONTINUATION); + + len -= used; + cmdline += used; + } + if (len) + pr_notice("%s%s\n", KERNEL_CMDLINE_PREFIX, cmdline); +} + asmlinkage __visible __init __no_sanitize_address __noreturn __no_stack_protector void start_kernel(void) { @@ -942,7 +1037,7 @@ void start_kernel(void) early_numa_node_init(); boot_cpu_hotplug_init(); - pr_notice("Kernel command line: %s\n", saved_command_line); + print_kernel_cmdline(saved_command_line); /* parameters may set static keys */ parse_early_param(); after_dashes = parse_args("Booting kernel", diff --git a/ipc/namespace.c b/ipc/namespace.c index c0dbfdd9015f..535f16ea40e1 100644 --- a/ipc/namespace.c +++ b/ipc/namespace.c @@ -76,10 +76,10 @@ static struct ipc_namespace *create_ipc_ns(struct user_namespace *user_ns, err = -ENOMEM; if (!setup_mq_sysctls(ns)) - goto fail_put; + goto fail_mq_mount; if (!setup_ipc_sysctls(ns)) - goto fail_mq; + goto fail_mq_sysctls; err = msg_init_ns(ns); if (err) @@ -93,9 +93,10 @@ static struct ipc_namespace *create_ipc_ns(struct user_namespace *user_ns, fail_ipc: retire_ipc_sysctls(ns); -fail_mq: +fail_mq_sysctls: retire_mq_sysctls(ns); - +fail_mq_mount: + mntput(ns->mq_mnt); fail_put: put_user_ns(ns->user_ns); ns_common_free(ns); diff --git a/kernel/Kconfig.kexec b/kernel/Kconfig.kexec index 54e581072617..15632358bcf7 100644 --- a/kernel/Kconfig.kexec +++ b/kernel/Kconfig.kexec @@ -94,30 +94,6 @@ config KEXEC_JUMP Jump between original kernel and kexeced kernel and invoke code in physical address mode via KEXEC -config KEXEC_HANDOVER - bool "kexec handover" - depends on ARCH_SUPPORTS_KEXEC_HANDOVER && ARCH_SUPPORTS_KEXEC_FILE - depends on !DEFERRED_STRUCT_PAGE_INIT - select MEMBLOCK_KHO_SCRATCH - select KEXEC_FILE - select DEBUG_FS - select LIBFDT - select CMA - help - Allow kexec to hand over state across kernels by generating and - passing additional metadata to the target kernel. This is useful - to keep data or state alive across the kexec. For this to work, - both source and target kernels need to have this option enabled. - -config KEXEC_HANDOVER_DEBUG - bool "Enable Kexec Handover debug checks" - depends on KEXEC_HANDOVER - help - This option enables extra sanity checks for the Kexec Handover - subsystem. Since, KHO performance is crucial in live update - scenarios and the extra code might be adding overhead it is - only optionally enabled. - config CRASH_DUMP bool "kernel crash dumps" default ARCH_DEFAULT_CRASH_DUMP diff --git a/kernel/Makefile b/kernel/Makefile index 9fe722305c9b..e83669841b8c 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -52,6 +52,7 @@ obj-y += printk/ obj-y += irq/ obj-y += rcu/ obj-y += livepatch/ +obj-y += liveupdate/ obj-y += dma/ obj-y += entry/ obj-y += unwind/ @@ -82,8 +83,6 @@ obj-$(CONFIG_CRASH_DUMP_KUNIT_TEST) += crash_core_test.o obj-$(CONFIG_KEXEC) += kexec.o obj-$(CONFIG_KEXEC_FILE) += kexec_file.o obj-$(CONFIG_KEXEC_ELF) += kexec_elf.o -obj-$(CONFIG_KEXEC_HANDOVER) += kexec_handover.o -obj-$(CONFIG_KEXEC_HANDOVER_DEBUG) += kexec_handover_debug.o obj-$(CONFIG_BACKTRACE_SELF_TEST) += backtracetest.o obj-$(CONFIG_COMPAT) += compat.o obj-$(CONFIG_CGROUPS) += cgroup/ diff --git a/kernel/bounds.c b/kernel/bounds.c index 29b2cd00df2c..02b619eb6106 100644 --- a/kernel/bounds.c +++ b/kernel/bounds.c @@ -6,6 +6,7 @@ */ #define __GENERATING_BOUNDS_H +#define COMPILE_OFFSETS /* Include headers that define the enum constants of interest */ #include <linux/page-flags.h> #include <linux/mmzone.h> diff --git a/kernel/configs/debug.config b/kernel/configs/debug.config index e81327d2cd63..9f6ab7dabf67 100644 --- a/kernel/configs/debug.config +++ b/kernel/configs/debug.config @@ -83,7 +83,7 @@ CONFIG_SLUB_DEBUG_ON=y # # Debug Oops, Lockups and Hangs # -# CONFIG_BOOTPARAM_HUNG_TASK_PANIC is not set +CONFIG_BOOTPARAM_HUNG_TASK_PANIC=0 # CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC is not set CONFIG_DEBUG_ATOMIC_SLEEP=y CONFIG_DETECT_HUNG_TASK=y diff --git a/kernel/crash_reserve.c b/kernel/crash_reserve.c index 87bf4d41eabb..62e60e0223cf 100644 --- a/kernel/crash_reserve.c +++ b/kernel/crash_reserve.c @@ -524,6 +524,9 @@ void __init reserve_crashkernel_cma(unsigned long long cma_size) #ifndef HAVE_ARCH_ADD_CRASH_RES_TO_IOMEM_EARLY static __init int insert_crashkernel_resources(void) { + if (!arch_add_crash_res_to_iomem()) + return 0; + if (crashk_res.start < crashk_res.end) insert_resource(&iomem_resource, &crashk_res); diff --git a/kernel/dma/dummy.c b/kernel/dma/dummy.c index 92de80e5b057..16a51736a2a3 100644 --- a/kernel/dma/dummy.c +++ b/kernel/dma/dummy.c @@ -11,17 +11,16 @@ static int dma_dummy_mmap(struct device *dev, struct vm_area_struct *vma, return -ENXIO; } -static dma_addr_t dma_dummy_map_page(struct device *dev, struct page *page, - unsigned long offset, size_t size, enum dma_data_direction dir, - unsigned long attrs) +static dma_addr_t dma_dummy_map_phys(struct device *dev, phys_addr_t phys, + size_t size, enum dma_data_direction dir, unsigned long attrs) { return DMA_MAPPING_ERROR; } -static void dma_dummy_unmap_page(struct device *dev, dma_addr_t dma_handle, +static void dma_dummy_unmap_phys(struct device *dev, dma_addr_t dma_handle, size_t size, enum dma_data_direction dir, unsigned long attrs) { /* - * Dummy ops doesn't support map_page, so unmap_page should never be + * Dummy ops doesn't support map_phys, so unmap_page should never be * called. */ WARN_ON_ONCE(true); @@ -51,8 +50,8 @@ static int dma_dummy_supported(struct device *hwdev, u64 mask) const struct dma_map_ops dma_dummy_ops = { .mmap = dma_dummy_mmap, - .map_page = dma_dummy_map_page, - .unmap_page = dma_dummy_unmap_page, + .map_phys = dma_dummy_map_phys, + .unmap_phys = dma_dummy_unmap_phys, .map_sg = dma_dummy_map_sg, .unmap_sg = dma_dummy_unmap_sg, .dma_supported = dma_dummy_supported, diff --git a/kernel/dma/map_benchmark.c b/kernel/dma/map_benchmark.c index cc19a3efea89..794041a39e65 100644 --- a/kernel/dma/map_benchmark.c +++ b/kernel/dma/map_benchmark.c @@ -11,13 +11,13 @@ #include <linux/dma-mapping.h> #include <linux/kernel.h> #include <linux/kthread.h> -#include <linux/map_benchmark.h> #include <linux/math64.h> #include <linux/module.h> #include <linux/pci.h> #include <linux/platform_device.h> #include <linux/slab.h> #include <linux/timekeeping.h> +#include <uapi/linux/map_benchmark.h> struct map_benchmark_data { struct map_benchmark bparam; diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index fe7472f13b10..37163eb49f9f 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -157,7 +157,7 @@ dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, { const struct dma_map_ops *ops = get_dma_ops(dev); bool is_mmio = attrs & DMA_ATTR_MMIO; - dma_addr_t addr; + dma_addr_t addr = DMA_MAPPING_ERROR; BUG_ON(!valid_dma_direction(dir)); @@ -169,21 +169,8 @@ dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, addr = dma_direct_map_phys(dev, phys, size, dir, attrs); else if (use_dma_iommu(dev)) addr = iommu_dma_map_phys(dev, phys, size, dir, attrs); - else if (is_mmio) { - if (!ops->map_resource) - return DMA_MAPPING_ERROR; - - addr = ops->map_resource(dev, phys, size, dir, attrs); - } else { - struct page *page = phys_to_page(phys); - size_t offset = offset_in_page(phys); - - /* - * The dma_ops API contract for ops->map_page() requires - * kmappable memory, while ops->map_resource() does not. - */ - addr = ops->map_page(dev, page, offset, size, dir, attrs); - } + else if (ops->map_phys) + addr = ops->map_phys(dev, phys, size, dir, attrs); if (!is_mmio) kmsan_handle_dma(phys, size, dir); @@ -223,11 +210,8 @@ void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, dma_direct_unmap_phys(dev, addr, size, dir, attrs); else if (use_dma_iommu(dev)) iommu_dma_unmap_phys(dev, addr, size, dir, attrs); - else if (is_mmio) { - if (ops->unmap_resource) - ops->unmap_resource(dev, addr, size, dir, attrs); - } else - ops->unmap_page(dev, addr, size, dir, attrs); + else if (ops->unmap_phys) + ops->unmap_phys(dev, addr, size, dir, attrs); trace_dma_unmap_phys(dev, addr, size, dir, attrs); debug_dma_unmap_phys(dev, addr, size, dir); } diff --git a/kernel/dma/ops_helpers.c b/kernel/dma/ops_helpers.c index 6f9d604d9d40..20caf9cabf69 100644 --- a/kernel/dma/ops_helpers.c +++ b/kernel/dma/ops_helpers.c @@ -64,6 +64,7 @@ struct page *dma_common_alloc_pages(struct device *dev, size_t size, { const struct dma_map_ops *ops = get_dma_ops(dev); struct page *page; + phys_addr_t phys; page = dma_alloc_contiguous(dev, size, gfp); if (!page) @@ -71,11 +72,12 @@ struct page *dma_common_alloc_pages(struct device *dev, size_t size, if (!page) return NULL; + phys = page_to_phys(page); if (use_dma_iommu(dev)) - *dma_handle = iommu_dma_map_phys(dev, page_to_phys(page), size, - dir, DMA_ATTR_SKIP_CPU_SYNC); + *dma_handle = iommu_dma_map_phys(dev, phys, size, dir, + DMA_ATTR_SKIP_CPU_SYNC); else - *dma_handle = ops->map_page(dev, page, 0, size, dir, + *dma_handle = ops->map_phys(dev, phys, size, dir, DMA_ATTR_SKIP_CPU_SYNC); if (*dma_handle == DMA_MAPPING_ERROR) { dma_free_contiguous(dev, page, size); @@ -94,8 +96,8 @@ void dma_common_free_pages(struct device *dev, size_t size, struct page *page, if (use_dma_iommu(dev)) iommu_dma_unmap_phys(dev, dma_handle, size, dir, DMA_ATTR_SKIP_CPU_SYNC); - else if (ops->unmap_page) - ops->unmap_page(dev, dma_handle, size, dir, + else if (ops->unmap_phys) + ops->unmap_phys(dev, dma_handle, size, dir, DMA_ATTR_SKIP_CPU_SYNC); dma_free_contiguous(dev, page, size); } diff --git a/kernel/exit.c b/kernel/exit.c index 4dc1918db67b..8a87021211ae 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -251,10 +251,8 @@ repeat: memset(&post, 0, sizeof(post)); /* don't need to get the RCU readlock here - the process is dead and - * can't be modifying its own credentials. But shut RCU-lockdep up */ - rcu_read_lock(); + * can't be modifying its own credentials. */ dec_rlimit_ucounts(task_ucounts(p), UCOUNT_RLIMIT_NPROC, 1); - rcu_read_unlock(); pidfs_exit(p); cgroup_task_release(p); diff --git a/kernel/fork.c b/kernel/fork.c index 198e02e21e6e..b1f3915d5f8e 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -208,15 +208,62 @@ struct vm_stack { struct vm_struct *stack_vm_area; }; +static struct vm_struct *alloc_thread_stack_node_from_cache(struct task_struct *tsk, int node) +{ + struct vm_struct *vm_area; + unsigned int i; + + /* + * If the node has memory, we are guaranteed the stacks are backed by local pages. + * Otherwise the pages are arbitrary. + * + * Note that depending on cpuset it is possible we will get migrated to a different + * node immediately after allocating here, so this does *not* guarantee locality for + * arbitrary callers. + */ + scoped_guard(preempt) { + if (node != NUMA_NO_NODE && numa_node_id() != node) + return NULL; + + for (i = 0; i < NR_CACHED_STACKS; i++) { + vm_area = this_cpu_xchg(cached_stacks[i], NULL); + if (vm_area) + return vm_area; + } + } + + return NULL; +} + static bool try_release_thread_stack_to_cache(struct vm_struct *vm_area) { unsigned int i; + int nid; - for (i = 0; i < NR_CACHED_STACKS; i++) { - struct vm_struct *tmp = NULL; + /* + * Don't cache stacks if any of the pages don't match the local domain, unless + * there is no local memory to begin with. + * + * Note that lack of local memory does not automatically mean it makes no difference + * performance-wise which other domain backs the stack. In this case we are merely + * trying to avoid constantly going to vmalloc. + */ + scoped_guard(preempt) { + nid = numa_node_id(); + if (node_state(nid, N_MEMORY)) { + for (i = 0; i < vm_area->nr_pages; i++) { + struct page *page = vm_area->pages[i]; + if (page_to_nid(page) != nid) + return false; + } + } + + for (i = 0; i < NR_CACHED_STACKS; i++) { + struct vm_struct *tmp = NULL; - if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm_area)) - return true; + if (this_cpu_try_cmpxchg(cached_stacks[i], &tmp, vm_area)) + return true; + } } return false; } @@ -283,13 +330,9 @@ static int alloc_thread_stack_node(struct task_struct *tsk, int node) { struct vm_struct *vm_area; void *stack; - int i; - - for (i = 0; i < NR_CACHED_STACKS; i++) { - vm_area = this_cpu_xchg(cached_stacks[i], NULL); - if (!vm_area) - continue; + vm_area = alloc_thread_stack_node_from_cache(tsk, node); + if (vm_area) { if (memcg_charge_kernel_stack(vm_area)) { vfree(vm_area->addr); return -ENOMEM; diff --git a/kernel/hung_task.c b/kernel/hung_task.c index b2c1f14b8129..d2254c91450b 100644 --- a/kernel/hung_task.c +++ b/kernel/hung_task.c @@ -24,6 +24,7 @@ #include <linux/sched/sysctl.h> #include <linux/hung_task.h> #include <linux/rwsem.h> +#include <linux/sys_info.h> #include <trace/events/sched.h> @@ -50,7 +51,6 @@ static unsigned long __read_mostly sysctl_hung_task_detect_count; * Zero means infinite timeout - no checking done: */ unsigned long __read_mostly sysctl_hung_task_timeout_secs = CONFIG_DEFAULT_HUNG_TASK_TIMEOUT; -EXPORT_SYMBOL_GPL(sysctl_hung_task_timeout_secs); /* * Zero (default value) means use sysctl_hung_task_timeout_secs: @@ -60,12 +60,17 @@ static unsigned long __read_mostly sysctl_hung_task_check_interval_secs; static int __read_mostly sysctl_hung_task_warnings = 10; static int __read_mostly did_panic; -static bool hung_task_show_lock; static bool hung_task_call_panic; -static bool hung_task_show_all_bt; static struct task_struct *watchdog_task; +/* + * A bitmask to control what kinds of system info to be printed when + * a hung task is detected, it could be task, memory, lock etc. Refer + * include/linux/sys_info.h for detailed bit definition. + */ +static unsigned long hung_task_si_mask; + #ifdef CONFIG_SMP /* * Should we dump all CPUs backtraces in a hung task event? @@ -81,7 +86,7 @@ static unsigned int __read_mostly sysctl_hung_task_all_cpu_backtrace; * hung task is detected: */ static unsigned int __read_mostly sysctl_hung_task_panic = - IS_ENABLED(CONFIG_BOOTPARAM_HUNG_TASK_PANIC); + CONFIG_BOOTPARAM_HUNG_TASK_PANIC; static int hung_task_panic(struct notifier_block *this, unsigned long event, void *ptr) @@ -218,8 +223,11 @@ static inline void debug_show_blocker(struct task_struct *task, unsigned long ti } #endif -static void check_hung_task(struct task_struct *t, unsigned long timeout) +static void check_hung_task(struct task_struct *t, unsigned long timeout, + unsigned long prev_detect_count) { + unsigned long total_hung_task; + if (!task_is_hung(t, timeout)) return; @@ -229,11 +237,11 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout) */ sysctl_hung_task_detect_count++; + total_hung_task = sysctl_hung_task_detect_count - prev_detect_count; trace_sched_process_hang(t); - if (sysctl_hung_task_panic) { + if (sysctl_hung_task_panic && total_hung_task >= sysctl_hung_task_panic) { console_verbose(); - hung_task_show_lock = true; hung_task_call_panic = true; } @@ -256,10 +264,7 @@ static void check_hung_task(struct task_struct *t, unsigned long timeout) " disables this message.\n"); sched_show_task(t); debug_show_blocker(t, timeout); - hung_task_show_lock = true; - if (sysctl_hung_task_all_cpu_backtrace) - hung_task_show_all_bt = true; if (!sysctl_hung_task_warnings) pr_info("Future hung task reports are suppressed, see sysctl kernel.hung_task_warnings\n"); } @@ -300,6 +305,9 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout) int max_count = sysctl_hung_task_check_count; unsigned long last_break = jiffies; struct task_struct *g, *t; + unsigned long prev_detect_count = sysctl_hung_task_detect_count; + int need_warning = sysctl_hung_task_warnings; + unsigned long si_mask = hung_task_si_mask; /* * If the system crashed already then all bets are off, @@ -308,7 +316,7 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout) if (test_taint(TAINT_DIE) || did_panic) return; - hung_task_show_lock = false; + rcu_read_lock(); for_each_process_thread(g, t) { @@ -320,18 +328,23 @@ static void check_hung_uninterruptible_tasks(unsigned long timeout) last_break = jiffies; } - check_hung_task(t, timeout); + check_hung_task(t, timeout, prev_detect_count); } unlock: rcu_read_unlock(); - if (hung_task_show_lock) - debug_show_all_locks(); - if (hung_task_show_all_bt) { - hung_task_show_all_bt = false; - trigger_all_cpu_backtrace(); + if (!(sysctl_hung_task_detect_count - prev_detect_count)) + return; + + if (need_warning || hung_task_call_panic) { + si_mask |= SYS_INFO_LOCKS; + + if (sysctl_hung_task_all_cpu_backtrace) + si_mask |= SYS_INFO_ALL_BT; } + sys_info(si_mask); + if (hung_task_call_panic) panic("hung_task: blocked tasks"); } @@ -389,7 +402,7 @@ static const struct ctl_table hung_task_sysctls[] = { .mode = 0644, .proc_handler = proc_dointvec_minmax, .extra1 = SYSCTL_ZERO, - .extra2 = SYSCTL_ONE, + .extra2 = SYSCTL_INT_MAX, }, { .procname = "hung_task_check_count", @@ -430,6 +443,13 @@ static const struct ctl_table hung_task_sysctls[] = { .mode = 0444, .proc_handler = proc_doulongvec_minmax, }, + { + .procname = "hung_task_sys_info", + .data = &hung_task_si_mask, + .maxlen = sizeof(hung_task_si_mask), + .mode = 0644, + .proc_handler = sysctl_sys_info_handler, + }, }; static void __init hung_task_sysctl_init(void) diff --git a/kernel/kexec_core.c b/kernel/kexec_core.c index fa00b239c5d9..0f92acdd354d 100644 --- a/kernel/kexec_core.c +++ b/kernel/kexec_core.c @@ -15,6 +15,7 @@ #include <linux/kexec.h> #include <linux/mutex.h> #include <linux/list.h> +#include <linux/liveupdate.h> #include <linux/highmem.h> #include <linux/syscalls.h> #include <linux/reboot.h> @@ -41,6 +42,7 @@ #include <linux/objtool.h> #include <linux/kmsg_dump.h> #include <linux/dma-map-ops.h> +#include <linux/sysfs.h> #include <asm/page.h> #include <asm/sections.h> @@ -742,7 +744,6 @@ static int kimage_load_cma_segment(struct kimage *image, int idx) struct kexec_segment *segment = &image->segment[idx]; struct page *cma = image->segment_cma[idx]; char *ptr = page_address(cma); - unsigned long maddr; size_t ubytes, mbytes; int result = 0; unsigned char __user *buf = NULL; @@ -754,15 +755,12 @@ static int kimage_load_cma_segment(struct kimage *image, int idx) buf = segment->buf; ubytes = segment->bufsz; mbytes = segment->memsz; - maddr = segment->mem; /* Then copy from source buffer to the CMA one */ while (mbytes) { size_t uchunk, mchunk; - ptr += maddr & ~PAGE_MASK; - mchunk = min_t(size_t, mbytes, - PAGE_SIZE - (maddr & ~PAGE_MASK)); + mchunk = min_t(size_t, mbytes, PAGE_SIZE); uchunk = min(ubytes, mchunk); if (uchunk) { @@ -784,7 +782,6 @@ static int kimage_load_cma_segment(struct kimage *image, int idx) } ptr += mchunk; - maddr += mchunk; mbytes -= mchunk; cond_resched(); @@ -839,9 +836,7 @@ static int kimage_load_normal_segment(struct kimage *image, int idx) ptr = kmap_local_page(page); /* Start with a clear page */ clear_page(ptr); - ptr += maddr & ~PAGE_MASK; - mchunk = min_t(size_t, mbytes, - PAGE_SIZE - (maddr & ~PAGE_MASK)); + mchunk = min_t(size_t, mbytes, PAGE_SIZE); uchunk = min(ubytes, mchunk); if (uchunk) { @@ -904,9 +899,7 @@ static int kimage_load_crash_segment(struct kimage *image, int idx) } arch_kexec_post_alloc_pages(page_address(page), 1, 0); ptr = kmap_local_page(page); - ptr += maddr & ~PAGE_MASK; - mchunk = min_t(size_t, mbytes, - PAGE_SIZE - (maddr & ~PAGE_MASK)); + mchunk = min_t(size_t, mbytes, PAGE_SIZE); uchunk = min(ubytes, mchunk); if (mchunk > uchunk) { /* Zero the trailing part of the page */ @@ -1146,6 +1139,10 @@ int kernel_kexec(void) goto Unlock; } + error = liveupdate_reboot(); + if (error) + goto Unlock; + #ifdef CONFIG_KEXEC_JUMP if (kexec_image->preserve_context) { /* @@ -1229,3 +1226,143 @@ int kernel_kexec(void) kexec_unlock(); return error; } + +static ssize_t loaded_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", !!kexec_image); +} +static struct kobj_attribute loaded_attr = __ATTR_RO(loaded); + +#ifdef CONFIG_CRASH_DUMP +static ssize_t crash_loaded_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + return sysfs_emit(buf, "%d\n", kexec_crash_loaded()); +} +static struct kobj_attribute crash_loaded_attr = __ATTR_RO(crash_loaded); + +#ifdef CONFIG_CRASH_RESERVE +static ssize_t crash_cma_ranges_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + + ssize_t len = 0; + int i; + + for (i = 0; i < crashk_cma_cnt; ++i) { + len += sysfs_emit_at(buf, len, "%08llx-%08llx\n", + crashk_cma_ranges[i].start, + crashk_cma_ranges[i].end); + } + return len; +} +static struct kobj_attribute crash_cma_ranges_attr = __ATTR_RO(crash_cma_ranges); +#endif + +static ssize_t crash_size_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + ssize_t size = crash_get_memory_size(); + + if (size < 0) + return size; + + return sysfs_emit(buf, "%zd\n", size); +} +static ssize_t crash_size_store(struct kobject *kobj, + struct kobj_attribute *attr, + const char *buf, size_t count) +{ + unsigned long cnt; + int ret; + + if (kstrtoul(buf, 0, &cnt)) + return -EINVAL; + + ret = crash_shrink_memory(cnt); + return ret < 0 ? ret : count; +} +static struct kobj_attribute crash_size_attr = __ATTR_RW(crash_size); + +#ifdef CONFIG_CRASH_HOTPLUG +static ssize_t crash_elfcorehdr_size_show(struct kobject *kobj, + struct kobj_attribute *attr, char *buf) +{ + unsigned int sz = crash_get_elfcorehdr_size(); + + return sysfs_emit(buf, "%u\n", sz); +} +static struct kobj_attribute crash_elfcorehdr_size_attr = __ATTR_RO(crash_elfcorehdr_size); + +#endif /* CONFIG_CRASH_HOTPLUG */ +#endif /* CONFIG_CRASH_DUMP */ + +static struct attribute *kexec_attrs[] = { + &loaded_attr.attr, +#ifdef CONFIG_CRASH_DUMP + &crash_loaded_attr.attr, + &crash_size_attr.attr, +#ifdef CONFIG_CRASH_RESERVE + &crash_cma_ranges_attr.attr, +#endif +#ifdef CONFIG_CRASH_HOTPLUG + &crash_elfcorehdr_size_attr.attr, +#endif +#endif + NULL +}; + +struct kexec_link_entry { + const char *target; + const char *name; +}; + +static struct kexec_link_entry kexec_links[] = { + { "loaded", "kexec_loaded" }, +#ifdef CONFIG_CRASH_DUMP + { "crash_loaded", "kexec_crash_loaded" }, + { "crash_size", "kexec_crash_size" }, +#ifdef CONFIG_CRASH_RESERVE + {"crash_cma_ranges", "kexec_crash_cma_ranges"}, +#endif +#ifdef CONFIG_CRASH_HOTPLUG + { "crash_elfcorehdr_size", "crash_elfcorehdr_size" }, +#endif +#endif +}; + +static struct kobject *kexec_kobj; +ATTRIBUTE_GROUPS(kexec); + +static int __init init_kexec_sysctl(void) +{ + int error; + int i; + + kexec_kobj = kobject_create_and_add("kexec", kernel_kobj); + if (!kexec_kobj) { + pr_err("failed to create kexec kobject\n"); + return -ENOMEM; + } + + error = sysfs_create_groups(kexec_kobj, kexec_groups); + if (error) + goto kset_exit; + + for (i = 0; i < ARRAY_SIZE(kexec_links); i++) { + error = compat_only_sysfs_link_entry_to_kobj(kernel_kobj, kexec_kobj, + kexec_links[i].target, + kexec_links[i].name); + if (error) + pr_err("Unable to create %s symlink (%d)", kexec_links[i].name, error); + } + + return 0; + +kset_exit: + kobject_put(kexec_kobj); + return error; +} + +subsys_initcall(init_kexec_sysctl); diff --git a/kernel/kexec_handover_internal.h b/kernel/kexec_handover_internal.h deleted file mode 100644 index 3c3c7148ceed..000000000000 --- a/kernel/kexec_handover_internal.h +++ /dev/null @@ -1,20 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 */ -#ifndef LINUX_KEXEC_HANDOVER_INTERNAL_H -#define LINUX_KEXEC_HANDOVER_INTERNAL_H - -#include <linux/kexec_handover.h> -#include <linux/types.h> - -extern struct kho_scratch *kho_scratch; -extern unsigned int kho_scratch_cnt; - -#ifdef CONFIG_KEXEC_HANDOVER_DEBUG -bool kho_scratch_overlap(phys_addr_t phys, size_t size); -#else -static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size) -{ - return false; -} -#endif /* CONFIG_KEXEC_HANDOVER_DEBUG */ - -#endif /* LINUX_KEXEC_HANDOVER_INTERNAL_H */ diff --git a/kernel/ksysfs.c b/kernel/ksysfs.c index eefb67d9883c..a9e6354d9e25 100644 --- a/kernel/ksysfs.c +++ b/kernel/ksysfs.c @@ -12,7 +12,7 @@ #include <linux/sysfs.h> #include <linux/export.h> #include <linux/init.h> -#include <linux/kexec.h> +#include <linux/vmcore_info.h> #include <linux/profile.h> #include <linux/stat.h> #include <linux/sched.h> @@ -119,50 +119,6 @@ static ssize_t profiling_store(struct kobject *kobj, KERNEL_ATTR_RW(profiling); #endif -#ifdef CONFIG_KEXEC_CORE -static ssize_t kexec_loaded_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "%d\n", !!kexec_image); -} -KERNEL_ATTR_RO(kexec_loaded); - -#ifdef CONFIG_CRASH_DUMP -static ssize_t kexec_crash_loaded_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - return sysfs_emit(buf, "%d\n", kexec_crash_loaded()); -} -KERNEL_ATTR_RO(kexec_crash_loaded); - -static ssize_t kexec_crash_size_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - ssize_t size = crash_get_memory_size(); - - if (size < 0) - return size; - - return sysfs_emit(buf, "%zd\n", size); -} -static ssize_t kexec_crash_size_store(struct kobject *kobj, - struct kobj_attribute *attr, - const char *buf, size_t count) -{ - unsigned long cnt; - int ret; - - if (kstrtoul(buf, 0, &cnt)) - return -EINVAL; - - ret = crash_shrink_memory(cnt); - return ret < 0 ? ret : count; -} -KERNEL_ATTR_RW(kexec_crash_size); - -#endif /* CONFIG_CRASH_DUMP*/ -#endif /* CONFIG_KEXEC_CORE */ - #ifdef CONFIG_VMCORE_INFO static ssize_t vmcoreinfo_show(struct kobject *kobj, @@ -174,18 +130,6 @@ static ssize_t vmcoreinfo_show(struct kobject *kobj, } KERNEL_ATTR_RO(vmcoreinfo); -#ifdef CONFIG_CRASH_HOTPLUG -static ssize_t crash_elfcorehdr_size_show(struct kobject *kobj, - struct kobj_attribute *attr, char *buf) -{ - unsigned int sz = crash_get_elfcorehdr_size(); - - return sysfs_emit(buf, "%u\n", sz); -} -KERNEL_ATTR_RO(crash_elfcorehdr_size); - -#endif - #endif /* CONFIG_VMCORE_INFO */ /* whether file capabilities are enabled */ @@ -255,18 +199,8 @@ static struct attribute * kernel_attrs[] = { #ifdef CONFIG_PROFILING &profiling_attr.attr, #endif -#ifdef CONFIG_KEXEC_CORE - &kexec_loaded_attr.attr, -#ifdef CONFIG_CRASH_DUMP - &kexec_crash_loaded_attr.attr, - &kexec_crash_size_attr.attr, -#endif -#endif #ifdef CONFIG_VMCORE_INFO &vmcoreinfo_attr.attr, -#ifdef CONFIG_CRASH_HOTPLUG - &crash_elfcorehdr_size_attr.attr, -#endif #endif #ifndef CONFIG_TINY_RCU &rcu_expedited_attr.attr, diff --git a/kernel/liveupdate/Kconfig b/kernel/liveupdate/Kconfig new file mode 100644 index 000000000000..9b2515f31afb --- /dev/null +++ b/kernel/liveupdate/Kconfig @@ -0,0 +1,75 @@ +# SPDX-License-Identifier: GPL-2.0-only + +menu "Live Update and Kexec HandOver" + depends on !DEFERRED_STRUCT_PAGE_INIT + +config KEXEC_HANDOVER + bool "kexec handover" + depends on ARCH_SUPPORTS_KEXEC_HANDOVER && ARCH_SUPPORTS_KEXEC_FILE + depends on !DEFERRED_STRUCT_PAGE_INIT + select MEMBLOCK_KHO_SCRATCH + select KEXEC_FILE + select LIBFDT + select CMA + help + Allow kexec to hand over state across kernels by generating and + passing additional metadata to the target kernel. This is useful + to keep data or state alive across the kexec. For this to work, + both source and target kernels need to have this option enabled. + +config KEXEC_HANDOVER_DEBUG + bool "Enable Kexec Handover debug checks" + depends on KEXEC_HANDOVER + help + This option enables extra sanity checks for the Kexec Handover + subsystem. Since, KHO performance is crucial in live update + scenarios and the extra code might be adding overhead it is + only optionally enabled. + +config KEXEC_HANDOVER_DEBUGFS + bool "kexec handover debugfs interface" + default KEXEC_HANDOVER + depends on KEXEC_HANDOVER + select DEBUG_FS + help + Allow to control kexec handover device tree via debugfs + interface, i.e. finalize the state or aborting the finalization. + Also, enables inspecting the KHO fdt trees with the debugfs binary + blobs. + +config KEXEC_HANDOVER_ENABLE_DEFAULT + bool "Enable kexec handover by default" + depends on KEXEC_HANDOVER + help + Enable Kexec Handover by default. This avoids the need to + explicitly pass 'kho=on' on the kernel command line. + + This is useful for systems where KHO is a prerequisite for other + features, such as Live Update, ensuring the mechanism is always + active. + + The default behavior can still be overridden at boot time by + passing 'kho=off'. + +config LIVEUPDATE + bool "Live Update Orchestrator" + depends on KEXEC_HANDOVER + help + Enable the Live Update Orchestrator. Live Update is a mechanism, + typically based on kexec, that allows the kernel to be updated + while keeping selected devices operational across the transition. + These devices are intended to be reclaimed by the new kernel and + re-attached to their original workload without requiring a device + reset. + + Ability to handover a device from current to the next kernel depends + on specific support within device drivers and related kernel + subsystems. + + This feature primarily targets virtual machine hosts to quickly update + the kernel hypervisor with minimal disruption to the running virtual + machines. + + If unsure, say N. + +endmenu diff --git a/kernel/liveupdate/Makefile b/kernel/liveupdate/Makefile new file mode 100644 index 000000000000..7cad2eece32d --- /dev/null +++ b/kernel/liveupdate/Makefile @@ -0,0 +1,12 @@ +# SPDX-License-Identifier: GPL-2.0 + +luo-y := \ + luo_core.o \ + luo_file.o \ + luo_session.o + +obj-$(CONFIG_KEXEC_HANDOVER) += kexec_handover.o +obj-$(CONFIG_KEXEC_HANDOVER_DEBUG) += kexec_handover_debug.o +obj-$(CONFIG_KEXEC_HANDOVER_DEBUGFS) += kexec_handover_debugfs.o + +obj-$(CONFIG_LIVEUPDATE) += luo.o diff --git a/kernel/kexec_handover.c b/kernel/liveupdate/kexec_handover.c index 03d12e27189f..9dc51fab604f 100644 --- a/kernel/kexec_handover.c +++ b/kernel/liveupdate/kexec_handover.c @@ -4,21 +4,22 @@ * Copyright (C) 2023 Alexander Graf <graf@amazon.com> * Copyright (C) 2025 Microsoft Corporation, Mike Rapoport <rppt@kernel.org> * Copyright (C) 2025 Google LLC, Changyuan Lyu <changyuanl@google.com> + * Copyright (C) 2025 Pasha Tatashin <pasha.tatashin@soleen.com> */ #define pr_fmt(fmt) "KHO: " fmt #include <linux/cleanup.h> #include <linux/cma.h> +#include <linux/kmemleak.h> #include <linux/count_zeros.h> -#include <linux/debugfs.h> #include <linux/kexec.h> #include <linux/kexec_handover.h> #include <linux/libfdt.h> #include <linux/list.h> #include <linux/memblock.h> -#include <linux/notifier.h> #include <linux/page-isolation.h> +#include <linux/unaligned.h> #include <linux/vmalloc.h> #include <asm/early_ioremap.h> @@ -28,8 +29,9 @@ * KHO is tightly coupled with mm init and needs access to some of mm * internal APIs. */ -#include "../mm/internal.h" -#include "kexec_internal.h" +#include "../../mm/internal.h" +#include "../kexec_internal.h" +#include "kexec_handover_internal.h" #define KHO_FDT_COMPATIBLE "kho-v1" #define PROP_PRESERVED_MEMORY_MAP "preserved-memory-map" @@ -51,7 +53,7 @@ union kho_page_info { static_assert(sizeof(union kho_page_info) == sizeof(((struct page *)0)->private)); -static bool kho_enable __ro_after_init; +static bool kho_enable __ro_after_init = IS_ENABLED(CONFIG_KEXEC_HANDOVER_ENABLE_DEFAULT); bool kho_is_enabled(void) { @@ -103,34 +105,19 @@ struct kho_mem_track { struct khoser_mem_chunk; -struct kho_serialization { - struct page *fdt; - struct list_head fdt_list; - struct dentry *sub_fdt_dir; - struct kho_mem_track track; - /* First chunk of serialized preserved memory map */ - struct khoser_mem_chunk *preserved_mem_map; -}; - struct kho_out { - struct blocking_notifier_head chain_head; - - struct dentry *dir; - + void *fdt; + bool finalized; struct mutex lock; /* protects KHO FDT finalization */ - struct kho_serialization ser; - bool finalized; + struct kho_mem_track track; + struct kho_debugfs dbg; }; static struct kho_out kho_out = { - .chain_head = BLOCKING_NOTIFIER_INIT(kho_out.chain_head), .lock = __MUTEX_INITIALIZER(kho_out.lock), - .ser = { - .fdt_list = LIST_HEAD_INIT(kho_out.ser.fdt_list), - .track = { - .orders = XARRAY_INIT(kho_out.ser.track.orders, 0), - }, + .track = { + .orders = XARRAY_INIT(kho_out.track.orders, 0), }, .finalized = false, }; @@ -159,26 +146,33 @@ static void *xa_load_or_alloc(struct xarray *xa, unsigned long index) return no_free_ptr(elm); } -static void __kho_unpreserve(struct kho_mem_track *track, unsigned long pfn, - unsigned long end_pfn) +static void __kho_unpreserve_order(struct kho_mem_track *track, unsigned long pfn, + unsigned int order) { struct kho_mem_phys_bits *bits; struct kho_mem_phys *physxa; + const unsigned long pfn_high = pfn >> order; - while (pfn < end_pfn) { - const unsigned int order = - min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn)); - const unsigned long pfn_high = pfn >> order; + physxa = xa_load(&track->orders, order); + if (WARN_ON_ONCE(!physxa)) + return; - physxa = xa_load(&track->orders, order); - if (WARN_ON_ONCE(!physxa)) - return; + bits = xa_load(&physxa->phys_bits, pfn_high / PRESERVE_BITS); + if (WARN_ON_ONCE(!bits)) + return; - bits = xa_load(&physxa->phys_bits, pfn_high / PRESERVE_BITS); - if (WARN_ON_ONCE(!bits)) - return; + clear_bit(pfn_high % PRESERVE_BITS, bits->preserve); +} - clear_bit(pfn_high % PRESERVE_BITS, bits->preserve); +static void __kho_unpreserve(struct kho_mem_track *track, unsigned long pfn, + unsigned long end_pfn) +{ + unsigned int order; + + while (pfn < end_pfn) { + order = min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn)); + + __kho_unpreserve_order(track, pfn, order); pfn += 1 << order; } @@ -192,10 +186,6 @@ static int __kho_preserve_order(struct kho_mem_track *track, unsigned long pfn, const unsigned long pfn_high = pfn >> order; might_sleep(); - - if (kho_out.finalized) - return -EBUSY; - physxa = xa_load(&track->orders, order); if (!physxa) { int err; @@ -229,11 +219,11 @@ static int __kho_preserve_order(struct kho_mem_track *track, unsigned long pfn, return 0; } -static struct page *kho_restore_page(phys_addr_t phys) +static struct page *kho_restore_page(phys_addr_t phys, bool is_folio) { struct page *page = pfn_to_online_page(PHYS_PFN(phys)); + unsigned int nr_pages, ref_cnt; union kho_page_info info; - unsigned int nr_pages; if (!page) return NULL; @@ -253,11 +243,16 @@ static struct page *kho_restore_page(phys_addr_t phys) /* Head page gets refcount of 1. */ set_page_count(page, 1); - /* For higher order folios, tail pages get a page count of zero. */ + /* + * For higher order folios, tail pages get a page count of zero. + * For physically contiguous order-0 pages every pages gets a page + * count of 1 + */ + ref_cnt = is_folio ? 0 : 1; for (unsigned int i = 1; i < nr_pages; i++) - set_page_count(page + i, 0); + set_page_count(page + i, ref_cnt); - if (info.order > 0) + if (is_folio && info.order) prep_compound_page(page, info.order); adjust_managed_page_count(page, nr_pages); @@ -272,7 +267,7 @@ static struct page *kho_restore_page(phys_addr_t phys) */ struct folio *kho_restore_folio(phys_addr_t phys) { - struct page *page = kho_restore_page(phys); + struct page *page = kho_restore_page(phys, true); return page ? page_folio(page) : NULL; } @@ -297,11 +292,10 @@ struct page *kho_restore_pages(phys_addr_t phys, unsigned int nr_pages) while (pfn < end_pfn) { const unsigned int order = min(count_trailing_zeros(pfn), ilog2(end_pfn - pfn)); - struct page *page = kho_restore_page(PFN_PHYS(pfn)); + struct page *page = kho_restore_page(PFN_PHYS(pfn), false); if (!page) return NULL; - split_page(page, order); pfn += 1 << order; } @@ -371,11 +365,32 @@ static void kho_mem_ser_free(struct khoser_mem_chunk *first_chunk) struct khoser_mem_chunk *tmp = chunk; chunk = KHOSER_LOAD_PTR(chunk->hdr.next); - kfree(tmp); + free_page((unsigned long)tmp); } } -static int kho_mem_serialize(struct kho_serialization *ser) +/* + * Update memory map property, if old one is found discard it via + * kho_mem_ser_free(). + */ +static void kho_update_memory_map(struct khoser_mem_chunk *first_chunk) +{ + void *ptr; + u64 phys; + + ptr = fdt_getprop_w(kho_out.fdt, 0, PROP_PRESERVED_MEMORY_MAP, NULL); + + /* Check and discard previous memory map */ + phys = get_unaligned((u64 *)ptr); + if (phys) + kho_mem_ser_free((struct khoser_mem_chunk *)phys_to_virt(phys)); + + /* Update with the new value */ + phys = first_chunk ? (u64)virt_to_phys(first_chunk) : 0; + put_unaligned(phys, (u64 *)ptr); +} + +static int kho_mem_serialize(struct kho_out *kho_out) { struct khoser_mem_chunk *first_chunk = NULL; struct khoser_mem_chunk *chunk = NULL; @@ -383,7 +398,7 @@ static int kho_mem_serialize(struct kho_serialization *ser) unsigned long order; int err = -ENOMEM; - xa_for_each(&ser->track.orders, order, physxa) { + xa_for_each(&kho_out->track.orders, order, physxa) { struct kho_mem_phys_bits *bits; unsigned long phys; @@ -415,7 +430,7 @@ static int kho_mem_serialize(struct kho_serialization *ser) } } - ser->preserved_mem_map = first_chunk; + kho_update_memory_map(first_chunk); return 0; @@ -445,20 +460,27 @@ static void __init deserialize_bitmap(unsigned int order, } } -static void __init kho_mem_deserialize(const void *fdt) +/* Return true if memory was deserizlied */ +static bool __init kho_mem_deserialize(const void *fdt) { struct khoser_mem_chunk *chunk; - const phys_addr_t *mem; + const void *mem_ptr; + u64 mem; int len; - mem = fdt_getprop(fdt, 0, PROP_PRESERVED_MEMORY_MAP, &len); - - if (!mem || len != sizeof(*mem)) { + mem_ptr = fdt_getprop(fdt, 0, PROP_PRESERVED_MEMORY_MAP, &len); + if (!mem_ptr || len != sizeof(u64)) { pr_err("failed to get preserved memory bitmaps\n"); - return; + return false; } - chunk = *mem ? phys_to_virt(*mem) : NULL; + mem = get_unaligned((const u64 *)mem_ptr); + chunk = mem ? phys_to_virt(mem) : NULL; + + /* No preserved physical pages were passed, no deserialization */ + if (!chunk) + return false; + while (chunk) { unsigned int i; @@ -467,6 +489,8 @@ static void __init kho_mem_deserialize(const void *fdt) &chunk->bitmaps[i]); chunk = KHOSER_LOAD_PTR(chunk->hdr.next); } + + return true; } /* @@ -674,40 +698,8 @@ err_disable_kho: kho_enable = false; } -struct fdt_debugfs { - struct list_head list; - struct debugfs_blob_wrapper wrapper; - struct dentry *file; -}; - -static int kho_debugfs_fdt_add(struct list_head *list, struct dentry *dir, - const char *name, const void *fdt) -{ - struct fdt_debugfs *f; - struct dentry *file; - - f = kmalloc(sizeof(*f), GFP_KERNEL); - if (!f) - return -ENOMEM; - - f->wrapper.data = (void *)fdt; - f->wrapper.size = fdt_totalsize(fdt); - - file = debugfs_create_blob(name, 0400, dir, &f->wrapper); - if (IS_ERR(file)) { - kfree(f); - return PTR_ERR(file); - } - - f->file = file; - list_add(&f->list, list); - - return 0; -} - /** * kho_add_subtree - record the physical address of a sub FDT in KHO root tree. - * @ser: serialization control object passed by KHO notifiers. * @name: name of the sub tree. * @fdt: the sub tree blob. * @@ -716,38 +708,76 @@ static int kho_debugfs_fdt_add(struct list_head *list, struct dentry *dir, * by KHO for the new kernel to retrieve it after kexec. * * A debugfs blob entry is also created at - * ``/sys/kernel/debug/kho/out/sub_fdts/@name``. + * ``/sys/kernel/debug/kho/out/sub_fdts/@name`` when kernel is configured with + * CONFIG_KEXEC_HANDOVER_DEBUGFS * * Return: 0 on success, error code on failure */ -int kho_add_subtree(struct kho_serialization *ser, const char *name, void *fdt) +int kho_add_subtree(const char *name, void *fdt) { - int err = 0; - u64 phys = (u64)virt_to_phys(fdt); - void *root = page_to_virt(ser->fdt); + phys_addr_t phys = virt_to_phys(fdt); + void *root_fdt = kho_out.fdt; + int err = -ENOMEM; + int off, fdt_err; - err |= fdt_begin_node(root, name); - err |= fdt_property(root, PROP_SUB_FDT, &phys, sizeof(phys)); - err |= fdt_end_node(root); + guard(mutex)(&kho_out.lock); - if (err) + fdt_err = fdt_open_into(root_fdt, root_fdt, PAGE_SIZE); + if (fdt_err < 0) return err; - return kho_debugfs_fdt_add(&ser->fdt_list, ser->sub_fdt_dir, name, fdt); + off = fdt_add_subnode(root_fdt, 0, name); + if (off < 0) { + if (off == -FDT_ERR_EXISTS) + err = -EEXIST; + goto out_pack; + } + + err = fdt_setprop(root_fdt, off, PROP_SUB_FDT, &phys, sizeof(phys)); + if (err < 0) + goto out_pack; + + WARN_ON_ONCE(kho_debugfs_fdt_add(&kho_out.dbg, name, fdt, false)); + +out_pack: + fdt_pack(root_fdt); + + return err; } EXPORT_SYMBOL_GPL(kho_add_subtree); -int register_kho_notifier(struct notifier_block *nb) +void kho_remove_subtree(void *fdt) { - return blocking_notifier_chain_register(&kho_out.chain_head, nb); -} -EXPORT_SYMBOL_GPL(register_kho_notifier); + phys_addr_t target_phys = virt_to_phys(fdt); + void *root_fdt = kho_out.fdt; + int off; + int err; -int unregister_kho_notifier(struct notifier_block *nb) -{ - return blocking_notifier_chain_unregister(&kho_out.chain_head, nb); + guard(mutex)(&kho_out.lock); + + err = fdt_open_into(root_fdt, root_fdt, PAGE_SIZE); + if (err < 0) + return; + + for (off = fdt_first_subnode(root_fdt, 0); off >= 0; + off = fdt_next_subnode(root_fdt, off)) { + const u64 *val; + int len; + + val = fdt_getprop(root_fdt, off, PROP_SUB_FDT, &len); + if (!val || len != sizeof(phys_addr_t)) + continue; + + if ((phys_addr_t)*val == target_phys) { + fdt_del_node(root_fdt, off); + kho_debugfs_fdt_remove(&kho_out.dbg, fdt); + break; + } + } + + fdt_pack(root_fdt); } -EXPORT_SYMBOL_GPL(unregister_kho_notifier); +EXPORT_SYMBOL_GPL(kho_remove_subtree); /** * kho_preserve_folio - preserve a folio across kexec. @@ -762,7 +792,7 @@ int kho_preserve_folio(struct folio *folio) { const unsigned long pfn = folio_pfn(folio); const unsigned int order = folio_order(folio); - struct kho_mem_track *track = &kho_out.ser.track; + struct kho_mem_track *track = &kho_out.track; if (WARN_ON(kho_scratch_overlap(pfn << PAGE_SHIFT, PAGE_SIZE << order))) return -EINVAL; @@ -772,6 +802,24 @@ int kho_preserve_folio(struct folio *folio) EXPORT_SYMBOL_GPL(kho_preserve_folio); /** + * kho_unpreserve_folio - unpreserve a folio. + * @folio: folio to unpreserve. + * + * Instructs KHO to unpreserve a folio that was preserved by + * kho_preserve_folio() before. The provided @folio (pfn and order) + * must exactly match a previously preserved folio. + */ +void kho_unpreserve_folio(struct folio *folio) +{ + const unsigned long pfn = folio_pfn(folio); + const unsigned int order = folio_order(folio); + struct kho_mem_track *track = &kho_out.track; + + __kho_unpreserve_order(track, pfn, order); +} +EXPORT_SYMBOL_GPL(kho_unpreserve_folio); + +/** * kho_preserve_pages - preserve contiguous pages across kexec * @page: first page in the list. * @nr_pages: number of pages. @@ -783,7 +831,7 @@ EXPORT_SYMBOL_GPL(kho_preserve_folio); */ int kho_preserve_pages(struct page *page, unsigned int nr_pages) { - struct kho_mem_track *track = &kho_out.ser.track; + struct kho_mem_track *track = &kho_out.track; const unsigned long start_pfn = page_to_pfn(page); const unsigned long end_pfn = start_pfn + nr_pages; unsigned long pfn = start_pfn; @@ -815,6 +863,26 @@ int kho_preserve_pages(struct page *page, unsigned int nr_pages) } EXPORT_SYMBOL_GPL(kho_preserve_pages); +/** + * kho_unpreserve_pages - unpreserve contiguous pages. + * @page: first page in the list. + * @nr_pages: number of pages. + * + * Instructs KHO to unpreserve @nr_pages contiguous pages starting from @page. + * This must be called with the same @page and @nr_pages as the corresponding + * kho_preserve_pages() call. Unpreserving arbitrary sub-ranges of larger + * preserved blocks is not supported. + */ +void kho_unpreserve_pages(struct page *page, unsigned int nr_pages) +{ + struct kho_mem_track *track = &kho_out.track; + const unsigned long start_pfn = page_to_pfn(page); + const unsigned long end_pfn = start_pfn + nr_pages; + + __kho_unpreserve(track, start_pfn, end_pfn); +} +EXPORT_SYMBOL_GPL(kho_unpreserve_pages); + struct kho_vmalloc_hdr { DECLARE_KHOSER_PTR(next, struct kho_vmalloc_chunk *); }; @@ -885,7 +953,7 @@ err_free: static void kho_vmalloc_unpreserve_chunk(struct kho_vmalloc_chunk *chunk, unsigned short order) { - struct kho_mem_track *track = &kho_out.ser.track; + struct kho_mem_track *track = &kho_out.track; unsigned long pfn = PHYS_PFN(virt_to_phys(chunk)); __kho_unpreserve(track, pfn, pfn + 1); @@ -896,20 +964,6 @@ static void kho_vmalloc_unpreserve_chunk(struct kho_vmalloc_chunk *chunk, } } -static void kho_vmalloc_free_chunks(struct kho_vmalloc *kho_vmalloc) -{ - struct kho_vmalloc_chunk *chunk = KHOSER_LOAD_PTR(kho_vmalloc->first); - - while (chunk) { - struct kho_vmalloc_chunk *tmp = chunk; - - kho_vmalloc_unpreserve_chunk(chunk, kho_vmalloc->order); - - chunk = KHOSER_LOAD_PTR(chunk->hdr.next); - free_page((unsigned long)tmp); - } -} - /** * kho_preserve_vmalloc - preserve memory allocated with vmalloc() across kexec * @ptr: pointer to the area in vmalloc address space @@ -971,12 +1025,34 @@ int kho_preserve_vmalloc(void *ptr, struct kho_vmalloc *preservation) return 0; err_free: - kho_vmalloc_free_chunks(preservation); + kho_unpreserve_vmalloc(preservation); return err; } EXPORT_SYMBOL_GPL(kho_preserve_vmalloc); /** + * kho_unpreserve_vmalloc - unpreserve memory allocated with vmalloc() + * @preservation: preservation metadata returned by kho_preserve_vmalloc() + * + * Instructs KHO to unpreserve the area in vmalloc address space that was + * previously preserved with kho_preserve_vmalloc(). + */ +void kho_unpreserve_vmalloc(struct kho_vmalloc *preservation) +{ + struct kho_vmalloc_chunk *chunk = KHOSER_LOAD_PTR(preservation->first); + + while (chunk) { + struct kho_vmalloc_chunk *tmp = chunk; + + kho_vmalloc_unpreserve_chunk(chunk, preservation->order); + + chunk = KHOSER_LOAD_PTR(chunk->hdr.next); + free_page((unsigned long)tmp); + } +} +EXPORT_SYMBOL_GPL(kho_unpreserve_vmalloc); + +/** * kho_restore_vmalloc - recreates and populates an area in vmalloc address * space from the preserved memory. * @preservation: preservation metadata. @@ -1024,7 +1100,7 @@ void *kho_restore_vmalloc(const struct kho_vmalloc *preservation) goto err_free_pages_array; for (int j = 0; j < contig_pages; j++) - pages[idx++] = page; + pages[idx++] = page + j; phys += contig_pages * PAGE_SIZE; } @@ -1065,217 +1141,122 @@ err_free_pages_array: } EXPORT_SYMBOL_GPL(kho_restore_vmalloc); -/* Handling for debug/kho/out */ - -static struct dentry *debugfs_root; - -static int kho_out_update_debugfs_fdt(void) -{ - int err = 0; - struct fdt_debugfs *ff, *tmp; - - if (kho_out.finalized) { - err = kho_debugfs_fdt_add(&kho_out.ser.fdt_list, kho_out.dir, - "fdt", page_to_virt(kho_out.ser.fdt)); - } else { - list_for_each_entry_safe(ff, tmp, &kho_out.ser.fdt_list, list) { - debugfs_remove(ff->file); - list_del(&ff->list); - kfree(ff); - } - } - - return err; -} - -static int kho_abort(void) +/** + * kho_alloc_preserve - Allocate, zero, and preserve memory. + * @size: The number of bytes to allocate. + * + * Allocates a physically contiguous block of zeroed pages that is large + * enough to hold @size bytes. The allocated memory is then registered with + * KHO for preservation across a kexec. + * + * Note: The actual allocated size will be rounded up to the nearest + * power-of-two page boundary. + * + * @return A virtual pointer to the allocated and preserved memory on success, + * or an ERR_PTR() encoded error on failure. + */ +void *kho_alloc_preserve(size_t size) { - int err; - unsigned long order; - struct kho_mem_phys *physxa; + struct folio *folio; + int order, ret; - xa_for_each(&kho_out.ser.track.orders, order, physxa) { - struct kho_mem_phys_bits *bits; - unsigned long phys; + if (!size) + return ERR_PTR(-EINVAL); - xa_for_each(&physxa->phys_bits, phys, bits) - kfree(bits); + order = get_order(size); + if (order > MAX_PAGE_ORDER) + return ERR_PTR(-E2BIG); - xa_destroy(&physxa->phys_bits); - kfree(physxa); - } - xa_destroy(&kho_out.ser.track.orders); + folio = folio_alloc(GFP_KERNEL | __GFP_ZERO, order); + if (!folio) + return ERR_PTR(-ENOMEM); - if (kho_out.ser.preserved_mem_map) { - kho_mem_ser_free(kho_out.ser.preserved_mem_map); - kho_out.ser.preserved_mem_map = NULL; + ret = kho_preserve_folio(folio); + if (ret) { + folio_put(folio); + return ERR_PTR(ret); } - err = blocking_notifier_call_chain(&kho_out.chain_head, KEXEC_KHO_ABORT, - NULL); - err = notifier_to_errno(err); - - if (err) - pr_err("Failed to abort KHO finalization: %d\n", err); - - return err; + return folio_address(folio); } +EXPORT_SYMBOL_GPL(kho_alloc_preserve); -static int kho_finalize(void) +/** + * kho_unpreserve_free - Unpreserve and free memory. + * @mem: Pointer to the memory allocated by kho_alloc_preserve(). + * + * Unregisters the memory from KHO preservation and frees the underlying + * pages back to the system. This function should be called to clean up + * memory allocated with kho_alloc_preserve(). + */ +void kho_unpreserve_free(void *mem) { - int err = 0; - u64 *preserved_mem_map; - void *fdt = page_to_virt(kho_out.ser.fdt); - - err |= fdt_create(fdt, PAGE_SIZE); - err |= fdt_finish_reservemap(fdt); - err |= fdt_begin_node(fdt, ""); - err |= fdt_property_string(fdt, "compatible", KHO_FDT_COMPATIBLE); - /** - * Reserve the preserved-memory-map property in the root FDT, so - * that all property definitions will precede subnodes created by - * KHO callers. - */ - err |= fdt_property_placeholder(fdt, PROP_PRESERVED_MEMORY_MAP, - sizeof(*preserved_mem_map), - (void **)&preserved_mem_map); - if (err) - goto abort; - - err = kho_preserve_folio(page_folio(kho_out.ser.fdt)); - if (err) - goto abort; - - err = blocking_notifier_call_chain(&kho_out.chain_head, - KEXEC_KHO_FINALIZE, &kho_out.ser); - err = notifier_to_errno(err); - if (err) - goto abort; - - err = kho_mem_serialize(&kho_out.ser); - if (err) - goto abort; - - *preserved_mem_map = (u64)virt_to_phys(kho_out.ser.preserved_mem_map); - - err |= fdt_end_node(fdt); - err |= fdt_finish(fdt); + struct folio *folio; -abort: - if (err) { - pr_err("Failed to convert KHO state tree: %d\n", err); - kho_abort(); - } + if (!mem) + return; - return err; + folio = virt_to_folio(mem); + kho_unpreserve_folio(folio); + folio_put(folio); } +EXPORT_SYMBOL_GPL(kho_unpreserve_free); -static int kho_out_finalize_get(void *data, u64 *val) +/** + * kho_restore_free - Restore and free memory after kexec. + * @mem: Pointer to the memory (in the new kernel's address space) + * that was allocated by the old kernel. + * + * This function is intended to be called in the new kernel (post-kexec) + * to take ownership of and free a memory region that was preserved by the + * old kernel using kho_alloc_preserve(). + * + * It first restores the pages from KHO (using their physical address) + * and then frees the pages back to the new kernel's page allocator. + */ +void kho_restore_free(void *mem) { - mutex_lock(&kho_out.lock); - *val = kho_out.finalized; - mutex_unlock(&kho_out.lock); + struct folio *folio; - return 0; + if (!mem) + return; + + folio = kho_restore_folio(__pa(mem)); + if (!WARN_ON(!folio)) + folio_put(folio); } +EXPORT_SYMBOL_GPL(kho_restore_free); -static int kho_out_finalize_set(void *data, u64 _val) +int kho_finalize(void) { - int ret = 0; - bool val = !!_val; - - mutex_lock(&kho_out.lock); - - if (val == kho_out.finalized) { - if (kho_out.finalized) - ret = -EEXIST; - else - ret = -ENOENT; - goto unlock; - } + int ret; - if (val) - ret = kho_finalize(); - else - ret = kho_abort(); + if (!kho_enable) + return -EOPNOTSUPP; + guard(mutex)(&kho_out.lock); + ret = kho_mem_serialize(&kho_out); if (ret) - goto unlock; - - kho_out.finalized = val; - ret = kho_out_update_debugfs_fdt(); - -unlock: - mutex_unlock(&kho_out.lock); - return ret; -} - -DEFINE_DEBUGFS_ATTRIBUTE(fops_kho_out_finalize, kho_out_finalize_get, - kho_out_finalize_set, "%llu\n"); + return ret; -static int scratch_phys_show(struct seq_file *m, void *v) -{ - for (int i = 0; i < kho_scratch_cnt; i++) - seq_printf(m, "0x%llx\n", kho_scratch[i].addr); + kho_out.finalized = true; return 0; } -DEFINE_SHOW_ATTRIBUTE(scratch_phys); -static int scratch_len_show(struct seq_file *m, void *v) +bool kho_finalized(void) { - for (int i = 0; i < kho_scratch_cnt; i++) - seq_printf(m, "0x%llx\n", kho_scratch[i].size); - - return 0; -} -DEFINE_SHOW_ATTRIBUTE(scratch_len); - -static __init int kho_out_debugfs_init(void) -{ - struct dentry *dir, *f, *sub_fdt_dir; - - dir = debugfs_create_dir("out", debugfs_root); - if (IS_ERR(dir)) - return -ENOMEM; - - sub_fdt_dir = debugfs_create_dir("sub_fdts", dir); - if (IS_ERR(sub_fdt_dir)) - goto err_rmdir; - - f = debugfs_create_file("scratch_phys", 0400, dir, NULL, - &scratch_phys_fops); - if (IS_ERR(f)) - goto err_rmdir; - - f = debugfs_create_file("scratch_len", 0400, dir, NULL, - &scratch_len_fops); - if (IS_ERR(f)) - goto err_rmdir; - - f = debugfs_create_file("finalize", 0600, dir, NULL, - &fops_kho_out_finalize); - if (IS_ERR(f)) - goto err_rmdir; - - kho_out.dir = dir; - kho_out.ser.sub_fdt_dir = sub_fdt_dir; - return 0; - -err_rmdir: - debugfs_remove_recursive(dir); - return -ENOENT; + guard(mutex)(&kho_out.lock); + return kho_out.finalized; } struct kho_in { - struct dentry *dir; phys_addr_t fdt_phys; phys_addr_t scratch_phys; - struct list_head fdt_list; + struct kho_debugfs dbg; }; static struct kho_in kho_in = { - .fdt_list = LIST_HEAD_INIT(kho_in.fdt_list), }; static const void *kho_get_fdt(void) @@ -1339,91 +1320,52 @@ int kho_retrieve_subtree(const char *name, phys_addr_t *phys) } EXPORT_SYMBOL_GPL(kho_retrieve_subtree); -/* Handling for debugfs/kho/in */ - -static __init int kho_in_debugfs_init(const void *fdt) +static __init int kho_out_fdt_setup(void) { - struct dentry *sub_fdt_dir; - int err, child; - - kho_in.dir = debugfs_create_dir("in", debugfs_root); - if (IS_ERR(kho_in.dir)) - return PTR_ERR(kho_in.dir); - - sub_fdt_dir = debugfs_create_dir("sub_fdts", kho_in.dir); - if (IS_ERR(sub_fdt_dir)) { - err = PTR_ERR(sub_fdt_dir); - goto err_rmdir; - } - - err = kho_debugfs_fdt_add(&kho_in.fdt_list, kho_in.dir, "fdt", fdt); - if (err) - goto err_rmdir; - - fdt_for_each_subnode(child, fdt, 0) { - int len = 0; - const char *name = fdt_get_name(fdt, child, NULL); - const u64 *fdt_phys; - - fdt_phys = fdt_getprop(fdt, child, "fdt", &len); - if (!fdt_phys) - continue; - if (len != sizeof(*fdt_phys)) { - pr_warn("node `%s`'s prop `fdt` has invalid length: %d\n", - name, len); - continue; - } - err = kho_debugfs_fdt_add(&kho_in.fdt_list, sub_fdt_dir, name, - phys_to_virt(*fdt_phys)); - if (err) { - pr_warn("failed to add fdt `%s` to debugfs: %d\n", name, - err); - continue; - } - } + void *root = kho_out.fdt; + u64 empty_mem_map = 0; + int err; - return 0; + err = fdt_create(root, PAGE_SIZE); + err |= fdt_finish_reservemap(root); + err |= fdt_begin_node(root, ""); + err |= fdt_property_string(root, "compatible", KHO_FDT_COMPATIBLE); + err |= fdt_property(root, PROP_PRESERVED_MEMORY_MAP, &empty_mem_map, + sizeof(empty_mem_map)); + err |= fdt_end_node(root); + err |= fdt_finish(root); -err_rmdir: - debugfs_remove_recursive(kho_in.dir); return err; } static __init int kho_init(void) { - int err = 0; const void *fdt = kho_get_fdt(); + int err = 0; if (!kho_enable) return 0; - kho_out.ser.fdt = alloc_page(GFP_KERNEL); - if (!kho_out.ser.fdt) { - err = -ENOMEM; + kho_out.fdt = kho_alloc_preserve(PAGE_SIZE); + if (IS_ERR(kho_out.fdt)) { + err = PTR_ERR(kho_out.fdt); goto err_free_scratch; } - debugfs_root = debugfs_create_dir("kho", NULL); - if (IS_ERR(debugfs_root)) { - err = -ENOENT; + err = kho_debugfs_init(); + if (err) goto err_free_fdt; - } - err = kho_out_debugfs_init(); + err = kho_out_debugfs_init(&kho_out.dbg); if (err) goto err_free_fdt; - if (fdt) { - err = kho_in_debugfs_init(fdt); - /* - * Failure to create /sys/kernel/debug/kho/in does not prevent - * reviving state from KHO and setting up KHO for the next - * kexec. - */ - if (err) - pr_err("failed exposing handover FDT in debugfs: %d\n", - err); + err = kho_out_fdt_setup(); + if (err) + goto err_free_fdt; + if (fdt) { + kho_in_debugfs_init(&kho_in.dbg, fdt); return 0; } @@ -1432,17 +1374,29 @@ static __init int kho_init(void) unsigned long count = kho_scratch[i].size >> PAGE_SHIFT; unsigned long pfn; + /* + * When debug_pagealloc is enabled, __free_pages() clears the + * corresponding PRESENT bit in the kernel page table. + * Subsequent kmemleak scans of these pages cause the + * non-PRESENT page faults. + * Mark scratch areas with kmemleak_ignore_phys() to exclude + * them from kmemleak scanning. + */ + kmemleak_ignore_phys(kho_scratch[i].addr); for (pfn = base_pfn; pfn < base_pfn + count; pfn += pageblock_nr_pages) init_cma_reserved_pageblock(pfn_to_page(pfn)); } + WARN_ON_ONCE(kho_debugfs_fdt_add(&kho_out.dbg, "fdt", + kho_out.fdt, true)); + return 0; err_free_fdt: - put_page(kho_out.ser.fdt); - kho_out.ser.fdt = NULL; + kho_unpreserve_free(kho_out.fdt); err_free_scratch: + kho_out.fdt = NULL; for (int i = 0; i < kho_scratch_cnt; i++) { void *start = __va(kho_scratch[i].addr); void *end = start + kho_scratch[i].size; @@ -1452,7 +1406,7 @@ err_free_scratch: kho_enable = false; return err; } -late_initcall(kho_init); +fs_initcall(kho_init); static void __init kho_release_scratch(void) { @@ -1480,16 +1434,12 @@ static void __init kho_release_scratch(void) void __init kho_memory_init(void) { - struct folio *folio; - if (kho_in.scratch_phys) { kho_scratch = phys_to_virt(kho_in.scratch_phys); kho_release_scratch(); - kho_mem_deserialize(kho_get_fdt()); - folio = kho_restore_folio(kho_in.fdt_phys); - if (!folio) - pr_warn("failed to restore folio for KHO fdt\n"); + if (!kho_mem_deserialize(kho_get_fdt())) + kho_in.fdt_phys = 0; } else { kho_reserve_scratch(); } @@ -1545,8 +1495,8 @@ void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len, memblock_add(area->addr, size); err = memblock_mark_kho_scratch(area->addr, size); if (WARN_ON(err)) { - pr_warn("failed to mark the scratch region 0x%pa+0x%pa: %d", - &area->addr, &size, err); + pr_warn("failed to mark the scratch region 0x%pa+0x%pa: %pe", + &area->addr, &size, ERR_PTR(err)); goto out; } pr_debug("Marked 0x%pa+0x%pa as scratch", &area->addr, &size); @@ -1566,7 +1516,7 @@ void __init kho_populate(phys_addr_t fdt_phys, u64 fdt_len, kho_in.fdt_phys = fdt_phys; kho_in.scratch_phys = scratch_phys; kho_scratch_cnt = scratch_cnt; - pr_info("found kexec handover data. Will skip init for some devices\n"); + pr_info("found kexec handover data.\n"); out: if (fdt) @@ -1585,10 +1535,10 @@ int kho_fill_kimage(struct kimage *image) int err = 0; struct kexec_buf scratch; - if (!kho_out.finalized) + if (!kho_enable) return 0; - image->kho.fdt = page_to_phys(kho_out.ser.fdt); + image->kho.fdt = virt_to_phys(kho_out.fdt); scratch_size = sizeof(*kho_scratch) * kho_scratch_cnt; scratch = (struct kexec_buf){ diff --git a/kernel/kexec_handover_debug.c b/kernel/liveupdate/kexec_handover_debug.c index 6efb696f5426..6efb696f5426 100644 --- a/kernel/kexec_handover_debug.c +++ b/kernel/liveupdate/kexec_handover_debug.c diff --git a/kernel/liveupdate/kexec_handover_debugfs.c b/kernel/liveupdate/kexec_handover_debugfs.c new file mode 100644 index 000000000000..2abbf62ba942 --- /dev/null +++ b/kernel/liveupdate/kexec_handover_debugfs.c @@ -0,0 +1,221 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * kexec_handover_debugfs.c - kexec handover debugfs interfaces + * Copyright (C) 2023 Alexander Graf <graf@amazon.com> + * Copyright (C) 2025 Microsoft Corporation, Mike Rapoport <rppt@kernel.org> + * Copyright (C) 2025 Google LLC, Changyuan Lyu <changyuanl@google.com> + * Copyright (C) 2025 Google LLC, Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +#define pr_fmt(fmt) "KHO: " fmt + +#include <linux/init.h> +#include <linux/io.h> +#include <linux/libfdt.h> +#include <linux/mm.h> +#include "kexec_handover_internal.h" + +static struct dentry *debugfs_root; + +struct fdt_debugfs { + struct list_head list; + struct debugfs_blob_wrapper wrapper; + struct dentry *file; +}; + +static int __kho_debugfs_fdt_add(struct list_head *list, struct dentry *dir, + const char *name, const void *fdt) +{ + struct fdt_debugfs *f; + struct dentry *file; + + f = kmalloc(sizeof(*f), GFP_KERNEL); + if (!f) + return -ENOMEM; + + f->wrapper.data = (void *)fdt; + f->wrapper.size = fdt_totalsize(fdt); + + file = debugfs_create_blob(name, 0400, dir, &f->wrapper); + if (IS_ERR(file)) { + kfree(f); + return PTR_ERR(file); + } + + f->file = file; + list_add(&f->list, list); + + return 0; +} + +int kho_debugfs_fdt_add(struct kho_debugfs *dbg, const char *name, + const void *fdt, bool root) +{ + struct dentry *dir; + + if (root) + dir = dbg->dir; + else + dir = dbg->sub_fdt_dir; + + return __kho_debugfs_fdt_add(&dbg->fdt_list, dir, name, fdt); +} + +void kho_debugfs_fdt_remove(struct kho_debugfs *dbg, void *fdt) +{ + struct fdt_debugfs *ff; + + list_for_each_entry(ff, &dbg->fdt_list, list) { + if (ff->wrapper.data == fdt) { + debugfs_remove(ff->file); + list_del(&ff->list); + kfree(ff); + break; + } + } +} + +static int kho_out_finalize_get(void *data, u64 *val) +{ + *val = kho_finalized(); + + return 0; +} + +static int kho_out_finalize_set(void *data, u64 val) +{ + if (val) + return kho_finalize(); + else + return -EINVAL; +} + +DEFINE_DEBUGFS_ATTRIBUTE(kho_out_finalize_fops, kho_out_finalize_get, + kho_out_finalize_set, "%llu\n"); + +static int scratch_phys_show(struct seq_file *m, void *v) +{ + for (int i = 0; i < kho_scratch_cnt; i++) + seq_printf(m, "0x%llx\n", kho_scratch[i].addr); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(scratch_phys); + +static int scratch_len_show(struct seq_file *m, void *v) +{ + for (int i = 0; i < kho_scratch_cnt; i++) + seq_printf(m, "0x%llx\n", kho_scratch[i].size); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(scratch_len); + +__init void kho_in_debugfs_init(struct kho_debugfs *dbg, const void *fdt) +{ + struct dentry *dir, *sub_fdt_dir; + int err, child; + + INIT_LIST_HEAD(&dbg->fdt_list); + + dir = debugfs_create_dir("in", debugfs_root); + if (IS_ERR(dir)) { + err = PTR_ERR(dir); + goto err_out; + } + + sub_fdt_dir = debugfs_create_dir("sub_fdts", dir); + if (IS_ERR(sub_fdt_dir)) { + err = PTR_ERR(sub_fdt_dir); + goto err_rmdir; + } + + err = __kho_debugfs_fdt_add(&dbg->fdt_list, dir, "fdt", fdt); + if (err) + goto err_rmdir; + + fdt_for_each_subnode(child, fdt, 0) { + int len = 0; + const char *name = fdt_get_name(fdt, child, NULL); + const u64 *fdt_phys; + + fdt_phys = fdt_getprop(fdt, child, "fdt", &len); + if (!fdt_phys) + continue; + if (len != sizeof(*fdt_phys)) { + pr_warn("node %s prop fdt has invalid length: %d\n", + name, len); + continue; + } + err = __kho_debugfs_fdt_add(&dbg->fdt_list, sub_fdt_dir, name, + phys_to_virt(*fdt_phys)); + if (err) { + pr_warn("failed to add fdt %s to debugfs: %pe\n", name, + ERR_PTR(err)); + continue; + } + } + + dbg->dir = dir; + dbg->sub_fdt_dir = sub_fdt_dir; + + return; +err_rmdir: + debugfs_remove_recursive(dir); +err_out: + /* + * Failure to create /sys/kernel/debug/kho/in does not prevent + * reviving state from KHO and setting up KHO for the next + * kexec. + */ + if (err) { + pr_err("failed exposing handover FDT in debugfs: %pe\n", + ERR_PTR(err)); + } +} + +__init int kho_out_debugfs_init(struct kho_debugfs *dbg) +{ + struct dentry *dir, *f, *sub_fdt_dir; + + INIT_LIST_HEAD(&dbg->fdt_list); + + dir = debugfs_create_dir("out", debugfs_root); + if (IS_ERR(dir)) + return -ENOMEM; + + sub_fdt_dir = debugfs_create_dir("sub_fdts", dir); + if (IS_ERR(sub_fdt_dir)) + goto err_rmdir; + + f = debugfs_create_file("scratch_phys", 0400, dir, NULL, + &scratch_phys_fops); + if (IS_ERR(f)) + goto err_rmdir; + + f = debugfs_create_file("scratch_len", 0400, dir, NULL, + &scratch_len_fops); + if (IS_ERR(f)) + goto err_rmdir; + + f = debugfs_create_file("finalize", 0600, dir, NULL, + &kho_out_finalize_fops); + if (IS_ERR(f)) + goto err_rmdir; + + dbg->dir = dir; + dbg->sub_fdt_dir = sub_fdt_dir; + return 0; + +err_rmdir: + debugfs_remove_recursive(dir); + return -ENOENT; +} + +__init int kho_debugfs_init(void) +{ + debugfs_root = debugfs_create_dir("kho", NULL); + if (IS_ERR(debugfs_root)) + return -ENOENT; + return 0; +} diff --git a/kernel/liveupdate/kexec_handover_internal.h b/kernel/liveupdate/kexec_handover_internal.h new file mode 100644 index 000000000000..0202c85ad14f --- /dev/null +++ b/kernel/liveupdate/kexec_handover_internal.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef LINUX_KEXEC_HANDOVER_INTERNAL_H +#define LINUX_KEXEC_HANDOVER_INTERNAL_H + +#include <linux/kexec_handover.h> +#include <linux/list.h> +#include <linux/types.h> + +#ifdef CONFIG_KEXEC_HANDOVER_DEBUGFS +#include <linux/debugfs.h> + +struct kho_debugfs { + struct dentry *dir; + struct dentry *sub_fdt_dir; + struct list_head fdt_list; +}; + +#else +struct kho_debugfs {}; +#endif + +extern struct kho_scratch *kho_scratch; +extern unsigned int kho_scratch_cnt; + +bool kho_finalized(void); +int kho_finalize(void); + +#ifdef CONFIG_KEXEC_HANDOVER_DEBUGFS +int kho_debugfs_init(void); +void kho_in_debugfs_init(struct kho_debugfs *dbg, const void *fdt); +int kho_out_debugfs_init(struct kho_debugfs *dbg); +int kho_debugfs_fdt_add(struct kho_debugfs *dbg, const char *name, + const void *fdt, bool root); +void kho_debugfs_fdt_remove(struct kho_debugfs *dbg, void *fdt); +#else +static inline int kho_debugfs_init(void) { return 0; } +static inline void kho_in_debugfs_init(struct kho_debugfs *dbg, + const void *fdt) { } +static inline int kho_out_debugfs_init(struct kho_debugfs *dbg) { return 0; } +static inline int kho_debugfs_fdt_add(struct kho_debugfs *dbg, const char *name, + const void *fdt, bool root) { return 0; } +static inline void kho_debugfs_fdt_remove(struct kho_debugfs *dbg, + void *fdt) { } +#endif /* CONFIG_KEXEC_HANDOVER_DEBUGFS */ + +#ifdef CONFIG_KEXEC_HANDOVER_DEBUG +bool kho_scratch_overlap(phys_addr_t phys, size_t size); +#else +static inline bool kho_scratch_overlap(phys_addr_t phys, size_t size) +{ + return false; +} +#endif /* CONFIG_KEXEC_HANDOVER_DEBUG */ + +#endif /* LINUX_KEXEC_HANDOVER_INTERNAL_H */ diff --git a/kernel/liveupdate/luo_core.c b/kernel/liveupdate/luo_core.c new file mode 100644 index 000000000000..f7ecaf7740d1 --- /dev/null +++ b/kernel/liveupdate/luo_core.c @@ -0,0 +1,450 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +/** + * DOC: Live Update Orchestrator (LUO) + * + * Live Update is a specialized, kexec-based reboot process that allows a + * running kernel to be updated from one version to another while preserving + * the state of selected resources and keeping designated hardware devices + * operational. For these devices, DMA activity may continue throughout the + * kernel transition. + * + * While the primary use case driving this work is supporting live updates of + * the Linux kernel when it is used as a hypervisor in cloud environments, the + * LUO framework itself is designed to be workload-agnostic. Live Update + * facilitates a full kernel version upgrade for any type of system. + * + * For example, a non-hypervisor system running an in-memory cache like + * memcached with many gigabytes of data can use LUO. The userspace service + * can place its cache into a memfd, have its state preserved by LUO, and + * restore it immediately after the kernel kexec. + * + * Whether the system is running virtual machines, containers, a + * high-performance database, or networking services, LUO's primary goal is to + * enable a full kernel update by preserving critical userspace state and + * keeping essential devices operational. + * + * The core of LUO is a mechanism that tracks the progress of a live update, + * along with a callback API that allows other kernel subsystems to participate + * in the process. Example subsystems that can hook into LUO include: kvm, + * iommu, interrupts, vfio, participating filesystems, and memory management. + * + * LUO uses Kexec Handover to transfer memory state from the current kernel to + * the next kernel. For more details see + * Documentation/core-api/kho/concepts.rst. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/atomic.h> +#include <linux/errno.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/init.h> +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/kexec_handover.h> +#include <linux/kho/abi/luo.h> +#include <linux/kobject.h> +#include <linux/libfdt.h> +#include <linux/liveupdate.h> +#include <linux/miscdevice.h> +#include <linux/mm.h> +#include <linux/sizes.h> +#include <linux/string.h> +#include <linux/unaligned.h> + +#include "kexec_handover_internal.h" +#include "luo_internal.h" + +static struct { + bool enabled; + void *fdt_out; + void *fdt_in; + u64 liveupdate_num; +} luo_global; + +static int __init early_liveupdate_param(char *buf) +{ + return kstrtobool(buf, &luo_global.enabled); +} +early_param("liveupdate", early_liveupdate_param); + +static int __init luo_early_startup(void) +{ + phys_addr_t fdt_phys; + int err, ln_size; + const void *ptr; + + if (!kho_is_enabled()) { + if (liveupdate_enabled()) + pr_warn("Disabling liveupdate because KHO is disabled\n"); + luo_global.enabled = false; + return 0; + } + + /* Retrieve LUO subtree, and verify its format. */ + err = kho_retrieve_subtree(LUO_FDT_KHO_ENTRY_NAME, &fdt_phys); + if (err) { + if (err != -ENOENT) { + pr_err("failed to retrieve FDT '%s' from KHO: %pe\n", + LUO_FDT_KHO_ENTRY_NAME, ERR_PTR(err)); + return err; + } + + return 0; + } + + luo_global.fdt_in = phys_to_virt(fdt_phys); + err = fdt_node_check_compatible(luo_global.fdt_in, 0, + LUO_FDT_COMPATIBLE); + if (err) { + pr_err("FDT '%s' is incompatible with '%s' [%d]\n", + LUO_FDT_KHO_ENTRY_NAME, LUO_FDT_COMPATIBLE, err); + + return -EINVAL; + } + + ln_size = 0; + ptr = fdt_getprop(luo_global.fdt_in, 0, LUO_FDT_LIVEUPDATE_NUM, + &ln_size); + if (!ptr || ln_size != sizeof(luo_global.liveupdate_num)) { + pr_err("Unable to get live update number '%s' [%d]\n", + LUO_FDT_LIVEUPDATE_NUM, ln_size); + + return -EINVAL; + } + + luo_global.liveupdate_num = get_unaligned((u64 *)ptr); + pr_info("Retrieved live update data, liveupdate number: %lld\n", + luo_global.liveupdate_num); + + err = luo_session_setup_incoming(luo_global.fdt_in); + if (err) + return err; + + return 0; +} + +static int __init liveupdate_early_init(void) +{ + int err; + + err = luo_early_startup(); + if (err) { + luo_global.enabled = false; + luo_restore_fail("The incoming tree failed to initialize properly [%pe], disabling live update\n", + ERR_PTR(err)); + } + + return err; +} +early_initcall(liveupdate_early_init); + +/* Called during boot to create outgoing LUO fdt tree */ +static int __init luo_fdt_setup(void) +{ + const u64 ln = luo_global.liveupdate_num + 1; + void *fdt_out; + int err; + + fdt_out = kho_alloc_preserve(LUO_FDT_SIZE); + if (IS_ERR(fdt_out)) { + pr_err("failed to allocate/preserve FDT memory\n"); + return PTR_ERR(fdt_out); + } + + err = fdt_create(fdt_out, LUO_FDT_SIZE); + err |= fdt_finish_reservemap(fdt_out); + err |= fdt_begin_node(fdt_out, ""); + err |= fdt_property_string(fdt_out, "compatible", LUO_FDT_COMPATIBLE); + err |= fdt_property(fdt_out, LUO_FDT_LIVEUPDATE_NUM, &ln, sizeof(ln)); + err |= luo_session_setup_outgoing(fdt_out); + err |= fdt_end_node(fdt_out); + err |= fdt_finish(fdt_out); + if (err) + goto exit_free; + + err = kho_add_subtree(LUO_FDT_KHO_ENTRY_NAME, fdt_out); + if (err) + goto exit_free; + luo_global.fdt_out = fdt_out; + + return 0; + +exit_free: + kho_unpreserve_free(fdt_out); + pr_err("failed to prepare LUO FDT: %d\n", err); + + return err; +} + +/* + * late initcall because it initializes the outgoing tree that is needed only + * once userspace starts using /dev/liveupdate. + */ +static int __init luo_late_startup(void) +{ + int err; + + if (!liveupdate_enabled()) + return 0; + + err = luo_fdt_setup(); + if (err) + luo_global.enabled = false; + + return err; +} +late_initcall(luo_late_startup); + +/* Public Functions */ + +/** + * liveupdate_reboot() - Kernel reboot notifier for live update final + * serialization. + * + * This function is invoked directly from the reboot() syscall pathway + * if kexec is in progress. + * + * If any callback fails, this function aborts KHO, undoes the freeze() + * callbacks, and returns an error. + */ +int liveupdate_reboot(void) +{ + int err; + + if (!liveupdate_enabled()) + return 0; + + err = luo_session_serialize(); + if (err) + return err; + + err = kho_finalize(); + if (err) { + pr_err("kho_finalize failed %d\n", err); + /* + * kho_finalize() may return libfdt errors, to aboid passing to + * userspace unknown errors, change this to EAGAIN. + */ + err = -EAGAIN; + } + + return err; +} + +/** + * liveupdate_enabled - Check if the live update feature is enabled. + * + * This function returns the state of the live update feature flag, which + * can be controlled via the ``liveupdate`` kernel command-line parameter. + * + * @return true if live update is enabled, false otherwise. + */ +bool liveupdate_enabled(void) +{ + return luo_global.enabled; +} + +/** + * DOC: LUO ioctl Interface + * + * The IOCTL user-space control interface for the LUO subsystem. + * It registers a character device, typically found at ``/dev/liveupdate``, + * which allows a userspace agent to manage the LUO state machine and its + * associated resources, such as preservable file descriptors. + * + * To ensure that the state machine is controlled by a single entity, access + * to this device is exclusive: only one process is permitted to have + * ``/dev/liveupdate`` open at any given time. Subsequent open attempts will + * fail with -EBUSY until the first process closes its file descriptor. + * This singleton model simplifies state management by preventing conflicting + * commands from multiple userspace agents. + */ + +struct luo_device_state { + struct miscdevice miscdev; + atomic_t in_use; +}; + +static int luo_ioctl_create_session(struct luo_ucmd *ucmd) +{ + struct liveupdate_ioctl_create_session *argp = ucmd->cmd; + struct file *file; + int err; + + argp->fd = get_unused_fd_flags(O_CLOEXEC); + if (argp->fd < 0) + return argp->fd; + + err = luo_session_create(argp->name, &file); + if (err) + goto err_put_fd; + + err = luo_ucmd_respond(ucmd, sizeof(*argp)); + if (err) + goto err_put_file; + + fd_install(argp->fd, file); + + return 0; + +err_put_file: + fput(file); +err_put_fd: + put_unused_fd(argp->fd); + + return err; +} + +static int luo_ioctl_retrieve_session(struct luo_ucmd *ucmd) +{ + struct liveupdate_ioctl_retrieve_session *argp = ucmd->cmd; + struct file *file; + int err; + + argp->fd = get_unused_fd_flags(O_CLOEXEC); + if (argp->fd < 0) + return argp->fd; + + err = luo_session_retrieve(argp->name, &file); + if (err < 0) + goto err_put_fd; + + err = luo_ucmd_respond(ucmd, sizeof(*argp)); + if (err) + goto err_put_file; + + fd_install(argp->fd, file); + + return 0; + +err_put_file: + fput(file); +err_put_fd: + put_unused_fd(argp->fd); + + return err; +} + +static int luo_open(struct inode *inodep, struct file *filep) +{ + struct luo_device_state *ldev = container_of(filep->private_data, + struct luo_device_state, + miscdev); + + if (atomic_cmpxchg(&ldev->in_use, 0, 1)) + return -EBUSY; + + /* Always return -EIO to user if deserialization fail */ + if (luo_session_deserialize()) { + atomic_set(&ldev->in_use, 0); + return -EIO; + } + + return 0; +} + +static int luo_release(struct inode *inodep, struct file *filep) +{ + struct luo_device_state *ldev = container_of(filep->private_data, + struct luo_device_state, + miscdev); + atomic_set(&ldev->in_use, 0); + + return 0; +} + +union ucmd_buffer { + struct liveupdate_ioctl_create_session create; + struct liveupdate_ioctl_retrieve_session retrieve; +}; + +struct luo_ioctl_op { + unsigned int size; + unsigned int min_size; + unsigned int ioctl_num; + int (*execute)(struct luo_ucmd *ucmd); +}; + +#define IOCTL_OP(_ioctl, _fn, _struct, _last) \ + [_IOC_NR(_ioctl) - LIVEUPDATE_CMD_BASE] = { \ + .size = sizeof(_struct) + \ + BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \ + sizeof(_struct)), \ + .min_size = offsetofend(_struct, _last), \ + .ioctl_num = _ioctl, \ + .execute = _fn, \ + } + +static const struct luo_ioctl_op luo_ioctl_ops[] = { + IOCTL_OP(LIVEUPDATE_IOCTL_CREATE_SESSION, luo_ioctl_create_session, + struct liveupdate_ioctl_create_session, name), + IOCTL_OP(LIVEUPDATE_IOCTL_RETRIEVE_SESSION, luo_ioctl_retrieve_session, + struct liveupdate_ioctl_retrieve_session, name), +}; + +static long luo_ioctl(struct file *filep, unsigned int cmd, unsigned long arg) +{ + const struct luo_ioctl_op *op; + struct luo_ucmd ucmd = {}; + union ucmd_buffer buf; + unsigned int nr; + int err; + + nr = _IOC_NR(cmd); + if (nr < LIVEUPDATE_CMD_BASE || + (nr - LIVEUPDATE_CMD_BASE) >= ARRAY_SIZE(luo_ioctl_ops)) { + return -EINVAL; + } + + ucmd.ubuffer = (void __user *)arg; + err = get_user(ucmd.user_size, (u32 __user *)ucmd.ubuffer); + if (err) + return err; + + op = &luo_ioctl_ops[nr - LIVEUPDATE_CMD_BASE]; + if (op->ioctl_num != cmd) + return -ENOIOCTLCMD; + if (ucmd.user_size < op->min_size) + return -EINVAL; + + ucmd.cmd = &buf; + err = copy_struct_from_user(ucmd.cmd, op->size, ucmd.ubuffer, + ucmd.user_size); + if (err) + return err; + + return op->execute(&ucmd); +} + +static const struct file_operations luo_fops = { + .owner = THIS_MODULE, + .open = luo_open, + .release = luo_release, + .unlocked_ioctl = luo_ioctl, +}; + +static struct luo_device_state luo_dev = { + .miscdev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "liveupdate", + .fops = &luo_fops, + }, + .in_use = ATOMIC_INIT(0), +}; + +static int __init liveupdate_ioctl_init(void) +{ + if (!liveupdate_enabled()) + return 0; + + return misc_register(&luo_dev.miscdev); +} +late_initcall(liveupdate_ioctl_init); diff --git a/kernel/liveupdate/luo_file.c b/kernel/liveupdate/luo_file.c new file mode 100644 index 000000000000..ddff87917b21 --- /dev/null +++ b/kernel/liveupdate/luo_file.c @@ -0,0 +1,889 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +/** + * DOC: LUO File Descriptors + * + * LUO provides the infrastructure to preserve specific, stateful file + * descriptors across a kexec-based live update. The primary goal is to allow + * workloads, such as virtual machines using vfio, memfd, or iommufd, to + * retain access to their essential resources without interruption. + * + * The framework is built around a callback-based handler model and a well- + * defined lifecycle for each preserved file. + * + * Handler Registration: + * Kernel modules responsible for a specific file type (e.g., memfd, vfio) + * register a &struct liveupdate_file_handler. This handler provides a set of + * callbacks that LUO invokes at different stages of the update process, most + * notably: + * + * - can_preserve(): A lightweight check to determine if the handler is + * compatible with a given 'struct file'. + * - preserve(): The heavyweight operation that saves the file's state and + * returns an opaque u64 handle. This is typically performed while the + * workload is still active to minimize the downtime during the + * actual reboot transition. + * - unpreserve(): Cleans up any resources allocated by .preserve(), called + * if the preservation process is aborted before the reboot (i.e. session is + * closed). + * - freeze(): A final pre-reboot opportunity to prepare the state for kexec. + * We are already in reboot syscall, and therefore userspace cannot mutate + * the file anymore. + * - unfreeze(): Undoes the actions of .freeze(), called if the live update + * is aborted after the freeze phase. + * - retrieve(): Reconstructs the file in the new kernel from the preserved + * handle. + * - finish(): Performs final check and cleanup in the new kernel. After + * succesul finish call, LUO gives up ownership to this file. + * + * File Preservation Lifecycle happy path: + * + * 1. Preserve (Normal Operation): A userspace agent preserves files one by one + * via an ioctl. For each file, luo_preserve_file() finds a compatible + * handler, calls its .preserve() operation, and creates an internal &struct + * luo_file to track the live state. + * + * 2. Freeze (Pre-Reboot): Just before the kexec, luo_file_freeze() is called. + * It iterates through all preserved files, calls their respective .freeze() + * operation, and serializes their final metadata (compatible string, token, + * and data handle) into a contiguous memory block for KHO. + * + * 3. Deserialize: After kexec, luo_file_deserialize() runs when session gets + * deserialized (which is when /dev/liveupdate is first opened). It reads the + * serialized data from the KHO memory region and reconstructs the in-memory + * list of &struct luo_file instances for the new kernel, linking them to + * their corresponding handlers. + * + * 4. Retrieve (New Kernel - Userspace Ready): The userspace agent can now + * restore file descriptors by providing a token. luo_retrieve_file() + * searches for the matching token, calls the handler's .retrieve() op to + * re-create the 'struct file', and returns a new FD. Files can be + * retrieved in ANY order. + * + * 5. Finish (New Kernel - Cleanup): Once a session retrival is complete, + * luo_file_finish() is called. It iterates through all files, invokes their + * .finish() operations for final cleanup, and releases all associated kernel + * resources. + * + * File Preservation Lifecycle unhappy paths: + * + * 1. Abort Before Reboot: If the userspace agent aborts the live update + * process before calling reboot (e.g., by closing the session file + * descriptor), the session's release handler calls + * luo_file_unpreserve_files(). This invokes the .unpreserve() callback on + * all preserved files, ensuring all allocated resources are cleaned up and + * returning the system to a clean state. + * + * 2. Freeze Failure: During the reboot() syscall, if any handler's .freeze() + * op fails, the .unfreeze() op is invoked on all previously *successful* + * freezes to roll back their state. The reboot() syscall then returns an + * error to userspace, canceling the live update. + * + * 3. Finish Failure: In the new kernel, if a handler's .finish() op fails, + * the luo_file_finish() operation is aborted. LUO retains ownership of + * all files within that session, including those that were not yet + * processed. The userspace agent can attempt to call the finish operation + * again later. If the issue cannot be resolved, these resources will be held + * by LUO until the next live update cycle, at which point they will be + * discarded. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/cleanup.h> +#include <linux/compiler.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/io.h> +#include <linux/kexec_handover.h> +#include <linux/kho/abi/luo.h> +#include <linux/liveupdate.h> +#include <linux/module.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/string.h> +#include "luo_internal.h" + +static LIST_HEAD(luo_file_handler_list); + +/* 2 4K pages, give space for 128 files per file_set */ +#define LUO_FILE_PGCNT 2ul +#define LUO_FILE_MAX \ + ((LUO_FILE_PGCNT << PAGE_SHIFT) / sizeof(struct luo_file_ser)) + +/** + * struct luo_file - Represents a single preserved file instance. + * @fh: Pointer to the &struct liveupdate_file_handler that manages + * this type of file. + * @file: Pointer to the kernel's &struct file that is being preserved. + * This is NULL in the new kernel until the file is successfully + * retrieved. + * @serialized_data: The opaque u64 handle to the serialized state of the file. + * This handle is passed back to the handler's .freeze(), + * .retrieve(), and .finish() callbacks, allowing it to track + * and update its serialized state across phases. + * @private_data: Pointer to the private data for the file used to hold runtime + * state that is not preserved. Set by the handler's .preserve() + * callback, and must be freed in the handler's .unpreserve() + * callback. + * @retrieved: A flag indicating whether a user/kernel in the new kernel has + * successfully called retrieve() on this file. This prevents + * multiple retrieval attempts. + * @mutex: A mutex that protects the fields of this specific instance + * (e.g., @retrieved, @file), ensuring that operations like + * retrieving or finishing a file are atomic. + * @list: The list_head linking this instance into its parent + * file_set's list of preserved files. + * @token: The user-provided unique token used to identify this file. + * + * This structure is the core in-kernel representation of a single file being + * managed through a live update. An instance is created by luo_preserve_file() + * to link a 'struct file' to its corresponding handler, a user-provided token, + * and the serialized state handle returned by the handler's .preserve() + * operation. + * + * These instances are tracked in a per-file_set list. The @serialized_data + * field, which holds a handle to the file's serialized state, may be updated + * during the .freeze() callback before being serialized for the next kernel. + * After reboot, these structures are recreated by luo_file_deserialize() and + * are finally cleaned up by luo_file_finish(). + */ +struct luo_file { + struct liveupdate_file_handler *fh; + struct file *file; + u64 serialized_data; + void *private_data; + bool retrieved; + struct mutex mutex; + struct list_head list; + u64 token; +}; + +static int luo_alloc_files_mem(struct luo_file_set *file_set) +{ + size_t size; + void *mem; + + if (file_set->files) + return 0; + + WARN_ON_ONCE(file_set->count); + + size = LUO_FILE_PGCNT << PAGE_SHIFT; + mem = kho_alloc_preserve(size); + if (IS_ERR(mem)) + return PTR_ERR(mem); + + file_set->files = mem; + + return 0; +} + +static void luo_free_files_mem(struct luo_file_set *file_set) +{ + /* If file_set has files, no need to free preservation memory */ + if (file_set->count) + return; + + if (!file_set->files) + return; + + kho_unpreserve_free(file_set->files); + file_set->files = NULL; +} + +static bool luo_token_is_used(struct luo_file_set *file_set, u64 token) +{ + struct luo_file *iter; + + list_for_each_entry(iter, &file_set->files_list, list) { + if (iter->token == token) + return true; + } + + return false; +} + +/** + * luo_preserve_file - Initiate the preservation of a file descriptor. + * @file_set: The file_set to which the preserved file will be added. + * @token: A unique, user-provided identifier for the file. + * @fd: The file descriptor to be preserved. + * + * This function orchestrates the first phase of preserving a file. Upon entry, + * it takes a reference to the 'struct file' via fget(), effectively making LUO + * a co-owner of the file. This reference is held until the file is either + * unpreserved or successfully finished in the next kernel, preventing the file + * from being prematurely destroyed. + * + * This function orchestrates the first phase of preserving a file. It performs + * the following steps: + * + * 1. Validates that the @token is not already in use within the file_set. + * 2. Ensures the file_set's memory for files serialization is allocated + * (allocates if needed). + * 3. Iterates through registered handlers, calling can_preserve() to find one + * compatible with the given @fd. + * 4. Calls the handler's .preserve() operation, which saves the file's state + * and returns an opaque private data handle. + * 5. Adds the new instance to the file_set's internal list. + * + * On success, LUO takes a reference to the 'struct file' and considers it + * under its management until it is unpreserved or finished. + * + * In case of any failure, all intermediate allocations (file reference, memory + * for the 'luo_file' struct, etc.) are cleaned up before returning an error. + * + * Context: Can be called from an ioctl handler during normal system operation. + * Return: 0 on success. Returns a negative errno on failure: + * -EEXIST if the token is already used. + * -EBADF if the file descriptor is invalid. + * -ENOSPC if the file_set is full. + * -ENOENT if no compatible handler is found. + * -ENOMEM on memory allocation failure. + * Other erros might be returned by .preserve(). + */ +int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd) +{ + struct liveupdate_file_op_args args = {0}; + struct liveupdate_file_handler *fh; + struct luo_file *luo_file; + struct file *file; + int err; + + if (luo_token_is_used(file_set, token)) + return -EEXIST; + + if (file_set->count == LUO_FILE_MAX) + return -ENOSPC; + + file = fget(fd); + if (!file) + return -EBADF; + + err = luo_alloc_files_mem(file_set); + if (err) + goto err_fput; + + err = -ENOENT; + luo_list_for_each_private(fh, &luo_file_handler_list, list) { + if (fh->ops->can_preserve(fh, file)) { + err = 0; + break; + } + } + + /* err is still -ENOENT if no handler was found */ + if (err) + goto err_free_files_mem; + + luo_file = kzalloc(sizeof(*luo_file), GFP_KERNEL); + if (!luo_file) { + err = -ENOMEM; + goto err_free_files_mem; + } + + luo_file->file = file; + luo_file->fh = fh; + luo_file->token = token; + luo_file->retrieved = false; + mutex_init(&luo_file->mutex); + + args.handler = fh; + args.file = file; + err = fh->ops->preserve(&args); + if (err) + goto err_kfree; + + luo_file->serialized_data = args.serialized_data; + luo_file->private_data = args.private_data; + list_add_tail(&luo_file->list, &file_set->files_list); + file_set->count++; + + return 0; + +err_kfree: + kfree(luo_file); +err_free_files_mem: + luo_free_files_mem(file_set); +err_fput: + fput(file); + + return err; +} + +/** + * luo_file_unpreserve_files - Unpreserves all files from a file_set. + * @file_set: The files to be cleaned up. + * + * This function serves as the primary cleanup path for a file_set. It is + * invoked when the userspace agent closes the file_set's file descriptor. + * + * For each file, it performs the following cleanup actions: + * 1. Calls the handler's .unpreserve() callback to allow the handler to + * release any resources it allocated. + * 2. Removes the file from the file_set's internal tracking list. + * 3. Releases the reference to the 'struct file' that was taken by + * luo_preserve_file() via fput(), returning ownership. + * 4. Frees the memory associated with the internal 'struct luo_file'. + * + * After all individual files are unpreserved, it frees the contiguous memory + * block that was allocated to hold their serialization data. + */ +void luo_file_unpreserve_files(struct luo_file_set *file_set) +{ + struct luo_file *luo_file; + + while (!list_empty(&file_set->files_list)) { + struct liveupdate_file_op_args args = {0}; + + luo_file = list_last_entry(&file_set->files_list, + struct luo_file, list); + + args.handler = luo_file->fh; + args.file = luo_file->file; + args.serialized_data = luo_file->serialized_data; + args.private_data = luo_file->private_data; + luo_file->fh->ops->unpreserve(&args); + + list_del(&luo_file->list); + file_set->count--; + + fput(luo_file->file); + mutex_destroy(&luo_file->mutex); + kfree(luo_file); + } + + luo_free_files_mem(file_set); +} + +static int luo_file_freeze_one(struct luo_file_set *file_set, + struct luo_file *luo_file) +{ + int err = 0; + + guard(mutex)(&luo_file->mutex); + + if (luo_file->fh->ops->freeze) { + struct liveupdate_file_op_args args = {0}; + + args.handler = luo_file->fh; + args.file = luo_file->file; + args.serialized_data = luo_file->serialized_data; + args.private_data = luo_file->private_data; + + err = luo_file->fh->ops->freeze(&args); + if (!err) + luo_file->serialized_data = args.serialized_data; + } + + return err; +} + +static void luo_file_unfreeze_one(struct luo_file_set *file_set, + struct luo_file *luo_file) +{ + guard(mutex)(&luo_file->mutex); + + if (luo_file->fh->ops->unfreeze) { + struct liveupdate_file_op_args args = {0}; + + args.handler = luo_file->fh; + args.file = luo_file->file; + args.serialized_data = luo_file->serialized_data; + args.private_data = luo_file->private_data; + + luo_file->fh->ops->unfreeze(&args); + } + + luo_file->serialized_data = 0; +} + +static void __luo_file_unfreeze(struct luo_file_set *file_set, + struct luo_file *failed_entry) +{ + struct list_head *files_list = &file_set->files_list; + struct luo_file *luo_file; + + list_for_each_entry(luo_file, files_list, list) { + if (luo_file == failed_entry) + break; + + luo_file_unfreeze_one(file_set, luo_file); + } + + memset(file_set->files, 0, LUO_FILE_PGCNT << PAGE_SHIFT); +} + +/** + * luo_file_freeze - Freezes all preserved files and serializes their metadata. + * @file_set: The file_set whose files are to be frozen. + * @file_set_ser: Where to put the serialized file_set. + * + * This function is called from the reboot() syscall path, just before the + * kernel transitions to the new image via kexec. Its purpose is to perform the + * final preparation and serialization of all preserved files in the file_set. + * + * It iterates through each preserved file in FIFO order (the order of + * preservation) and performs two main actions: + * + * 1. Freezes the File: It calls the handler's .freeze() callback for each + * file. This gives the handler a final opportunity to quiesce the device or + * prepare its state for the upcoming reboot. The handler may update its + * private data handle during this step. + * + * 2. Serializes Metadata: After a successful freeze, it copies the final file + * metadata—the handler's compatible string, the user token, and the final + * private data handle—into the pre-allocated contiguous memory buffer + * (file_set->files) that will be handed over to the next kernel via KHO. + * + * Error Handling (Rollback): + * This function is atomic. If any handler's .freeze() operation fails, the + * entire live update is aborted. The __luo_file_unfreeze() helper is + * immediately called to invoke the .unfreeze() op on all files that were + * successfully frozen before the point of failure, rolling them back to a + * running state. The function then returns an error, causing the reboot() + * syscall to fail. + * + * Context: Called only from the liveupdate_reboot() path. + * Return: 0 on success, or a negative errno on failure. + */ +int luo_file_freeze(struct luo_file_set *file_set, + struct luo_file_set_ser *file_set_ser) +{ + struct luo_file_ser *file_ser = file_set->files; + struct luo_file *luo_file; + int err; + int i; + + if (!file_set->count) + return 0; + + if (WARN_ON(!file_ser)) + return -EINVAL; + + i = 0; + list_for_each_entry(luo_file, &file_set->files_list, list) { + err = luo_file_freeze_one(file_set, luo_file); + if (err < 0) { + pr_warn("Freeze failed for token[%#0llx] handler[%s] err[%pe]\n", + luo_file->token, luo_file->fh->compatible, + ERR_PTR(err)); + goto err_unfreeze; + } + + strscpy(file_ser[i].compatible, luo_file->fh->compatible, + sizeof(file_ser[i].compatible)); + file_ser[i].data = luo_file->serialized_data; + file_ser[i].token = luo_file->token; + i++; + } + + file_set_ser->count = file_set->count; + if (file_set->files) + file_set_ser->files = virt_to_phys(file_set->files); + + return 0; + +err_unfreeze: + __luo_file_unfreeze(file_set, luo_file); + + return err; +} + +/** + * luo_file_unfreeze - Unfreezes all files in a file_set and clear serialization + * @file_set: The file_set whose files are to be unfrozen. + * @file_set_ser: Serialized file_set. + * + * This function rolls back the state of all files in a file_set after the + * freeze phase has begun but must be aborted. It is the counterpart to + * luo_file_freeze(). + * + * It invokes the __luo_file_unfreeze() helper with a NULL argument, which + * signals the helper to iterate through all files in the file_set and call + * their respective .unfreeze() handler callbacks. + * + * Context: This is called when the live update is aborted during + * the reboot() syscall, after luo_file_freeze() has been called. + */ +void luo_file_unfreeze(struct luo_file_set *file_set, + struct luo_file_set_ser *file_set_ser) +{ + if (!file_set->count) + return; + + __luo_file_unfreeze(file_set, NULL); + memset(file_set_ser, 0, sizeof(*file_set_ser)); +} + +/** + * luo_retrieve_file - Restores a preserved file from a file_set by its token. + * @file_set: The file_set from which to retrieve the file. + * @token: The unique token identifying the file to be restored. + * @filep: Output parameter; on success, this is populated with a pointer + * to the newly retrieved 'struct file'. + * + * This function is the primary mechanism for recreating a file in the new + * kernel after a live update. It searches the file_set's list of deserialized + * files for an entry matching the provided @token. + * + * The operation is idempotent: if a file has already been successfully + * retrieved, this function will simply return a pointer to the existing + * 'struct file' and report success without re-executing the retrieve + * operation. This is handled by checking the 'retrieved' flag under a lock. + * + * File retrieval can happen in any order; it is not bound by the order of + * preservation. + * + * Context: Can be called from an ioctl or other in-kernel code in the new + * kernel. + * Return: 0 on success. Returns a negative errno on failure: + * -ENOENT if no file with the matching token is found. + * Any error code returned by the handler's .retrieve() op. + */ +int luo_retrieve_file(struct luo_file_set *file_set, u64 token, + struct file **filep) +{ + struct liveupdate_file_op_args args = {0}; + struct luo_file *luo_file; + int err; + + if (list_empty(&file_set->files_list)) + return -ENOENT; + + list_for_each_entry(luo_file, &file_set->files_list, list) { + if (luo_file->token == token) + break; + } + + if (luo_file->token != token) + return -ENOENT; + + guard(mutex)(&luo_file->mutex); + if (luo_file->retrieved) { + /* + * Someone is asking for this file again, so get a reference + * for them. + */ + get_file(luo_file->file); + *filep = luo_file->file; + return 0; + } + + args.handler = luo_file->fh; + args.serialized_data = luo_file->serialized_data; + err = luo_file->fh->ops->retrieve(&args); + if (!err) { + luo_file->file = args.file; + + /* Get reference so we can keep this file in LUO until finish */ + get_file(luo_file->file); + *filep = luo_file->file; + luo_file->retrieved = true; + } + + return err; +} + +static int luo_file_can_finish_one(struct luo_file_set *file_set, + struct luo_file *luo_file) +{ + bool can_finish = true; + + guard(mutex)(&luo_file->mutex); + + if (luo_file->fh->ops->can_finish) { + struct liveupdate_file_op_args args = {0}; + + args.handler = luo_file->fh; + args.file = luo_file->file; + args.serialized_data = luo_file->serialized_data; + args.retrieved = luo_file->retrieved; + can_finish = luo_file->fh->ops->can_finish(&args); + } + + return can_finish ? 0 : -EBUSY; +} + +static void luo_file_finish_one(struct luo_file_set *file_set, + struct luo_file *luo_file) +{ + struct liveupdate_file_op_args args = {0}; + + guard(mutex)(&luo_file->mutex); + + args.handler = luo_file->fh; + args.file = luo_file->file; + args.serialized_data = luo_file->serialized_data; + args.retrieved = luo_file->retrieved; + + luo_file->fh->ops->finish(&args); +} + +/** + * luo_file_finish - Completes the lifecycle for all files in a file_set. + * @file_set: The file_set to be finalized. + * + * This function orchestrates the final teardown of a live update file_set in + * the new kernel. It should be called after all necessary files have been + * retrieved and the userspace agent is ready to release the preserved state. + * + * The function iterates through all tracked files. For each file, it performs + * the following sequence of cleanup actions: + * + * 1. If file is not yet retrieved, retrieves it, and calls can_finish() on + * every file in the file_set. If all can_finish return true, continue to + * finish. + * 2. Calls the handler's .finish() callback (via luo_file_finish_one) to + * allow for final resource cleanup within the handler. + * 3. Releases LUO's ownership reference on the 'struct file' via fput(). This + * is the counterpart to the get_file() call in luo_retrieve_file(). + * 4. Removes the 'struct luo_file' from the file_set's internal list. + * 5. Frees the memory for the 'struct luo_file' instance itself. + * + * After successfully finishing all individual files, it frees the + * contiguous memory block that was used to transfer the serialized metadata + * from the previous kernel. + * + * Error Handling (Atomic Failure): + * This operation is atomic. If any handler's .can_finish() op fails, the entire + * function aborts immediately and returns an error. + * + * Context: Can be called from an ioctl handler in the new kernel. + * Return: 0 on success, or a negative errno on failure. + */ +int luo_file_finish(struct luo_file_set *file_set) +{ + struct list_head *files_list = &file_set->files_list; + struct luo_file *luo_file; + int err; + + if (!file_set->count) + return 0; + + list_for_each_entry(luo_file, files_list, list) { + err = luo_file_can_finish_one(file_set, luo_file); + if (err) + return err; + } + + while (!list_empty(&file_set->files_list)) { + luo_file = list_last_entry(&file_set->files_list, + struct luo_file, list); + + luo_file_finish_one(file_set, luo_file); + + if (luo_file->file) + fput(luo_file->file); + list_del(&luo_file->list); + file_set->count--; + mutex_destroy(&luo_file->mutex); + kfree(luo_file); + } + + if (file_set->files) { + kho_restore_free(file_set->files); + file_set->files = NULL; + } + + return 0; +} + +/** + * luo_file_deserialize - Reconstructs the list of preserved files in the new kernel. + * @file_set: The incoming file_set to fill with deserialized data. + * @file_set_ser: Serialized KHO file_set data from the previous kernel. + * + * This function is called during the early boot process of the new kernel. It + * takes the raw, contiguous memory block of 'struct luo_file_ser' entries, + * provided by the previous kernel, and transforms it back into a live, + * in-memory linked list of 'struct luo_file' instances. + * + * For each serialized entry, it performs the following steps: + * 1. Reads the 'compatible' string. + * 2. Searches the global list of registered file handlers for one that + * matches the compatible string. + * 3. Allocates a new 'struct luo_file'. + * 4. Populates the new structure with the deserialized data (token, private + * data handle) and links it to the found handler. The 'file' pointer is + * initialized to NULL, as the file has not been retrieved yet. + * 5. Adds the new 'struct luo_file' to the file_set's files_list. + * + * This prepares the file_set for userspace, which can later call + * luo_retrieve_file() to restore the actual file descriptors. + * + * Context: Called from session deserialization. + */ +int luo_file_deserialize(struct luo_file_set *file_set, + struct luo_file_set_ser *file_set_ser) +{ + struct luo_file_ser *file_ser; + u64 i; + + if (!file_set_ser->files) { + WARN_ON(file_set_ser->count); + return 0; + } + + file_set->count = file_set_ser->count; + file_set->files = phys_to_virt(file_set_ser->files); + + /* + * Note on error handling: + * + * If deserialization fails (e.g., allocation failure or corrupt data), + * we intentionally skip cleanup of files that were already restored. + * + * A partial failure leaves the preserved state inconsistent. + * Implementing a safe "undo" to unwind complex dependencies (sessions, + * files, hardware state) is error-prone and provides little value, as + * the system is effectively in a broken state. + * + * We treat these resources as leaked. The expected recovery path is for + * userspace to detect the failure and trigger a reboot, which will + * reliably reset devices and reclaim memory. + */ + file_ser = file_set->files; + for (i = 0; i < file_set->count; i++) { + struct liveupdate_file_handler *fh; + bool handler_found = false; + struct luo_file *luo_file; + + luo_list_for_each_private(fh, &luo_file_handler_list, list) { + if (!strcmp(fh->compatible, file_ser[i].compatible)) { + handler_found = true; + break; + } + } + + if (!handler_found) { + pr_warn("No registered handler for compatible '%s'\n", + file_ser[i].compatible); + return -ENOENT; + } + + luo_file = kzalloc(sizeof(*luo_file), GFP_KERNEL); + if (!luo_file) + return -ENOMEM; + + luo_file->fh = fh; + luo_file->file = NULL; + luo_file->serialized_data = file_ser[i].data; + luo_file->token = file_ser[i].token; + luo_file->retrieved = false; + mutex_init(&luo_file->mutex); + list_add_tail(&luo_file->list, &file_set->files_list); + } + + return 0; +} + +void luo_file_set_init(struct luo_file_set *file_set) +{ + INIT_LIST_HEAD(&file_set->files_list); +} + +void luo_file_set_destroy(struct luo_file_set *file_set) +{ + WARN_ON(file_set->count); + WARN_ON(!list_empty(&file_set->files_list)); +} + +/** + * liveupdate_register_file_handler - Register a file handler with LUO. + * @fh: Pointer to a caller-allocated &struct liveupdate_file_handler. + * The caller must initialize this structure, including a unique + * 'compatible' string and a valid 'fh' callbacks. This function adds the + * handler to the global list of supported file handlers. + * + * Context: Typically called during module initialization for file types that + * support live update preservation. + * + * Return: 0 on success. Negative errno on failure. + */ +int liveupdate_register_file_handler(struct liveupdate_file_handler *fh) +{ + struct liveupdate_file_handler *fh_iter; + int err; + + if (!liveupdate_enabled()) + return -EOPNOTSUPP; + + /* Sanity check that all required callbacks are set */ + if (!fh->ops->preserve || !fh->ops->unpreserve || !fh->ops->retrieve || + !fh->ops->finish || !fh->ops->can_preserve) { + return -EINVAL; + } + + /* + * Ensure the system is quiescent (no active sessions). + * This prevents registering new handlers while sessions are active or + * while deserialization is in progress. + */ + if (!luo_session_quiesce()) + return -EBUSY; + + /* Check for duplicate compatible strings */ + luo_list_for_each_private(fh_iter, &luo_file_handler_list, list) { + if (!strcmp(fh_iter->compatible, fh->compatible)) { + pr_err("File handler registration failed: Compatible string '%s' already registered.\n", + fh->compatible); + err = -EEXIST; + goto err_resume; + } + } + + /* Pin the module implementing the handler */ + if (!try_module_get(fh->ops->owner)) { + err = -EAGAIN; + goto err_resume; + } + + INIT_LIST_HEAD(&ACCESS_PRIVATE(fh, list)); + list_add_tail(&ACCESS_PRIVATE(fh, list), &luo_file_handler_list); + luo_session_resume(); + + return 0; + +err_resume: + luo_session_resume(); + return err; +} + +/** + * liveupdate_unregister_file_handler - Unregister a liveupdate file handler + * @fh: The file handler to unregister + * + * Unregisters the file handler from the liveupdate core. This function + * reverses the operations of liveupdate_register_file_handler(). + * + * It ensures safe removal by checking that: + * No live update session is currently in progress. + * + * If the unregistration fails, the internal test state is reverted. + * + * Return: 0 Success. -EOPNOTSUPP when live update is not enabled. -EBUSY A live + * update is in progress, can't quiesce live update. + */ +int liveupdate_unregister_file_handler(struct liveupdate_file_handler *fh) +{ + if (!liveupdate_enabled()) + return -EOPNOTSUPP; + + if (!luo_session_quiesce()) + return -EBUSY; + + list_del(&ACCESS_PRIVATE(fh, list)); + module_put(fh->ops->owner); + luo_session_resume(); + + return 0; +} diff --git a/kernel/liveupdate/luo_internal.h b/kernel/liveupdate/luo_internal.h new file mode 100644 index 000000000000..c8973b543d1d --- /dev/null +++ b/kernel/liveupdate/luo_internal.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +#ifndef _LINUX_LUO_INTERNAL_H +#define _LINUX_LUO_INTERNAL_H + +#include <linux/liveupdate.h> +#include <linux/uaccess.h> + +struct luo_ucmd { + void __user *ubuffer; + u32 user_size; + void *cmd; +}; + +static inline int luo_ucmd_respond(struct luo_ucmd *ucmd, + size_t kernel_cmd_size) +{ + /* + * Copy the minimum of what the user provided and what we actually + * have. + */ + if (copy_to_user(ucmd->ubuffer, ucmd->cmd, + min_t(size_t, ucmd->user_size, kernel_cmd_size))) { + return -EFAULT; + } + return 0; +} + +/* + * Handles a deserialization failure: devices and memory is in unpredictable + * state. + * + * Continuing the boot process after a failure is dangerous because it could + * lead to leaks of private data. + */ +#define luo_restore_fail(__fmt, ...) panic(__fmt, ##__VA_ARGS__) + +/* Mimics list_for_each_entry() but for private list head entries */ +#define luo_list_for_each_private(pos, head, member) \ + for (struct list_head *__iter = (head)->next; \ + __iter != (head) && \ + ({ pos = container_of(__iter, typeof(*(pos)), member); 1; }); \ + __iter = __iter->next) + +/** + * struct luo_file_set - A set of files that belong to the same sessions. + * @files_list: An ordered list of files associated with this session, it is + * ordered by preservation time. + * @files: The physically contiguous memory block that holds the serialized + * state of files. + * @count: A counter tracking the number of files currently stored in the + * @files_list for this session. + */ +struct luo_file_set { + struct list_head files_list; + struct luo_file_ser *files; + long count; +}; + +/** + * struct luo_session - Represents an active or incoming Live Update session. + * @name: A unique name for this session, used for identification and + * retrieval. + * @ser: Pointer to the serialized data for this session. + * @list: A list_head member used to link this session into a global list + * of either outgoing (to be preserved) or incoming (restored from + * previous kernel) sessions. + * @retrieved: A boolean flag indicating whether this session has been + * retrieved by a consumer in the new kernel. + * @file_set: A set of files that belong to this session. + * @mutex: protects fields in the luo_session. + */ +struct luo_session { + char name[LIVEUPDATE_SESSION_NAME_LENGTH]; + struct luo_session_ser *ser; + struct list_head list; + bool retrieved; + struct luo_file_set file_set; + struct mutex mutex; +}; + +int luo_session_create(const char *name, struct file **filep); +int luo_session_retrieve(const char *name, struct file **filep); +int __init luo_session_setup_outgoing(void *fdt); +int __init luo_session_setup_incoming(void *fdt); +int luo_session_serialize(void); +int luo_session_deserialize(void); +bool luo_session_quiesce(void); +void luo_session_resume(void); + +int luo_preserve_file(struct luo_file_set *file_set, u64 token, int fd); +void luo_file_unpreserve_files(struct luo_file_set *file_set); +int luo_file_freeze(struct luo_file_set *file_set, + struct luo_file_set_ser *file_set_ser); +void luo_file_unfreeze(struct luo_file_set *file_set, + struct luo_file_set_ser *file_set_ser); +int luo_retrieve_file(struct luo_file_set *file_set, u64 token, + struct file **filep); +int luo_file_finish(struct luo_file_set *file_set); +int luo_file_deserialize(struct luo_file_set *file_set, + struct luo_file_set_ser *file_set_ser); +void luo_file_set_init(struct luo_file_set *file_set); +void luo_file_set_destroy(struct luo_file_set *file_set); + +#endif /* _LINUX_LUO_INTERNAL_H */ diff --git a/kernel/liveupdate/luo_session.c b/kernel/liveupdate/luo_session.c new file mode 100644 index 000000000000..dbdbc3bd7929 --- /dev/null +++ b/kernel/liveupdate/luo_session.c @@ -0,0 +1,646 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +/** + * DOC: LUO Sessions + * + * LUO Sessions provide the core mechanism for grouping and managing `struct + * file *` instances that need to be preserved across a kexec-based live + * update. Each session acts as a named container for a set of file objects, + * allowing a userspace agent to manage the lifecycle of resources critical to a + * workload. + * + * Core Concepts: + * + * - Named Containers: Sessions are identified by a unique, user-provided name, + * which is used for both creation in the current kernel and retrieval in the + * next kernel. + * + * - Userspace Interface: Session management is driven from userspace via + * ioctls on /dev/liveupdate. + * + * - Serialization: Session metadata is preserved using the KHO framework. When + * a live update is triggered via kexec, an array of `struct luo_session_ser` + * is populated and placed in a preserved memory region. An FDT node is also + * created, containing the count of sessions and the physical address of this + * array. + * + * Session Lifecycle: + * + * 1. Creation: A userspace agent calls `luo_session_create()` to create a + * new, empty session and receives a file descriptor for it. + * + * 2. Serialization: When the `reboot(LINUX_REBOOT_CMD_KEXEC)` syscall is + * made, `luo_session_serialize()` is called. It iterates through all + * active sessions and writes their metadata into a memory area preserved + * by KHO. + * + * 3. Deserialization (in new kernel): After kexec, `luo_session_deserialize()` + * runs, reading the serialized data and creating a list of `struct + * luo_session` objects representing the preserved sessions. + * + * 4. Retrieval: A userspace agent in the new kernel can then call + * `luo_session_retrieve()` with a session name to get a new file + * descriptor and access the preserved state. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/anon_inodes.h> +#include <linux/cleanup.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/file.h> +#include <linux/fs.h> +#include <linux/io.h> +#include <linux/kexec_handover.h> +#include <linux/kho/abi/luo.h> +#include <linux/libfdt.h> +#include <linux/list.h> +#include <linux/liveupdate.h> +#include <linux/mutex.h> +#include <linux/rwsem.h> +#include <linux/slab.h> +#include <linux/unaligned.h> +#include <uapi/linux/liveupdate.h> +#include "luo_internal.h" + +/* 16 4K pages, give space for 744 sessions */ +#define LUO_SESSION_PGCNT 16ul +#define LUO_SESSION_MAX (((LUO_SESSION_PGCNT << PAGE_SHIFT) - \ + sizeof(struct luo_session_header_ser)) / \ + sizeof(struct luo_session_ser)) + +/** + * struct luo_session_header - Header struct for managing LUO sessions. + * @count: The number of sessions currently tracked in the @list. + * @list: The head of the linked list of `struct luo_session` instances. + * @rwsem: A read-write semaphore providing synchronized access to the + * session list and other fields in this structure. + * @header_ser: The header data of serialization array. + * @ser: The serialized session data (an array of + * `struct luo_session_ser`). + * @active: Set to true when first initialized. If previous kernel did not + * send session data, active stays false for incoming. + */ +struct luo_session_header { + long count; + struct list_head list; + struct rw_semaphore rwsem; + struct luo_session_header_ser *header_ser; + struct luo_session_ser *ser; + bool active; +}; + +/** + * struct luo_session_global - Global container for managing LUO sessions. + * @incoming: The sessions passed from the previous kernel. + * @outgoing: The sessions that are going to be passed to the next kernel. + */ +struct luo_session_global { + struct luo_session_header incoming; + struct luo_session_header outgoing; +}; + +static struct luo_session_global luo_session_global = { + .incoming = { + .list = LIST_HEAD_INIT(luo_session_global.incoming.list), + .rwsem = __RWSEM_INITIALIZER(luo_session_global.incoming.rwsem), + }, + .outgoing = { + .list = LIST_HEAD_INIT(luo_session_global.outgoing.list), + .rwsem = __RWSEM_INITIALIZER(luo_session_global.outgoing.rwsem), + }, +}; + +static struct luo_session *luo_session_alloc(const char *name) +{ + struct luo_session *session = kzalloc(sizeof(*session), GFP_KERNEL); + + if (!session) + return ERR_PTR(-ENOMEM); + + strscpy(session->name, name, sizeof(session->name)); + INIT_LIST_HEAD(&session->file_set.files_list); + luo_file_set_init(&session->file_set); + INIT_LIST_HEAD(&session->list); + mutex_init(&session->mutex); + + return session; +} + +static void luo_session_free(struct luo_session *session) +{ + luo_file_set_destroy(&session->file_set); + mutex_destroy(&session->mutex); + kfree(session); +} + +static int luo_session_insert(struct luo_session_header *sh, + struct luo_session *session) +{ + struct luo_session *it; + + guard(rwsem_write)(&sh->rwsem); + + /* + * For outgoing we should make sure there is room in serialization array + * for new session. + */ + if (sh == &luo_session_global.outgoing) { + if (sh->count == LUO_SESSION_MAX) + return -ENOMEM; + } + + /* + * For small number of sessions this loop won't hurt performance + * but if we ever start using a lot of sessions, this might + * become a bottle neck during deserialization time, as it would + * cause O(n*n) complexity. + */ + list_for_each_entry(it, &sh->list, list) { + if (!strncmp(it->name, session->name, sizeof(it->name))) + return -EEXIST; + } + list_add_tail(&session->list, &sh->list); + sh->count++; + + return 0; +} + +static void luo_session_remove(struct luo_session_header *sh, + struct luo_session *session) +{ + guard(rwsem_write)(&sh->rwsem); + list_del(&session->list); + sh->count--; +} + +static int luo_session_finish_one(struct luo_session *session) +{ + guard(mutex)(&session->mutex); + return luo_file_finish(&session->file_set); +} + +static void luo_session_unfreeze_one(struct luo_session *session, + struct luo_session_ser *ser) +{ + guard(mutex)(&session->mutex); + luo_file_unfreeze(&session->file_set, &ser->file_set_ser); +} + +static int luo_session_freeze_one(struct luo_session *session, + struct luo_session_ser *ser) +{ + guard(mutex)(&session->mutex); + return luo_file_freeze(&session->file_set, &ser->file_set_ser); +} + +static int luo_session_release(struct inode *inodep, struct file *filep) +{ + struct luo_session *session = filep->private_data; + struct luo_session_header *sh; + + /* If retrieved is set, it means this session is from incoming list */ + if (session->retrieved) { + int err = luo_session_finish_one(session); + + if (err) { + pr_warn("Unable to finish session [%s] on release\n", + session->name); + return err; + } + sh = &luo_session_global.incoming; + } else { + scoped_guard(mutex, &session->mutex) + luo_file_unpreserve_files(&session->file_set); + sh = &luo_session_global.outgoing; + } + + luo_session_remove(sh, session); + luo_session_free(session); + + return 0; +} + +static int luo_session_preserve_fd(struct luo_session *session, + struct luo_ucmd *ucmd) +{ + struct liveupdate_session_preserve_fd *argp = ucmd->cmd; + int err; + + guard(mutex)(&session->mutex); + err = luo_preserve_file(&session->file_set, argp->token, argp->fd); + if (err) + return err; + + err = luo_ucmd_respond(ucmd, sizeof(*argp)); + if (err) + pr_warn("The file was successfully preserved, but response to user failed\n"); + + return err; +} + +static int luo_session_retrieve_fd(struct luo_session *session, + struct luo_ucmd *ucmd) +{ + struct liveupdate_session_retrieve_fd *argp = ucmd->cmd; + struct file *file; + int err; + + argp->fd = get_unused_fd_flags(O_CLOEXEC); + if (argp->fd < 0) + return argp->fd; + + guard(mutex)(&session->mutex); + err = luo_retrieve_file(&session->file_set, argp->token, &file); + if (err < 0) + goto err_put_fd; + + err = luo_ucmd_respond(ucmd, sizeof(*argp)); + if (err) + goto err_put_file; + + fd_install(argp->fd, file); + + return 0; + +err_put_file: + fput(file); +err_put_fd: + put_unused_fd(argp->fd); + + return err; +} + +static int luo_session_finish(struct luo_session *session, + struct luo_ucmd *ucmd) +{ + struct liveupdate_session_finish *argp = ucmd->cmd; + int err = luo_session_finish_one(session); + + if (err) + return err; + + return luo_ucmd_respond(ucmd, sizeof(*argp)); +} + +union ucmd_buffer { + struct liveupdate_session_finish finish; + struct liveupdate_session_preserve_fd preserve; + struct liveupdate_session_retrieve_fd retrieve; +}; + +struct luo_ioctl_op { + unsigned int size; + unsigned int min_size; + unsigned int ioctl_num; + int (*execute)(struct luo_session *session, struct luo_ucmd *ucmd); +}; + +#define IOCTL_OP(_ioctl, _fn, _struct, _last) \ + [_IOC_NR(_ioctl) - LIVEUPDATE_CMD_SESSION_BASE] = { \ + .size = sizeof(_struct) + \ + BUILD_BUG_ON_ZERO(sizeof(union ucmd_buffer) < \ + sizeof(_struct)), \ + .min_size = offsetofend(_struct, _last), \ + .ioctl_num = _ioctl, \ + .execute = _fn, \ + } + +static const struct luo_ioctl_op luo_session_ioctl_ops[] = { + IOCTL_OP(LIVEUPDATE_SESSION_FINISH, luo_session_finish, + struct liveupdate_session_finish, reserved), + IOCTL_OP(LIVEUPDATE_SESSION_PRESERVE_FD, luo_session_preserve_fd, + struct liveupdate_session_preserve_fd, token), + IOCTL_OP(LIVEUPDATE_SESSION_RETRIEVE_FD, luo_session_retrieve_fd, + struct liveupdate_session_retrieve_fd, token), +}; + +static long luo_session_ioctl(struct file *filep, unsigned int cmd, + unsigned long arg) +{ + struct luo_session *session = filep->private_data; + const struct luo_ioctl_op *op; + struct luo_ucmd ucmd = {}; + union ucmd_buffer buf; + unsigned int nr; + int ret; + + nr = _IOC_NR(cmd); + if (nr < LIVEUPDATE_CMD_SESSION_BASE || (nr - LIVEUPDATE_CMD_SESSION_BASE) >= + ARRAY_SIZE(luo_session_ioctl_ops)) { + return -EINVAL; + } + + ucmd.ubuffer = (void __user *)arg; + ret = get_user(ucmd.user_size, (u32 __user *)ucmd.ubuffer); + if (ret) + return ret; + + op = &luo_session_ioctl_ops[nr - LIVEUPDATE_CMD_SESSION_BASE]; + if (op->ioctl_num != cmd) + return -ENOIOCTLCMD; + if (ucmd.user_size < op->min_size) + return -EINVAL; + + ucmd.cmd = &buf; + ret = copy_struct_from_user(ucmd.cmd, op->size, ucmd.ubuffer, + ucmd.user_size); + if (ret) + return ret; + + return op->execute(session, &ucmd); +} + +static const struct file_operations luo_session_fops = { + .owner = THIS_MODULE, + .release = luo_session_release, + .unlocked_ioctl = luo_session_ioctl, +}; + +/* Create a "struct file" for session */ +static int luo_session_getfile(struct luo_session *session, struct file **filep) +{ + char name_buf[128]; + struct file *file; + + lockdep_assert_held(&session->mutex); + snprintf(name_buf, sizeof(name_buf), "[luo_session] %s", session->name); + file = anon_inode_getfile(name_buf, &luo_session_fops, session, O_RDWR); + if (IS_ERR(file)) + return PTR_ERR(file); + + *filep = file; + + return 0; +} + +int luo_session_create(const char *name, struct file **filep) +{ + struct luo_session *session; + int err; + + session = luo_session_alloc(name); + if (IS_ERR(session)) + return PTR_ERR(session); + + err = luo_session_insert(&luo_session_global.outgoing, session); + if (err) + goto err_free; + + scoped_guard(mutex, &session->mutex) + err = luo_session_getfile(session, filep); + if (err) + goto err_remove; + + return 0; + +err_remove: + luo_session_remove(&luo_session_global.outgoing, session); +err_free: + luo_session_free(session); + + return err; +} + +int luo_session_retrieve(const char *name, struct file **filep) +{ + struct luo_session_header *sh = &luo_session_global.incoming; + struct luo_session *session = NULL; + struct luo_session *it; + int err; + + scoped_guard(rwsem_read, &sh->rwsem) { + list_for_each_entry(it, &sh->list, list) { + if (!strncmp(it->name, name, sizeof(it->name))) { + session = it; + break; + } + } + } + + if (!session) + return -ENOENT; + + guard(mutex)(&session->mutex); + if (session->retrieved) + return -EINVAL; + + err = luo_session_getfile(session, filep); + if (!err) + session->retrieved = true; + + return err; +} + +int __init luo_session_setup_outgoing(void *fdt_out) +{ + struct luo_session_header_ser *header_ser; + u64 header_ser_pa; + int err; + + header_ser = kho_alloc_preserve(LUO_SESSION_PGCNT << PAGE_SHIFT); + if (IS_ERR(header_ser)) + return PTR_ERR(header_ser); + header_ser_pa = virt_to_phys(header_ser); + + err = fdt_begin_node(fdt_out, LUO_FDT_SESSION_NODE_NAME); + err |= fdt_property_string(fdt_out, "compatible", + LUO_FDT_SESSION_COMPATIBLE); + err |= fdt_property(fdt_out, LUO_FDT_SESSION_HEADER, &header_ser_pa, + sizeof(header_ser_pa)); + err |= fdt_end_node(fdt_out); + + if (err) + goto err_unpreserve; + + luo_session_global.outgoing.header_ser = header_ser; + luo_session_global.outgoing.ser = (void *)(header_ser + 1); + luo_session_global.outgoing.active = true; + + return 0; + +err_unpreserve: + kho_unpreserve_free(header_ser); + return err; +} + +int __init luo_session_setup_incoming(void *fdt_in) +{ + struct luo_session_header_ser *header_ser; + int err, header_size, offset; + u64 header_ser_pa; + const void *ptr; + + offset = fdt_subnode_offset(fdt_in, 0, LUO_FDT_SESSION_NODE_NAME); + if (offset < 0) { + pr_err("Unable to get session node: [%s]\n", + LUO_FDT_SESSION_NODE_NAME); + return -EINVAL; + } + + err = fdt_node_check_compatible(fdt_in, offset, + LUO_FDT_SESSION_COMPATIBLE); + if (err) { + pr_err("Session node incompatible [%s]\n", + LUO_FDT_SESSION_COMPATIBLE); + return -EINVAL; + } + + header_size = 0; + ptr = fdt_getprop(fdt_in, offset, LUO_FDT_SESSION_HEADER, &header_size); + if (!ptr || header_size != sizeof(u64)) { + pr_err("Unable to get session header '%s' [%d]\n", + LUO_FDT_SESSION_HEADER, header_size); + return -EINVAL; + } + + header_ser_pa = get_unaligned((u64 *)ptr); + header_ser = phys_to_virt(header_ser_pa); + + luo_session_global.incoming.header_ser = header_ser; + luo_session_global.incoming.ser = (void *)(header_ser + 1); + luo_session_global.incoming.active = true; + + return 0; +} + +int luo_session_deserialize(void) +{ + struct luo_session_header *sh = &luo_session_global.incoming; + static bool is_deserialized; + static int err; + + /* If has been deserialized, always return the same error code */ + if (is_deserialized) + return err; + + is_deserialized = true; + if (!sh->active) + return 0; + + /* + * Note on error handling: + * + * If deserialization fails (e.g., allocation failure or corrupt data), + * we intentionally skip cleanup of sessions that were already restored. + * + * A partial failure leaves the preserved state inconsistent. + * Implementing a safe "undo" to unwind complex dependencies (sessions, + * files, hardware state) is error-prone and provides little value, as + * the system is effectively in a broken state. + * + * We treat these resources as leaked. The expected recovery path is for + * userspace to detect the failure and trigger a reboot, which will + * reliably reset devices and reclaim memory. + */ + for (int i = 0; i < sh->header_ser->count; i++) { + struct luo_session *session; + + session = luo_session_alloc(sh->ser[i].name); + if (IS_ERR(session)) { + pr_warn("Failed to allocate session [%s] during deserialization %pe\n", + sh->ser[i].name, session); + return PTR_ERR(session); + } + + err = luo_session_insert(sh, session); + if (err) { + pr_warn("Failed to insert session [%s] %pe\n", + session->name, ERR_PTR(err)); + luo_session_free(session); + return err; + } + + scoped_guard(mutex, &session->mutex) { + luo_file_deserialize(&session->file_set, + &sh->ser[i].file_set_ser); + } + } + + kho_restore_free(sh->header_ser); + sh->header_ser = NULL; + sh->ser = NULL; + + return 0; +} + +int luo_session_serialize(void) +{ + struct luo_session_header *sh = &luo_session_global.outgoing; + struct luo_session *session; + int i = 0; + int err; + + guard(rwsem_write)(&sh->rwsem); + list_for_each_entry(session, &sh->list, list) { + err = luo_session_freeze_one(session, &sh->ser[i]); + if (err) + goto err_undo; + + strscpy(sh->ser[i].name, session->name, + sizeof(sh->ser[i].name)); + i++; + } + sh->header_ser->count = sh->count; + + return 0; + +err_undo: + list_for_each_entry_continue_reverse(session, &sh->list, list) { + i--; + luo_session_unfreeze_one(session, &sh->ser[i]); + memset(sh->ser[i].name, 0, sizeof(sh->ser[i].name)); + } + + return err; +} + +/** + * luo_session_quiesce - Ensure no active sessions exist and lock session lists. + * + * Acquires exclusive write locks on both incoming and outgoing session lists. + * It then validates no sessions exist in either list. + * + * This mechanism is used during file handler un/registration to ensure that no + * sessions are currently using the handler, and no new sessions can be created + * while un/registration is in progress. + * + * This prevents registering new handlers while sessions are active or + * while deserialization is in progress. + * + * Return: + * true - System is quiescent (0 sessions) and locked. + * false - Active sessions exist. The locks are released internally. + */ +bool luo_session_quiesce(void) +{ + down_write(&luo_session_global.incoming.rwsem); + down_write(&luo_session_global.outgoing.rwsem); + + if (luo_session_global.incoming.count || + luo_session_global.outgoing.count) { + up_write(&luo_session_global.outgoing.rwsem); + up_write(&luo_session_global.incoming.rwsem); + return false; + } + + return true; +} + +/** + * luo_session_resume - Unlock session lists and resume normal activity. + * + * Releases the exclusive locks acquired by a successful call to + * luo_session_quiesce(). + */ +void luo_session_resume(void) +{ + up_write(&luo_session_global.outgoing.rwsem); + up_write(&luo_session_global.incoming.rwsem); +} diff --git a/kernel/module/main.c b/kernel/module/main.c index 7b3ec2fa6e7c..710ee30b3bea 100644 --- a/kernel/module/main.c +++ b/kernel/module/main.c @@ -954,7 +954,7 @@ size_t module_flags_taint(unsigned long taints, char *buf) int i; for (i = 0; i < TAINT_FLAGS_COUNT; i++) { - if (taint_flags[i].module && test_bit(i, &taints)) + if (test_bit(i, &taints)) buf[l++] = taint_flags[i].c_true; } diff --git a/kernel/panic.c b/kernel/panic.c index b2f2470af7e5..0d52210a9e2b 100644 --- a/kernel/panic.c +++ b/kernel/panic.c @@ -401,7 +401,7 @@ static void panic_trigger_all_cpu_backtrace(void) */ static void panic_other_cpus_shutdown(bool crash_kexec) { - if (panic_print & SYS_INFO_ALL_CPU_BT) + if (panic_print & SYS_INFO_ALL_BT) panic_trigger_all_cpu_backtrace(); /* @@ -628,38 +628,40 @@ void panic(const char *fmt, ...) } EXPORT_SYMBOL(panic); -#define TAINT_FLAG(taint, _c_true, _c_false, _module) \ +#define TAINT_FLAG(taint, _c_true, _c_false) \ [ TAINT_##taint ] = { \ .c_true = _c_true, .c_false = _c_false, \ - .module = _module, \ .desc = #taint, \ } /* - * TAINT_FORCED_RMMOD could be a per-module flag but the module - * is being removed anyway. + * NOTE: if you modify the taint_flags or TAINT_FLAGS_COUNT, + * please also modify tools/debugging/kernel-chktaint and + * Documentation/admin-guide/tainted-kernels.rst, including its + * small shell script that prints the TAINT_FLAGS_COUNT bits of + * /proc/sys/kernel/tainted. */ const struct taint_flag taint_flags[TAINT_FLAGS_COUNT] = { - TAINT_FLAG(PROPRIETARY_MODULE, 'P', 'G', true), - TAINT_FLAG(FORCED_MODULE, 'F', ' ', true), - TAINT_FLAG(CPU_OUT_OF_SPEC, 'S', ' ', false), - TAINT_FLAG(FORCED_RMMOD, 'R', ' ', false), - TAINT_FLAG(MACHINE_CHECK, 'M', ' ', false), - TAINT_FLAG(BAD_PAGE, 'B', ' ', false), - TAINT_FLAG(USER, 'U', ' ', false), - TAINT_FLAG(DIE, 'D', ' ', false), - TAINT_FLAG(OVERRIDDEN_ACPI_TABLE, 'A', ' ', false), - TAINT_FLAG(WARN, 'W', ' ', false), - TAINT_FLAG(CRAP, 'C', ' ', true), - TAINT_FLAG(FIRMWARE_WORKAROUND, 'I', ' ', false), - TAINT_FLAG(OOT_MODULE, 'O', ' ', true), - TAINT_FLAG(UNSIGNED_MODULE, 'E', ' ', true), - TAINT_FLAG(SOFTLOCKUP, 'L', ' ', false), - TAINT_FLAG(LIVEPATCH, 'K', ' ', true), - TAINT_FLAG(AUX, 'X', ' ', true), - TAINT_FLAG(RANDSTRUCT, 'T', ' ', true), - TAINT_FLAG(TEST, 'N', ' ', true), - TAINT_FLAG(FWCTL, 'J', ' ', true), + TAINT_FLAG(PROPRIETARY_MODULE, 'P', 'G'), + TAINT_FLAG(FORCED_MODULE, 'F', ' '), + TAINT_FLAG(CPU_OUT_OF_SPEC, 'S', ' '), + TAINT_FLAG(FORCED_RMMOD, 'R', ' '), + TAINT_FLAG(MACHINE_CHECK, 'M', ' '), + TAINT_FLAG(BAD_PAGE, 'B', ' '), + TAINT_FLAG(USER, 'U', ' '), + TAINT_FLAG(DIE, 'D', ' '), + TAINT_FLAG(OVERRIDDEN_ACPI_TABLE, 'A', ' '), + TAINT_FLAG(WARN, 'W', ' '), + TAINT_FLAG(CRAP, 'C', ' '), + TAINT_FLAG(FIRMWARE_WORKAROUND, 'I', ' '), + TAINT_FLAG(OOT_MODULE, 'O', ' '), + TAINT_FLAG(UNSIGNED_MODULE, 'E', ' '), + TAINT_FLAG(SOFTLOCKUP, 'L', ' '), + TAINT_FLAG(LIVEPATCH, 'K', ' '), + TAINT_FLAG(AUX, 'X', ' '), + TAINT_FLAG(RANDSTRUCT, 'T', ' '), + TAINT_FLAG(TEST, 'N', ' '), + TAINT_FLAG(FWCTL, 'J', ' '), }; #undef TAINT_FLAG diff --git a/kernel/resource.c b/kernel/resource.c index b9fa2a4ce089..e4e9bac12e6e 100644 --- a/kernel/resource.c +++ b/kernel/resource.c @@ -341,6 +341,8 @@ static int find_next_iomem_res(resource_size_t start, resource_size_t end, unsigned long flags, unsigned long desc, struct resource *res) { + /* Skip children until we find a top level range that matches */ + bool skip_children = true; struct resource *p; if (!res) @@ -351,7 +353,7 @@ static int find_next_iomem_res(resource_size_t start, resource_size_t end, read_lock(&resource_lock); - for_each_resource(&iomem_resource, p, false) { + for_each_resource(&iomem_resource, p, skip_children) { /* If we passed the resource we are looking for, stop */ if (p->start > end) { p = NULL; @@ -362,6 +364,12 @@ static int find_next_iomem_res(resource_size_t start, resource_size_t end, if (p->end < start) continue; + /* + * We found a top level range that matches what we are looking + * for. Time to start checking children too. + */ + skip_children = false; + /* Found a match, break */ if (is_type_match(p, flags, desc)) break; diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b7801cd05d5a..41ba0be16911 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -878,7 +878,7 @@ static enum hrtimer_restart hrtick(struct hrtimer *timer) rq_lock(rq, &rf); update_rq_clock(rq); - rq->donor->sched_class->task_tick(rq, rq->curr, 1); + rq->donor->sched_class->task_tick(rq, rq->donor, 1); rq_unlock(rq, &rf); return HRTIMER_NORESTART; @@ -7360,15 +7360,12 @@ void rt_mutex_setprio(struct task_struct *p, struct task_struct *pi_task) p->prio = prio; } out_unlock: - /* Avoid rq from going away on us: */ - preempt_disable(); + /* Caller holds task_struct::pi_lock, IRQs are still disabled */ rq_unpin_lock(rq, &rf); __balance_callbacks(rq); rq_repin_lock(rq, &rf); __task_rq_unlock(rq, p, &rf); - - preempt_enable(); } #endif /* CONFIG_RT_MUTEXES */ diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 769d7b7990df..da46c3164537 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -4034,6 +4034,9 @@ static inline bool cfs_rq_is_decayed(struct cfs_rq *cfs_rq) if (child_cfs_rq_on_list(cfs_rq)) return false; + if (cfs_rq->tg_load_avg_contrib) + return false; + return true; } diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index bbf513b3e76c..d30cca6870f5 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -1167,7 +1167,7 @@ struct rq { * one CPU and if it got migrated afterwards it may decrease * it on another CPU. Always updated under the runqueue lock: */ - unsigned long nr_uninterruptible; + unsigned long nr_uninterruptible; #ifdef CONFIG_SCHED_PROXY_EXEC struct task_struct __rcu *donor; /* Scheduling context */ diff --git a/kernel/sched/stats.h b/kernel/sched/stats.h index cbf7206b3f9d..c903f1a42891 100644 --- a/kernel/sched/stats.h +++ b/kernel/sched/stats.h @@ -180,8 +180,13 @@ static inline void psi_dequeue(struct task_struct *p, int flags) * avoid walking all ancestors twice, psi_task_switch() handles * TSK_RUNNING and TSK_IOWAIT for us when it moves TSK_ONCPU. * Do nothing here. + * + * In the SCHED_PROXY_EXECUTION case we may do sleeping + * dequeues that are not followed by a task switch, so check + * TSK_ONCPU is set to ensure the task switch is imminent. + * Otherwise clear the flags as usual. */ - if (flags & DEQUEUE_SLEEP) + if ((flags & DEQUEUE_SLEEP) && (p->psi_flags & TSK_ONCPU)) return; /* diff --git a/kernel/scs.c b/kernel/scs.c index d7809affe740..772488afd5b9 100644 --- a/kernel/scs.c +++ b/kernel/scs.c @@ -135,7 +135,7 @@ static void scs_check_usage(struct task_struct *tsk) if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE)) return; - for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) { + for (p = task_scs(tsk); p < __scs_magic(task_scs(tsk)); ++p) { if (!READ_ONCE_NOCHECK(*p)) break; used += sizeof(*p); diff --git a/kernel/trace/fgraph.c b/kernel/trace/fgraph.c index 65ac0f04a946..cc48d16be43e 100644 --- a/kernel/trace/fgraph.c +++ b/kernel/trace/fgraph.c @@ -163,7 +163,7 @@ enum { #define RET_STACK(t, offset) ((struct ftrace_ret_stack *)(&(t)->ret_stack[offset])) /* - * Each fgraph_ops has a reservered unsigned long at the end (top) of the + * Each fgraph_ops has a reserved unsigned long at the end (top) of the * ret_stack to store task specific state. */ #define SHADOW_STACK_TASK_VARS(ret_stack) \ diff --git a/kernel/trace/fprobe.c b/kernel/trace/fprobe.c index 0b1ee8e585f2..1188eefef07c 100644 --- a/kernel/trace/fprobe.c +++ b/kernel/trace/fprobe.c @@ -30,7 +30,7 @@ * fprobe_table: hold 'fprobe_hlist::hlist' for checking the fprobe still * exists. The key is the address of fprobe instance. * fprobe_ip_table: hold 'fprobe_hlist::array[*]' for searching the fprobe - * instance related to the funciton address. The key is the ftrace IP + * instance related to the function address. The key is the ftrace IP * address. * * When unregistering the fprobe, fprobe_hlist::fp and fprobe_hlist::array[*].fp diff --git a/kernel/trace/ring_buffer.c b/kernel/trace/ring_buffer.c index 8688c88534de..41c9f5d079be 100644 --- a/kernel/trace/ring_buffer.c +++ b/kernel/trace/ring_buffer.c @@ -1770,7 +1770,7 @@ static bool rb_meta_init(struct trace_buffer *buffer, int scratch_size) bmeta->total_size = total_size; bmeta->buffers_offset = (void *)ptr - (void *)bmeta; - /* Zero out the scatch pad */ + /* Zero out the scratch pad */ memset((void *)bmeta + sizeof(*bmeta), 0, bmeta->buffers_offset - sizeof(*bmeta)); return false; @@ -6089,7 +6089,7 @@ static void rb_clear_buffer_page(struct buffer_page *page) * id field, and updated via this function. * * But for a fixed memory mapped buffer, the id is already assigned for - * fixed memory ording in the memory layout and can not be used. Instead + * fixed memory ordering in the memory layout and can not be used. Instead * the index of where the page lies in the memory layout is used. * * For the normal pages, set the buffer page id with the passed in @id @@ -7669,7 +7669,7 @@ static __init int test_ringbuffer(void) /* * Show buffer is enabled before setting rb_test_started. * Yes there's a small race window where events could be - * dropped and the thread wont catch it. But when a ring + * dropped and the thread won't catch it. But when a ring * buffer gets enabled, there will always be some kind of * delay before other CPUs see it. Thus, we don't care about * those dropped events. We care about events dropped after diff --git a/kernel/trace/ring_buffer_benchmark.c b/kernel/trace/ring_buffer_benchmark.c index cdc3aea12c93..593e3b59e42e 100644 --- a/kernel/trace/ring_buffer_benchmark.c +++ b/kernel/trace/ring_buffer_benchmark.c @@ -433,7 +433,7 @@ static int __init ring_buffer_benchmark_init(void) { int ret; - /* make a one meg buffer in overwite mode */ + /* make a one meg buffer in overwrite mode */ buffer = ring_buffer_alloc(1000000, RB_FL_OVERWRITE); if (!buffer) return -ENOMEM; diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index c9fbb316dcbd..e575956ef9b5 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -125,7 +125,7 @@ cpumask_var_t __read_mostly tracing_buffer_mask; * If there is an oops (or kernel panic) and the ftrace_dump_on_oops * is set, then ftrace_dump is called. This will output the contents * of the ftrace buffers to the console. This is very useful for - * capturing traces that lead to crashes and outputing it to a + * capturing traces that lead to crashes and outputting it to a * serial console. * * It is default off, but you can enable it with either specifying @@ -134,7 +134,7 @@ cpumask_var_t __read_mostly tracing_buffer_mask; * Set 1 if you want to dump buffers of all CPUs * Set 2 if you want to dump the buffer of the CPU that triggered oops * Set instance name if you want to dump the specific trace instance - * Multiple instance dump is also supported, and instances are seperated + * Multiple instance dump is also supported, and instances are separated * by commas. */ /* Set to string format zero to disable by default */ @@ -4709,8 +4709,10 @@ __tracing_open(struct inode *inode, struct file *file, bool snapshot) * If pause-on-trace is enabled, then stop the trace while * dumping, unless this is the "snapshot" file */ - if (!iter->snapshot && (tr->trace_flags & TRACE_ITER(PAUSE_ON_TRACE))) + if (!iter->snapshot && (tr->trace_flags & TRACE_ITER(PAUSE_ON_TRACE))) { + iter->iter_flags |= TRACE_FILE_PAUSE; tracing_stop_tr(tr); + } if (iter->cpu_file == RING_BUFFER_ALL_CPUS) { for_each_tracing_cpu(cpu) { @@ -4842,7 +4844,7 @@ static int tracing_release(struct inode *inode, struct file *file) if (iter->trace && iter->trace->close) iter->trace->close(iter); - if (!iter->snapshot && tr->stop_count) + if (iter->iter_flags & TRACE_FILE_PAUSE) /* reenable tracing if it was previously enabled */ tracing_start_tr(tr); @@ -5276,7 +5278,7 @@ int set_tracer_flag(struct trace_array *tr, u64 mask, int enabled) return -EINVAL; /* * An instance must always have it set. - * by default, that's the global_trace instane. + * by default, that's the global_trace instance. */ if (printk_trace == tr) update_printk_trace(&global_trace); @@ -7554,7 +7556,7 @@ char *trace_user_fault_read(struct trace_user_buf_info *tinfo, migrate_disable(); /* - * Now preemption is being enabed and another task can come in + * Now preemption is being enabled and another task can come in * and use the same buffer and corrupt our data. */ preempt_enable_notrace(); @@ -11329,7 +11331,7 @@ __init static void do_allocate_snapshot(const char *name) /* * When allocate_snapshot is set, the next call to * allocate_trace_buffers() (called by trace_array_get_by_name()) - * will allocate the snapshot buffer. That will alse clear + * will allocate the snapshot buffer. That will also clear * this flag. */ allocate_snapshot = true; diff --git a/kernel/trace/trace_events.c b/kernel/trace/trace_events.c index 9b07ad9eb284..b16a5a158040 100644 --- a/kernel/trace/trace_events.c +++ b/kernel/trace/trace_events.c @@ -360,7 +360,7 @@ static bool process_string(const char *fmt, int len, struct trace_event_call *ca /* Anything else, this isn't a function */ break; } - /* A function could be wrapped in parethesis, try the next one */ + /* A function could be wrapped in parenthesis, try the next one */ s = r + 1; } while (s < e); @@ -567,7 +567,7 @@ static void test_event_printk(struct trace_event_call *call) * If start_arg is zero, then this is the start of the * first argument. The processing of the argument happens * when the end of the argument is found, as it needs to - * handle paranthesis and such. + * handle parenthesis and such. */ if (!start_arg) { start_arg = i; @@ -785,7 +785,7 @@ static int __ftrace_event_enable_disable(struct trace_event_file *file, * * When soft_disable is not set but the soft_mode is, * we do nothing. Do not disable the tracepoint, otherwise - * "soft enable"s (clearing the SOFT_DISABLED bit) wont work. + * "soft enable"s (clearing the SOFT_DISABLED bit) won't work. */ if (soft_disable) { if (atomic_dec_return(&file->sm_ref) > 0) @@ -1394,7 +1394,7 @@ int ftrace_set_clr_event(struct trace_array *tr, char *buf, int set) if (!tr) return -ENOENT; - /* Modules events can be appened with :mod:<module> */ + /* Modules events can be appended with :mod:<module> */ mod = strstr(buf, ":mod:"); if (mod) { *mod = '\0'; diff --git a/kernel/trace/trace_events_filter.c b/kernel/trace/trace_events_filter.c index 54226b48b2d1..385af8405392 100644 --- a/kernel/trace/trace_events_filter.c +++ b/kernel/trace/trace_events_filter.c @@ -142,7 +142,7 @@ static bool is_not(const char *str) } /** - * struct prog_entry - a singe entry in the filter program + * struct prog_entry - a single entry in the filter program * @target: Index to jump to on a branch (actually one minus the index) * @when_to_branch: The value of the result of the predicate to do a branch * @pred: The predicate to execute. diff --git a/kernel/trace/trace_events_hist.c b/kernel/trace/trace_events_hist.c index 289bdea98776..5e6e70540eef 100644 --- a/kernel/trace/trace_events_hist.c +++ b/kernel/trace/trace_events_hist.c @@ -5283,7 +5283,7 @@ hist_trigger_actions(struct hist_trigger_data *hist_data, * on the stack, so when the histogram trigger is initialized * a percpu array of 4 hist_pad structures is allocated. * This will cover every context from normal, softirq, irq and NMI - * in the very unlikely event that a tigger happens at each of + * in the very unlikely event that a trigger happens at each of * these contexts and interrupts a currently active trigger. */ struct hist_pad { diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c index 2f19bbe73d27..4554c458b78c 100644 --- a/kernel/trace/trace_events_synth.c +++ b/kernel/trace/trace_events_synth.c @@ -375,7 +375,6 @@ static enum print_line_t print_synth_event(struct trace_iterator *iter, n_u64++; } else { trace_seq_printf(s, print_fmt, se->fields[i]->name, - STR_VAR_LEN_MAX, (char *)&entry->fields[n_u64].as_u64, i == se->n_fields - 1 ? "" : " "); n_u64 += STR_VAR_LEN_MAX / sizeof(u64); diff --git a/kernel/trace/trace_events_trigger.c b/kernel/trace/trace_events_trigger.c index 96aad82b1628..06b75bcfc7b8 100644 --- a/kernel/trace/trace_events_trigger.c +++ b/kernel/trace/trace_events_trigger.c @@ -732,7 +732,7 @@ static void unregister_trigger(char *glob, * param - text following cmd and ':' and stripped of filter * filter - the optional filter text following (and including) 'if' * - * To illustrate the use of these componenents, here are some concrete + * To illustrate the use of these components, here are some concrete * examples. For the following triggers: * * echo 'traceon:5 if pid == 0' > trigger diff --git a/kernel/trace/trace_events_user.c b/kernel/trace/trace_events_user.c index b15854c75d4f..dca6e50b3b21 100644 --- a/kernel/trace/trace_events_user.c +++ b/kernel/trace/trace_events_user.c @@ -1041,7 +1041,7 @@ static int user_field_array_size(const char *type) static int user_field_size(const char *type) { - /* long is not allowed from a user, since it's ambigious in size */ + /* long is not allowed from a user, since it's ambiguous in size */ if (strcmp(type, "s64") == 0) return sizeof(s64); if (strcmp(type, "u64") == 0) @@ -1079,7 +1079,7 @@ static int user_field_size(const char *type) if (str_has_prefix(type, "__rel_loc ")) return sizeof(u32); - /* Uknown basic type, error */ + /* Unknown basic type, error */ return -EINVAL; } @@ -2465,7 +2465,7 @@ static long user_events_ioctl_reg(struct user_event_file_info *info, /* * Prevent users from using the same address and bit multiple times * within the same mm address space. This can cause unexpected behavior - * for user processes that is far easier to debug if this is explictly + * for user processes that is far easier to debug if this is explicitly * an error upon registering. */ if (current_user_event_enabler_exists((unsigned long)reg.enable_addr, diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index a9962d4497e8..827104d00bc0 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -329,7 +329,7 @@ static struct osnoise_data { u64 print_stack; /* print IRQ stack if total > */ int timerlat_tracer; /* timerlat tracer */ #endif - bool tainted; /* infor users and developers about a problem */ + bool tainted; /* info users and developers about a problem */ } osnoise_data = { .sample_period = DEFAULT_SAMPLE_PERIOD, .sample_runtime = DEFAULT_SAMPLE_RUNTIME, @@ -738,7 +738,7 @@ cond_move_thread_delta_start(struct osnoise_variables *osn_var, u64 duration) /* * get_int_safe_duration - Get the duration of a window * - * The irq, softirq and thread varaibles need to have its duration without + * The irq, softirq and thread variables need to have its duration without * the interference from higher priority interrupts. Instead of keeping a * variable to discount the interrupt interference from these variables, the * starting time of these variables are pushed forward with the interrupt's @@ -1460,7 +1460,7 @@ static int run_osnoise(void) stop_in = osnoise_data.stop_tracing * NSEC_PER_USEC; /* - * Start timestemp + * Start timestamp */ start = time_get(); @@ -1881,7 +1881,7 @@ static int timerlat_main(void *data) tlat->kthread = current; osn_var->pid = current->pid; /* - * Anotate the arrival time. + * Annotate the arrival time. */ tlat->abs_period = hrtimer_cb_get_time(&tlat->timer); @@ -1978,7 +1978,7 @@ static void stop_per_cpu_kthreads(void) } /* - * start_kthread - Start a workload tread + * start_kthread - Start a workload thread */ static int start_kthread(unsigned int cpu) { @@ -2705,7 +2705,7 @@ static int osnoise_create_cpu_timerlat_fd(struct dentry *top_dir) * Why not using tracing instance per_cpu/ dir? * * Because osnoise/timerlat have a single workload, having - * multiple files like these are wast of memory. + * multiple files like these are waste of memory. */ per_cpu = tracefs_create_dir("per_cpu", top_dir); if (!per_cpu) diff --git a/kernel/trace/trace_probe.c b/kernel/trace/trace_probe.c index bb67f6a2136c..2f571083ce9e 100644 --- a/kernel/trace/trace_probe.c +++ b/kernel/trace/trace_probe.c @@ -517,7 +517,7 @@ static void clear_btf_context(struct traceprobe_parse_context *ctx) } } -/* Return 1 if the field separater is arrow operator ('->') */ +/* Return 1 if the field separator is arrow operator ('->') */ static int split_next_field(char *varname, char **next_field, struct traceprobe_parse_context *ctx) { diff --git a/kernel/trace/trace_seq.c b/kernel/trace/trace_seq.c index c158d65a8a88..32684ef4fb9d 100644 --- a/kernel/trace/trace_seq.c +++ b/kernel/trace/trace_seq.c @@ -15,7 +15,7 @@ * * A write to the buffer will either succeed or fail. That is, unlike * sprintf() there will not be a partial write (well it may write into - * the buffer but it wont update the pointers). This allows users to + * the buffer but it won't update the pointers). This allows users to * try to write something into the trace_seq buffer and if it fails * they can flush it and try again. * diff --git a/kernel/vmcore_info.c b/kernel/vmcore_info.c index e066d31d08f8..fe9bf8db1922 100644 --- a/kernel/vmcore_info.c +++ b/kernel/vmcore_info.c @@ -31,6 +31,13 @@ u32 *vmcoreinfo_note; /* trusted vmcoreinfo, e.g. we can make a copy in the crash memory */ static unsigned char *vmcoreinfo_data_safecopy; +struct hwerr_info { + atomic_t count; + time64_t timestamp; +}; + +static struct hwerr_info hwerr_data[HWERR_RECOV_MAX]; + Elf_Word *append_elf_note(Elf_Word *buf, char *name, unsigned int type, void *data, size_t data_len) { @@ -118,6 +125,16 @@ phys_addr_t __weak paddr_vmcoreinfo_note(void) } EXPORT_SYMBOL(paddr_vmcoreinfo_note); +void hwerr_log_error_type(enum hwerr_error_type src) +{ + if (src < 0 || src >= HWERR_RECOV_MAX) + return; + + atomic_inc(&hwerr_data[src].count); + WRITE_ONCE(hwerr_data[src].timestamp, ktime_get_real_seconds()); +} +EXPORT_SYMBOL_GPL(hwerr_log_error_type); + static int __init crash_save_vmcoreinfo_init(void) { vmcoreinfo_data = (unsigned char *)get_zeroed_page(GFP_KERNEL); diff --git a/kernel/watchdog.c b/kernel/watchdog.c index a567600cf3ed..0685e3a8aa0a 100644 --- a/kernel/watchdog.c +++ b/kernel/watchdog.c @@ -25,6 +25,7 @@ #include <linux/stop_machine.h> #include <linux/sysctl.h> #include <linux/tick.h> +#include <linux/sys_info.h> #include <linux/sched/clock.h> #include <linux/sched/debug.h> @@ -65,6 +66,13 @@ int __read_mostly sysctl_hardlockup_all_cpu_backtrace; unsigned int __read_mostly hardlockup_panic = IS_ENABLED(CONFIG_BOOTPARAM_HARDLOCKUP_PANIC); +/* + * bitmasks to control what kinds of system info to be printed when + * hard lockup is detected, it could be task, memory, lock etc. + * Refer include/linux/sys_info.h for detailed bit definition. + */ +static unsigned long hardlockup_si_mask; + #ifdef CONFIG_SYSFS static unsigned int hardlockup_count; @@ -178,11 +186,15 @@ static void watchdog_hardlockup_kick(void) void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs) { + int hardlockup_all_cpu_backtrace; + if (per_cpu(watchdog_hardlockup_touched, cpu)) { per_cpu(watchdog_hardlockup_touched, cpu) = false; return; } + hardlockup_all_cpu_backtrace = (hardlockup_si_mask & SYS_INFO_ALL_BT) ? + 1 : sysctl_hardlockup_all_cpu_backtrace; /* * Check for a hardlockup by making sure the CPU's timer * interrupt is incrementing. The timer interrupt should have @@ -214,7 +226,7 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs) * Prevent multiple hard-lockup reports if one cpu is already * engaged in dumping all cpu back traces. */ - if (sysctl_hardlockup_all_cpu_backtrace) { + if (hardlockup_all_cpu_backtrace) { if (test_and_set_bit_lock(0, &hard_lockup_nmi_warn)) return; } @@ -243,12 +255,13 @@ void watchdog_hardlockup_check(unsigned int cpu, struct pt_regs *regs) trigger_single_cpu_backtrace(cpu); } - if (sysctl_hardlockup_all_cpu_backtrace) { + if (hardlockup_all_cpu_backtrace) { trigger_allbutcpu_cpu_backtrace(cpu); if (!hardlockup_panic) clear_bit_unlock(0, &hard_lockup_nmi_warn); } + sys_info(hardlockup_si_mask & ~SYS_INFO_ALL_BT); if (hardlockup_panic) nmi_panic(regs, "Hard LOCKUP"); @@ -339,6 +352,13 @@ static void lockup_detector_update_enable(void) int __read_mostly sysctl_softlockup_all_cpu_backtrace; #endif +/* + * bitmasks to control what kinds of system info to be printed when + * soft lockup is detected, it could be task, memory, lock etc. + * Refer include/linux/sys_info.h for detailed bit definition. + */ +static unsigned long softlockup_si_mask; + static struct cpumask watchdog_allowed_mask __read_mostly; /* Global variables, exported for sysctl */ @@ -755,7 +775,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) unsigned long touch_ts, period_ts, now; struct pt_regs *regs = get_irq_regs(); int duration; - int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace; + int softlockup_all_cpu_backtrace; unsigned long flags; if (!watchdog_enabled) @@ -767,6 +787,9 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) if (panic_in_progress()) return HRTIMER_NORESTART; + softlockup_all_cpu_backtrace = (softlockup_si_mask & SYS_INFO_ALL_BT) ? + 1 : sysctl_softlockup_all_cpu_backtrace; + watchdog_hardlockup_kick(); /* kick the softlockup detector */ @@ -855,6 +878,7 @@ static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer) } add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK); + sys_info(softlockup_si_mask & ~SYS_INFO_ALL_BT); if (softlockup_panic) panic("softlockup: hung tasks"); } @@ -1206,6 +1230,13 @@ static const struct ctl_table watchdog_sysctls[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "softlockup_sys_info", + .data = &softlockup_si_mask, + .maxlen = sizeof(softlockup_si_mask), + .mode = 0644, + .proc_handler = sysctl_sys_info_handler, + }, #ifdef CONFIG_SMP { .procname = "softlockup_all_cpu_backtrace", @@ -1228,6 +1259,13 @@ static const struct ctl_table watchdog_sysctls[] = { .extra1 = SYSCTL_ZERO, .extra2 = SYSCTL_ONE, }, + { + .procname = "hardlockup_sys_info", + .data = &hardlockup_si_mask, + .maxlen = sizeof(hardlockup_si_mask), + .mode = 0644, + .proc_handler = sysctl_sys_info_handler, + }, #ifdef CONFIG_SMP { .procname = "hardlockup_all_cpu_backtrace", diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index c2654075377e..ba36939fda79 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -342,8 +342,7 @@ config DEBUG_INFO_COMPRESSED_ZLIB depends on $(cc-option,-gz=zlib) depends on $(ld-option,--compress-debug-sections=zlib) help - Compress the debug information using zlib. Requires GCC 5.0+ or Clang - 5.0+, binutils 2.26+, and zlib. + Compress the debug information using zlib. Users of dpkg-deb via debian/rules may find an increase in size of their debug .deb packages with this config set, due to the @@ -493,23 +492,23 @@ config DEBUG_SECTION_MISMATCH bool "Enable full Section mismatch analysis" depends on CC_IS_GCC help - The section mismatch analysis checks if there are illegal - references from one section to another section. - During linktime or runtime, some sections are dropped; - any use of code/data previously in these sections would - most likely result in an oops. - In the code, functions and variables are annotated with - __init,, etc. (see the full list in include/linux/init.h), - which results in the code/data being placed in specific sections. + The section mismatch analysis checks if there are illegal references + from one section to another. During linktime or runtime, some + sections are dropped; any use of code/data previously in these + sections would most likely result in an oops. + + In the code, functions and variables are annotated with __init, + __initdata, and so on (see the full list in include/linux/init.h). + This directs the toolchain to place code/data in specific sections. + The section mismatch analysis is always performed after a full - kernel build, and enabling this option causes the following - additional step to occur: - - Add the option -fno-inline-functions-called-once to gcc commands. - When inlining a function annotated with __init in a non-init - function, we would lose the section information and thus - the analysis would not catch the illegal reference. - This option tells gcc to inline less (but it does result in - a larger kernel). + kernel build, and enabling this option causes the option + -fno-inline-functions-called-once to be added to gcc commands. + + However, when inlining a function annotated with __init in + a non-init function, we would lose the section information and thus + the analysis would not catch the illegal reference. This option + tells gcc to inline less (but it does result in a larger kernel). config SECTION_MISMATCH_WARN_ONLY bool "Make section mismatch errors non-fatal" @@ -1260,12 +1259,13 @@ config DEFAULT_HUNG_TASK_TIMEOUT Keeping the default should be fine in most cases. config BOOTPARAM_HUNG_TASK_PANIC - bool "Panic (Reboot) On Hung Tasks" + int "Number of hung tasks to trigger kernel panic" depends on DETECT_HUNG_TASK + default 0 help - Say Y here to enable the kernel to panic on "hung tasks", - which are bugs that cause the kernel to leave a task stuck - in uninterruptible "D" state. + When set to a non-zero value, a kernel panic will be triggered + if the number of hung tasks found during a single scan reaches + this value. The panic can be used in combination with panic_timeout, to cause the system to reboot automatically after a @@ -2817,8 +2817,25 @@ config CMDLINE_KUNIT_TEST If unsure, say N. +config BASE64_KUNIT + tristate "KUnit test for base64 decoding and encoding" if !KUNIT_ALL_TESTS + depends on KUNIT + default KUNIT_ALL_TESTS + help + This builds the base64 unit tests. + + The tests cover the encoding and decoding logic of Base64 functions + in the kernel. + In addition to correctness checks, simple performance benchmarks + for both encoding and decoding are also included. + + For more information on KUnit and unit tests in general please refer + to the KUnit documentation in Documentation/dev-tools/kunit/. + + If unsure, say N. + config BITS_TEST - tristate "KUnit test for bits.h" if !KUNIT_ALL_TESTS + tristate "KUnit test for bit functions and macros" if !KUNIT_ALL_TESTS depends on KUNIT default KUNIT_ALL_TESTS help diff --git a/lib/base64.c b/lib/base64.c index b736a7a431c5..41961a444028 100644 --- a/lib/base64.c +++ b/lib/base64.c @@ -1,12 +1,12 @@ // SPDX-License-Identifier: GPL-2.0 /* - * base64.c - RFC4648-compliant base64 encoding + * base64.c - Base64 with support for multiple variants * * Copyright (c) 2020 Hannes Reinecke, SUSE * * Based on the base64url routines from fs/crypto/fname.c - * (which are using the URL-safe base64 encoding), - * modified to use the standard coding table from RFC4648 section 4. + * (which are using the URL-safe Base64 encoding), + * modified to support multiple Base64 variants. */ #include <linux/kernel.h> @@ -15,89 +15,170 @@ #include <linux/string.h> #include <linux/base64.h> -static const char base64_table[65] = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +static const char base64_tables[][65] = { + [BASE64_STD] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/", + [BASE64_URLSAFE] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", + [BASE64_IMAP] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+,", +}; +/* + * Initialize the base64 reverse mapping for a single character + * This macro maps a character to its corresponding base64 value, + * returning -1 if the character is invalid. + * char 'A'-'Z' maps to 0-25, 'a'-'z' maps to 26-51, '0'-'9' maps to 52-61, + * ch_62 maps to 62, ch_63 maps to 63, and other characters return -1 + */ +#define INIT_1(v, ch_62, ch_63) \ + [v] = (v) >= 'A' && (v) <= 'Z' ? (v) - 'A' \ + : (v) >= 'a' && (v) <= 'z' ? (v) - 'a' + 26 \ + : (v) >= '0' && (v) <= '9' ? (v) - '0' + 52 \ + : (v) == (ch_62) ? 62 : (v) == (ch_63) ? 63 : -1 + +/* + * Recursive macros to generate multiple Base64 reverse mapping table entries. + * Each macro generates a sequence of entries in the lookup table: + * INIT_2 generates 2 entries, INIT_4 generates 4, INIT_8 generates 8, and so on up to INIT_32. + */ +#define INIT_2(v, ...) INIT_1(v, __VA_ARGS__), INIT_1((v) + 1, __VA_ARGS__) +#define INIT_4(v, ...) INIT_2(v, __VA_ARGS__), INIT_2((v) + 2, __VA_ARGS__) +#define INIT_8(v, ...) INIT_4(v, __VA_ARGS__), INIT_4((v) + 4, __VA_ARGS__) +#define INIT_16(v, ...) INIT_8(v, __VA_ARGS__), INIT_8((v) + 8, __VA_ARGS__) +#define INIT_32(v, ...) INIT_16(v, __VA_ARGS__), INIT_16((v) + 16, __VA_ARGS__) + +#define BASE64_REV_INIT(ch_62, ch_63) { \ + [0 ... 0x1f] = -1, \ + INIT_32(0x20, ch_62, ch_63), \ + INIT_32(0x40, ch_62, ch_63), \ + INIT_32(0x60, ch_62, ch_63), \ + [0x80 ... 0xff] = -1 } + +static const s8 base64_rev_maps[][256] = { + [BASE64_STD] = BASE64_REV_INIT('+', '/'), + [BASE64_URLSAFE] = BASE64_REV_INIT('-', '_'), + [BASE64_IMAP] = BASE64_REV_INIT('+', ',') +}; + +#undef BASE64_REV_INIT +#undef INIT_32 +#undef INIT_16 +#undef INIT_8 +#undef INIT_4 +#undef INIT_2 +#undef INIT_1 /** - * base64_encode() - base64-encode some binary data + * base64_encode() - Base64-encode some binary data * @src: the binary data to encode * @srclen: the length of @src in bytes - * @dst: (output) the base64-encoded string. Not NUL-terminated. + * @dst: (output) the Base64-encoded string. Not NUL-terminated. + * @padding: whether to append '=' padding characters + * @variant: which base64 variant to use * - * Encodes data using base64 encoding, i.e. the "Base 64 Encoding" specified - * by RFC 4648, including the '='-padding. + * Encodes data using the selected Base64 variant. * - * Return: the length of the resulting base64-encoded string in bytes. + * Return: the length of the resulting Base64-encoded string in bytes. */ -int base64_encode(const u8 *src, int srclen, char *dst) +int base64_encode(const u8 *src, int srclen, char *dst, bool padding, enum base64_variant variant) { u32 ac = 0; - int bits = 0; - int i; char *cp = dst; + const char *base64_table = base64_tables[variant]; - for (i = 0; i < srclen; i++) { - ac = (ac << 8) | src[i]; - bits += 8; - do { - bits -= 6; - *cp++ = base64_table[(ac >> bits) & 0x3f]; - } while (bits >= 6); - } - if (bits) { - *cp++ = base64_table[(ac << (6 - bits)) & 0x3f]; - bits -= 6; + while (srclen >= 3) { + ac = src[0] << 16 | src[1] << 8 | src[2]; + *cp++ = base64_table[ac >> 18]; + *cp++ = base64_table[(ac >> 12) & 0x3f]; + *cp++ = base64_table[(ac >> 6) & 0x3f]; + *cp++ = base64_table[ac & 0x3f]; + + src += 3; + srclen -= 3; } - while (bits < 0) { - *cp++ = '='; - bits += 2; + + switch (srclen) { + case 2: + ac = src[0] << 16 | src[1] << 8; + *cp++ = base64_table[ac >> 18]; + *cp++ = base64_table[(ac >> 12) & 0x3f]; + *cp++ = base64_table[(ac >> 6) & 0x3f]; + if (padding) + *cp++ = '='; + break; + case 1: + ac = src[0] << 16; + *cp++ = base64_table[ac >> 18]; + *cp++ = base64_table[(ac >> 12) & 0x3f]; + if (padding) { + *cp++ = '='; + *cp++ = '='; + } + break; } return cp - dst; } EXPORT_SYMBOL_GPL(base64_encode); /** - * base64_decode() - base64-decode a string + * base64_decode() - Base64-decode a string * @src: the string to decode. Doesn't need to be NUL-terminated. * @srclen: the length of @src in bytes * @dst: (output) the decoded binary data + * @padding: whether to append '=' padding characters + * @variant: which base64 variant to use * - * Decodes a string using base64 encoding, i.e. the "Base 64 Encoding" - * specified by RFC 4648, including the '='-padding. - * - * This implementation hasn't been optimized for performance. + * Decodes a string using the selected Base64 variant. * * Return: the length of the resulting decoded binary data in bytes, - * or -1 if the string isn't a valid base64 string. + * or -1 if the string isn't a valid Base64 string. */ -int base64_decode(const char *src, int srclen, u8 *dst) +int base64_decode(const char *src, int srclen, u8 *dst, bool padding, enum base64_variant variant) { - u32 ac = 0; - int bits = 0; - int i; u8 *bp = dst; + s8 input[4]; + s32 val; + const u8 *s = (const u8 *)src; + const s8 *base64_rev_tables = base64_rev_maps[variant]; - for (i = 0; i < srclen; i++) { - const char *p = strchr(base64_table, src[i]); + while (srclen >= 4) { + input[0] = base64_rev_tables[s[0]]; + input[1] = base64_rev_tables[s[1]]; + input[2] = base64_rev_tables[s[2]]; + input[3] = base64_rev_tables[s[3]]; - if (src[i] == '=') { - ac = (ac << 6); - bits += 6; - if (bits >= 8) - bits -= 8; - continue; - } - if (p == NULL || src[i] == 0) - return -1; - ac = (ac << 6) | (p - base64_table); - bits += 6; - if (bits >= 8) { - bits -= 8; - *bp++ = (u8)(ac >> bits); + val = input[0] << 18 | input[1] << 12 | input[2] << 6 | input[3]; + + if (unlikely(val < 0)) { + if (!padding || srclen != 4 || s[3] != '=') + return -1; + padding = 0; + srclen = s[2] == '=' ? 2 : 3; + break; } + + *bp++ = val >> 16; + *bp++ = val >> 8; + *bp++ = val; + + s += 4; + srclen -= 4; } - if (ac & ((1 << bits) - 1)) + + if (likely(!srclen)) + return bp - dst; + if (padding || srclen == 1) return -1; + + val = (base64_rev_tables[s[0]] << 12) | (base64_rev_tables[s[1]] << 6); + *bp++ = val >> 10; + + if (srclen == 2) { + if (val & 0x800003ff) + return -1; + } else { + val |= base64_rev_tables[s[2]]; + if (val & 0x80000003) + return -1; + *bp++ = val >> 2; + } return bp - dst; } EXPORT_SYMBOL_GPL(base64_decode); diff --git a/lib/dynamic_debug.c b/lib/dynamic_debug.c index 5a007952f7f2..7d7892e57a01 100644 --- a/lib/dynamic_debug.c +++ b/lib/dynamic_debug.c @@ -95,6 +95,7 @@ static const struct { unsigned flag:8; char opt_char; } opt_array[] = { { _DPRINTK_FLAGS_INCL_SOURCENAME, 's' }, { _DPRINTK_FLAGS_INCL_LINENO, 'l' }, { _DPRINTK_FLAGS_INCL_TID, 't' }, + { _DPRINTK_FLAGS_INCL_STACK, 'd' }, { _DPRINTK_FLAGS_NONE, '_' }, }; diff --git a/lib/fonts/Kconfig b/lib/fonts/Kconfig index ae59b5b4e225..7d03823e46dc 100644 --- a/lib/fonts/Kconfig +++ b/lib/fonts/Kconfig @@ -112,6 +112,17 @@ config FONT_SUN12x22 big letters (like the letters used in the SPARC PROM). If the standard font is unreadable for you, say Y, otherwise say N. +config FONT_TER10x18 + bool "Terminus 10x18 font (not supported by all drivers)" + depends on FRAMEBUFFER_CONSOLE || DRM_PANIC + depends on !SPARC && FONTS || SPARC + help + Terminus Font is a clean, fixed width bitmap font, designed + for long (8 and more hours per day) work with computers. + This is the high resolution version made for use with 13-16" laptops. + It fits between the normal 8x16 font and Terminus 16x32. + If other fonts are unreadable for you, say Y, otherwise say N. + config FONT_TER16x32 bool "Terminus 16x32 font (not supported by all drivers)" depends on FRAMEBUFFER_CONSOLE || DRM_PANIC @@ -140,6 +151,7 @@ config FONT_AUTOSELECT depends on !FONT_SUN8x16 depends on !FONT_SUN12x22 depends on !FONT_10x18 + depends on !FONT_TER10x18 depends on !FONT_TER16x32 depends on !FONT_6x8 select FONT_8x16 diff --git a/lib/fonts/Makefile b/lib/fonts/Makefile index e16f68492174..30a85a4292fa 100644 --- a/lib/fonts/Makefile +++ b/lib/fonts/Makefile @@ -14,6 +14,7 @@ font-objs-$(CONFIG_FONT_PEARL_8x8) += font_pearl_8x8.o font-objs-$(CONFIG_FONT_ACORN_8x8) += font_acorn_8x8.o font-objs-$(CONFIG_FONT_MINI_4x6) += font_mini_4x6.o font-objs-$(CONFIG_FONT_6x10) += font_6x10.o +font-objs-$(CONFIG_FONT_TER10x18) += font_ter10x18.o font-objs-$(CONFIG_FONT_TER16x32) += font_ter16x32.o font-objs-$(CONFIG_FONT_6x8) += font_6x8.o diff --git a/lib/fonts/font_ter10x18.c b/lib/fonts/font_ter10x18.c new file mode 100644 index 000000000000..80356e9d56c7 --- /dev/null +++ b/lib/fonts/font_ter10x18.c @@ -0,0 +1,5143 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/font.h> +#include <linux/module.h> + +#define FONTDATAMAX 9216 + +static const struct font_data fontdata_ter10x18 = { + { 0, 0, FONTDATAMAX, 0 }, { + /* 0 0x00 '^@' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 1 0x01 '^A' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x80, 0x40, /* #--------# */ + 0x80, 0x40, /* #--------# */ + 0xb3, 0x40, /* #-##--##-# */ + 0xb3, 0x40, /* #-##--##-# */ + 0x80, 0x40, /* #--------# */ + 0x80, 0x40, /* #--------# */ + 0xbf, 0x40, /* #-######-# */ + 0x9e, 0x40, /* #--####--# */ + 0x80, 0x40, /* #--------# */ + 0x80, 0x40, /* #--------# */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 2 0x02 '^B' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xcc, 0xc0, /* ##--##--## */ + 0xcc, 0xc0, /* ##--##--## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xc0, 0xc0, /* ##------## */ + 0xe1, 0xc0, /* ###----### */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 3 0x03 '^C' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x73, 0x80, /* -###--###- */ + 0xf3, 0xc0, /* ####--#### */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x7f, 0x80, /* -########- */ + 0x3f, 0x00, /* --######-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 4 0x04 '^D' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x7f, 0x80, /* -########- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x7f, 0x80, /* -########- */ + 0x3f, 0x00, /* --######-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 5 0x05 '^E' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x3f, 0x00, /* --######-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x6d, 0x80, /* -##-##-##- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x6d, 0x80, /* -##-##-##- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 6 0x06 '^F' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x7f, 0x80, /* -########- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x6d, 0x80, /* -##-##-##- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 7 0x07 '^G' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 8 0x08 '^H' */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xf3, 0xc0, /* ####--#### */ + 0xe1, 0xc0, /* ###----### */ + 0xe1, 0xc0, /* ###----### */ + 0xf3, 0xc0, /* ####--#### */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + + /* 9 0x09 '^I' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x21, 0x00, /* --#----#-- */ + 0x21, 0x00, /* --#----#-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 10 0x0a '^J' */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xe1, 0xc0, /* ###----### */ + 0xcc, 0xc0, /* ##--##--## */ + 0xde, 0xc0, /* ##-####-## */ + 0xde, 0xc0, /* ##-####-## */ + 0xcc, 0xc0, /* ##--##--## */ + 0xe1, 0xc0, /* ###----### */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + + /* 11 0x0b '^K' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0f, 0x80, /* ----#####- */ + 0x03, 0x80, /* ------###- */ + 0x06, 0x80, /* -----##-#- */ + 0x0c, 0x80, /* ----##--#- */ + 0x3e, 0x00, /* --#####--- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x3e, 0x00, /* --#####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 12 0x0c '^L' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x7f, 0x80, /* -########- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 13 0x0d '^M' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x80, /* --#######- */ + 0x31, 0x80, /* --##---##- */ + 0x31, 0x80, /* --##---##- */ + 0x3f, 0x80, /* --#######- */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0xf0, 0x00, /* ####------ */ + 0xe0, 0x00, /* ###------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 14 0x0e '^N' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x63, 0x80, /* -##---###- */ + 0xe3, 0x00, /* ###---##-- */ + 0xc0, 0x00, /* ##-------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 15 0x0f '^O' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0xcc, 0xc0, /* ##--##--## */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3f, 0x00, /* --######-- */ + 0x1e, 0x00, /* ---####--- */ + 0xf3, 0xc0, /* ####--#### */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0xcc, 0xc0, /* ##--##--## */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 16 0x10 '^P' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xc0, 0x00, /* ##-------- */ + 0xf0, 0x00, /* ####------ */ + 0xfc, 0x00, /* ######---- */ + 0xff, 0x00, /* ########-- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0x00, /* ########-- */ + 0xfc, 0x00, /* ######---- */ + 0xf0, 0x00, /* ####------ */ + 0xc0, 0x00, /* ##-------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 17 0x11 '^Q' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0xc0, /* --------## */ + 0x03, 0xc0, /* ------#### */ + 0x0f, 0xc0, /* ----###### */ + 0x3f, 0xc0, /* --######## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x3f, 0xc0, /* --######## */ + 0x0f, 0xc0, /* ----###### */ + 0x03, 0xc0, /* ------#### */ + 0x00, 0xc0, /* --------## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 18 0x12 '^R' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3f, 0x00, /* --######-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 19 0x13 '^S' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 20 0x14 '^T' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x80, /* --#######- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3d, 0x80, /* --####-##- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0d, 0x80, /* ----##-##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 21 0x15 '^U' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x30, 0x00, /* --##------ */ + 0x1c, 0x00, /* ---###---- */ + 0x36, 0x00, /* --##-##--- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1b, 0x00, /* ---##-##-- */ + 0x0e, 0x00, /* ----###--- */ + 0x03, 0x00, /* ------##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 22 0x16 '^V' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xff, 0x80, /* #########- */ + 0xff, 0x80, /* #########- */ + 0xff, 0x80, /* #########- */ + 0xff, 0x80, /* #########- */ + 0xff, 0x80, /* #########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 23 0x17 '^W' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3f, 0x00, /* --######-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 24 0x18 '^X' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 25 0x19 '^Y' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3f, 0x00, /* --######-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 26 0x1a '^Z' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x04, 0x00, /* -----#---- */ + 0x06, 0x00, /* -----##--- */ + 0x03, 0x00, /* ------##-- */ + 0xff, 0x80, /* #########- */ + 0xff, 0x80, /* #########- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x04, 0x00, /* -----#---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 27 0x1b '^[' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x10, 0x00, /* ---#------ */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0xff, 0x80, /* #########- */ + 0xff, 0x80, /* #########- */ + 0x60, 0x00, /* -##------- */ + 0x30, 0x00, /* --##------ */ + 0x10, 0x00, /* ---#------ */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 28 0x1c '^\' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 29 0x1d '^]' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x12, 0x00, /* ---#--#--- */ + 0x33, 0x00, /* --##--##-- */ + 0x61, 0x80, /* -##----##- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x12, 0x00, /* ---#--#--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 30 0x1e '^^' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x3f, 0x00, /* --######-- */ + 0x7f, 0x80, /* -########- */ + 0x7f, 0x80, /* -########- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 31 0x1f '^_' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x7f, 0x80, /* -########- */ + 0x7f, 0x80, /* -########- */ + 0x3f, 0x00, /* --######-- */ + 0x3f, 0x00, /* --######-- */ + 0x1e, 0x00, /* ---####--- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 32 0x20 ' ' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 33 0x21 '!' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 34 0x22 '"' */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 35 0x23 '#' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x7f, 0x80, /* -########- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x7f, 0x80, /* -########- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 36 0x24 '$' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6c, 0x00, /* -##-##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x3f, 0x00, /* --######-- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0d, 0x80, /* ----##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + + /* 37 0x25 '%' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x73, 0x00, /* -###--##-- */ + 0x53, 0x00, /* -#-#--##-- */ + 0x76, 0x00, /* -###-##--- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x37, 0x00, /* --##-###-- */ + 0x65, 0x00, /* -##--#-#-- */ + 0x67, 0x00, /* -##--###-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 38 0x26 '&' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3c, 0x00, /* --####---- */ + 0x66, 0x00, /* -##--##--- */ + 0x66, 0x00, /* -##--##--- */ + 0x66, 0x00, /* -##--##--- */ + 0x3c, 0x00, /* --####---- */ + 0x39, 0x80, /* --###--##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0xc7, 0x00, /* ##---###-- */ + 0xc3, 0x00, /* ##----##-- */ + 0xc3, 0x00, /* ##----##-- */ + 0x67, 0x80, /* -##--####- */ + 0x3d, 0x80, /* --####-##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 39 0x27 ''' */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 40 0x28 '(' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x06, 0x00, /* -----##--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 41 0x29 ')' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 42 0x2a '*' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x63, 0x00, /* -##---##-- */ + 0x36, 0x00, /* --##-##--- */ + 0x1c, 0x00, /* ---###---- */ + 0xff, 0x80, /* #########- */ + 0x1c, 0x00, /* ---###---- */ + 0x36, 0x00, /* --##-##--- */ + 0x63, 0x00, /* -##---##-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 43 0x2b '+' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x7f, 0x80, /* -########- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 44 0x2c ',' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 45 0x2d '-' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 46 0x2e '.' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 47 0x2f '/' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 48 0x30 '0' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x63, 0x80, /* -##---###- */ + 0x67, 0x80, /* -##--####- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x79, 0x80, /* -####--##- */ + 0x71, 0x80, /* -###---##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 49 0x31 '1' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x1c, 0x00, /* ---###---- */ + 0x3c, 0x00, /* --####---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 50 0x32 '2' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x01, 0x80, /* -------##- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 51 0x33 '3' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x1f, 0x00, /* ---#####-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 52 0x34 '4' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x01, 0x80, /* -------##- */ + 0x03, 0x80, /* ------###- */ + 0x07, 0x80, /* -----####- */ + 0x0d, 0x80, /* ----##-##- */ + 0x19, 0x80, /* ---##--##- */ + 0x31, 0x80, /* --##---##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 53 0x35 '5' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x00, /* -#######-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 54 0x36 '6' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1f, 0x00, /* ---#####-- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 55 0x37 '7' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x01, 0x80, /* -------##- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 56 0x38 '8' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 57 0x39 '9' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x03, 0x00, /* ------##-- */ + 0x3e, 0x00, /* --#####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 58 0x3a ':' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 59 0x3b ';' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 60 0x3c '<' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x30, 0x00, /* --##------ */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x06, 0x00, /* -----##--- */ + 0x03, 0x00, /* ------##-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 61 0x3d '=' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 62 0x3e '>' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x60, 0x00, /* -##------- */ + 0x30, 0x00, /* --##------ */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x06, 0x00, /* -----##--- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 63 0x3f '?' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x01, 0x80, /* -------##- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 64 0x40 '@' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xcf, 0x80, /* ##--#####- */ + 0xd9, 0x80, /* ##-##--##- */ + 0xd9, 0x80, /* ##-##--##- */ + 0xd9, 0x80, /* ##-##--##- */ + 0xd9, 0x80, /* ##-##--##- */ + 0xcf, 0x80, /* ##--#####- */ + 0xc0, 0x00, /* ##-------- */ + 0xc0, 0x00, /* ##-------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 65 0x41 'A' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 66 0x42 'B' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x00, /* -#######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 67 0x43 'C' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 68 0x44 'D' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7e, 0x00, /* -######--- */ + 0x63, 0x00, /* -##---##-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x63, 0x00, /* -##---##-- */ + 0x7e, 0x00, /* -######--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 69 0x45 'E' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7e, 0x00, /* -######--- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 70 0x46 'F' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7e, 0x00, /* -######--- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 71 0x47 'G' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x67, 0x80, /* -##--####- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 72 0x48 'H' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 73 0x49 'I' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 74 0x4a 'J' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x07, 0x80, /* -----####- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x3e, 0x00, /* --#####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 75 0x4b 'K' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x63, 0x00, /* -##---##-- */ + 0x66, 0x00, /* -##--##--- */ + 0x6c, 0x00, /* -##-##---- */ + 0x78, 0x00, /* -####----- */ + 0x78, 0x00, /* -####----- */ + 0x6c, 0x00, /* -##-##---- */ + 0x66, 0x00, /* -##--##--- */ + 0x63, 0x00, /* -##---##-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 76 0x4c 'L' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 77 0x4d 'M' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x80, 0x80, /* #-------#- */ + 0xc1, 0x80, /* ##-----##- */ + 0xe3, 0x80, /* ###---###- */ + 0xf7, 0x80, /* ####-####- */ + 0xdd, 0x80, /* ##-###-##- */ + 0xc9, 0x80, /* ##--#--##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 78 0x4e 'N' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x71, 0x80, /* -###---##- */ + 0x79, 0x80, /* -####--##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x67, 0x80, /* -##--####- */ + 0x63, 0x80, /* -##---###- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 79 0x4f 'O' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 80 0x50 'P' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x00, /* -#######-- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 81 0x51 'Q' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x67, 0x80, /* -##--####- */ + 0x3f, 0x00, /* --######-- */ + 0x03, 0x00, /* ------##-- */ + 0x01, 0x80, /* -------##- */ + 0x00, 0x00, /* ---------- */ + + /* 82 0x52 'R' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x00, /* -#######-- */ + 0x78, 0x00, /* -####----- */ + 0x6c, 0x00, /* -##-##---- */ + 0x66, 0x00, /* -##--##--- */ + 0x63, 0x00, /* -##---##-- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 83 0x53 'S' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x3f, 0x00, /* --######-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 84 0x54 'T' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 85 0x55 'U' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 86 0x56 'V' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 87 0x57 'W' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc9, 0x80, /* ##--#--##- */ + 0xdd, 0x80, /* ##-###-##- */ + 0xf7, 0x80, /* ####-####- */ + 0xe3, 0x80, /* ###---###- */ + 0xc1, 0x80, /* ##-----##- */ + 0x80, 0x80, /* #-------#- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 88 0x58 'X' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 89 0x59 'Y' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 90 0x5a 'Z' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 91 0x5b '[' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 92 0x5c '\' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 93 0x5d ']' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 94 0x5e '^' */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 95 0x5f '_' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + + /* 96 0x60 '`' */ + 0x30, 0x00, /* --##------ */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 97 0x61 'a' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x80, /* --#######- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 98 0x62 'b' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x00, /* -#######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 99 0x63 'c' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 100 0x64 'd' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x80, /* --#######- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 101 0x65 'e' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 102 0x66 'f' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x07, 0x80, /* -----####- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 103 0x67 'g' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x80, /* --#######- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x00, /* --######-- */ + + /* 104 0x68 'h' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 105 0x69 'i' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x1c, 0x00, /* ---###---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 106 0x6a 'j' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x00, 0x00, /* ---------- */ + 0x07, 0x00, /* -----###-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + + /* 107 0x6b 'k' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x63, 0x00, /* -##---##-- */ + 0x66, 0x00, /* -##--##--- */ + 0x6c, 0x00, /* -##-##---- */ + 0x78, 0x00, /* -####----- */ + 0x6c, 0x00, /* -##-##---- */ + 0x66, 0x00, /* -##--##--- */ + 0x63, 0x00, /* -##---##-- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 108 0x6c 'l' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1c, 0x00, /* ---###---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 109 0x6d 'm' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 110 0x6e 'n' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 111 0x6f 'o' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 112 0x70 'p' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x00, /* -#######-- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + + /* 113 0x71 'q' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x80, /* --#######- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + + /* 114 0x72 'r' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x6f, 0x80, /* -##-#####- */ + 0x78, 0x00, /* -####----- */ + 0x70, 0x00, /* -###------ */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 115 0x73 's' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x3f, 0x00, /* --######-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 116 0x74 't' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x7e, 0x00, /* -######--- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x18, 0x00, /* ---##----- */ + 0x0f, 0x00, /* ----####-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 117 0x75 'u' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 118 0x76 'v' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 119 0x77 'w' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 120 0x78 'x' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 121 0x79 'y' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x00, /* --######-- */ + + /* 122 0x7a 'z' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x01, 0x80, /* -------##- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 123 0x7b '{' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x07, 0x00, /* -----###-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x38, 0x00, /* --###----- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x07, 0x00, /* -----###-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 124 0x7c '|' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 125 0x7d '}' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x38, 0x00, /* --###----- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x07, 0x00, /* -----###-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x38, 0x00, /* --###----- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 126 0x7e '~' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x39, 0x80, /* --###--##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x67, 0x00, /* -##--###-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 127 0x7f '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x08, 0x00, /* ----#----- */ + 0x1c, 0x00, /* ---###---- */ + 0x36, 0x00, /* --##-##--- */ + 0x63, 0x00, /* -##---##-- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xc1, 0x80, /* ##-----##- */ + 0xff, 0x80, /* #########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 128 0x80 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + + /* 129 0x81 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 130 0x82 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 131 0x83 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x80, /* --#######- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 132 0x84 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x80, /* --#######- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 133 0x85 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x80, /* --#######- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 134 0x86 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x80, /* --#######- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 135 0x87 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + + /* 136 0x88 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 137 0x89 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 138 0x8a '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 139 0x8b '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x1c, 0x00, /* ---###---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 140 0x8c '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x1c, 0x00, /* ---###---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 141 0x8d '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x1c, 0x00, /* ---###---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 142 0x8e '' */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 143 0x8f '' */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x80, /* -########- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 144 0x90 '' */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7e, 0x00, /* -######--- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 145 0x91 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7b, 0x80, /* -####-###- */ + 0x0c, 0xc0, /* ----##--## */ + 0x0c, 0xc0, /* ----##--## */ + 0x7c, 0xc0, /* -#####--## */ + 0xcf, 0xc0, /* ##--###### */ + 0xcc, 0x00, /* ##--##---- */ + 0xcc, 0x00, /* ##--##---- */ + 0xcc, 0xc0, /* ##--##--## */ + 0x77, 0x80, /* -###-####- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 146 0x92 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0xc0, /* -######### */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xff, 0xc0, /* ########## */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc7, 0xc0, /* ##---##### */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 147 0x93 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 148 0x94 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 149 0x95 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 150 0x96 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 151 0x97 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 152 0x98 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x00, /* --######-- */ + + /* 153 0x99 '' */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 154 0x9a '' */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 155 0x9b '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6c, 0x00, /* -##-##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + + /* 156 0x9c '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x7e, 0x00, /* -######--- */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x31, 0x80, /* --##---##- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 157 0x9d '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x1e, 0x00, /* ---####--- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 158 0x9e '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xf8, 0x00, /* #####----- */ + 0xcc, 0x00, /* ##--##---- */ + 0xcc, 0x00, /* ##--##---- */ + 0xcc, 0x00, /* ##--##---- */ + 0xcc, 0x00, /* ##--##---- */ + 0xfb, 0x00, /* #####-##-- */ + 0xc3, 0x00, /* ##----##-- */ + 0xc7, 0x80, /* ##---####- */ + 0xc3, 0x00, /* ##----##-- */ + 0xc3, 0x00, /* ##----##-- */ + 0xc3, 0x00, /* ##----##-- */ + 0xc1, 0x80, /* ##-----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 159 0x9f '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x07, 0x00, /* -----###-- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x38, 0x00, /* --###----- */ + + /* 160 0xa0 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x3f, 0x80, /* --#######- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 161 0xa1 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x1c, 0x00, /* ---###---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 162 0xa2 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 163 0xa3 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x80, /* --#######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 164 0xa4 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3b, 0x80, /* --###-###- */ + 0x6e, 0x00, /* -##-###--- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 165 0xa5 '' */ + 0x3b, 0x80, /* --###-###- */ + 0x6e, 0x00, /* -##-###--- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x71, 0x80, /* -###---##- */ + 0x79, 0x80, /* -####--##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x67, 0x80, /* -##--####- */ + 0x63, 0x80, /* -##---###- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 166 0xa6 '' */ + 0x00, 0x00, /* ---------- */ + 0x3e, 0x00, /* --#####--- */ + 0x03, 0x00, /* ------##-- */ + 0x3f, 0x00, /* --######-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 167 0xa7 '' */ + 0x00, 0x00, /* ---------- */ + 0x3e, 0x00, /* --#####--- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x3e, 0x00, /* --#####--- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x00, /* -#######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 168 0xa8 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 169 0xa9 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 170 0xaa '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 171 0xab '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x30, 0x00, /* --##------ */ + 0x70, 0x00, /* -###------ */ + 0x30, 0x80, /* --##----#- */ + 0x31, 0x80, /* --##---##- */ + 0x33, 0x00, /* --##--##-- */ + 0x36, 0x00, /* --##-##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x37, 0x00, /* --##-###-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0xc1, 0x80, /* ##-----##- */ + 0x83, 0x00, /* #-----##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0f, 0x80, /* ----#####- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 172 0xac '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x30, 0x00, /* --##------ */ + 0x70, 0x00, /* -###------ */ + 0x30, 0x80, /* --##----#- */ + 0x31, 0x80, /* --##---##- */ + 0x33, 0x00, /* --##--##-- */ + 0x36, 0x00, /* --##-##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x19, 0x80, /* ---##--##- */ + 0x33, 0x80, /* --##--###- */ + 0x67, 0x80, /* -##--####- */ + 0xcd, 0x80, /* ##--##-##- */ + 0x8f, 0x80, /* #---#####- */ + 0x01, 0x80, /* -------##- */ + 0x01, 0x80, /* -------##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 173 0xad '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 174 0xae '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0xc0, /* ----##--## */ + 0x19, 0x80, /* ---##--##- */ + 0x33, 0x00, /* --##--##-- */ + 0x66, 0x00, /* -##--##--- */ + 0xcc, 0x00, /* ##--##---- */ + 0x66, 0x00, /* -##--##--- */ + 0x33, 0x00, /* --##--##-- */ + 0x19, 0x80, /* ---##--##- */ + 0x0c, 0xc0, /* ----##--## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 175 0xaf '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xcc, 0x00, /* ##--##---- */ + 0x66, 0x00, /* -##--##--- */ + 0x33, 0x00, /* --##--##-- */ + 0x19, 0x80, /* ---##--##- */ + 0x0c, 0xc0, /* ----##--## */ + 0x19, 0x80, /* ---##--##- */ + 0x33, 0x00, /* --##--##-- */ + 0x66, 0x00, /* -##--##--- */ + 0xcc, 0x00, /* ##--##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 176 0xb0 '' */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x00, 0x00, /* ---------- */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x00, 0x00, /* ---------- */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x00, 0x00, /* ---------- */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x00, 0x00, /* ---------- */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x00, 0x00, /* ---------- */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x00, 0x00, /* ---------- */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x00, 0x00, /* ---------- */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x00, 0x00, /* ---------- */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x00, 0x00, /* ---------- */ + + /* 177 0xb1 '' */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x55, 0x40, /* -#-#-#-#-# */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x55, 0x40, /* -#-#-#-#-# */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x55, 0x40, /* -#-#-#-#-# */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x55, 0x40, /* -#-#-#-#-# */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x55, 0x40, /* -#-#-#-#-# */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x55, 0x40, /* -#-#-#-#-# */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x55, 0x40, /* -#-#-#-#-# */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x55, 0x40, /* -#-#-#-#-# */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0x55, 0x40, /* -#-#-#-#-# */ + + /* 178 0xb2 '' */ + 0xff, 0xc0, /* ########## */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0xff, 0xc0, /* ########## */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0xff, 0xc0, /* ########## */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0xff, 0xc0, /* ########## */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0xff, 0xc0, /* ########## */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0xff, 0xc0, /* ########## */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0xff, 0xc0, /* ########## */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0xff, 0xc0, /* ########## */ + 0xaa, 0x80, /* #-#-#-#-#- */ + 0xff, 0xc0, /* ########## */ + 0xaa, 0x80, /* #-#-#-#-#- */ + + /* 179 0xb3 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 180 0xb4 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0xfc, 0x00, /* ######---- */ + 0xfc, 0x00, /* ######---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 181 0xb5 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0xfc, 0x00, /* ######---- */ + 0xfc, 0x00, /* ######---- */ + 0x0c, 0x00, /* ----##---- */ + 0xfc, 0x00, /* ######---- */ + 0xfc, 0x00, /* ######---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 182 0xb6 '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 183 0xb7 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xfe, 0x00, /* #######--- */ + 0xfe, 0x00, /* #######--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 184 0xb8 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xfc, 0x00, /* ######---- */ + 0xfc, 0x00, /* ######---- */ + 0x0c, 0x00, /* ----##---- */ + 0xfc, 0x00, /* ######---- */ + 0xfc, 0x00, /* ######---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 185 0xb9 '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0x06, 0x00, /* -----##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 186 0xba '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 187 0xbb '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xfe, 0x00, /* #######--- */ + 0xfe, 0x00, /* #######--- */ + 0x06, 0x00, /* -----##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 188 0xbc '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0xf6, 0x00, /* ####-##--- */ + 0x06, 0x00, /* -----##--- */ + 0xfe, 0x00, /* #######--- */ + 0xfe, 0x00, /* #######--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 189 0xbd '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0xfe, 0x00, /* #######--- */ + 0xfe, 0x00, /* #######--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 190 0xbe '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0xfc, 0x00, /* ######---- */ + 0xfc, 0x00, /* ######---- */ + 0x0c, 0x00, /* ----##---- */ + 0xfc, 0x00, /* ######---- */ + 0xfc, 0x00, /* ######---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 191 0xbf '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xfc, 0x00, /* ######---- */ + 0xfc, 0x00, /* ######---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 192 0xc0 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0f, 0xc0, /* ----###### */ + 0x0f, 0xc0, /* ----###### */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 193 0xc1 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 194 0xc2 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 195 0xc3 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0f, 0xc0, /* ----###### */ + 0x0f, 0xc0, /* ----###### */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 196 0xc4 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 197 0xc5 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 198 0xc6 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0f, 0xc0, /* ----###### */ + 0x0f, 0xc0, /* ----###### */ + 0x0c, 0x00, /* ----##---- */ + 0x0f, 0xc0, /* ----###### */ + 0x0f, 0xc0, /* ----###### */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 199 0xc7 '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x37, 0xc0, /* --##-##### */ + 0x37, 0xc0, /* --##-##### */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 200 0xc8 '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x37, 0xc0, /* --##-##### */ + 0x37, 0xc0, /* --##-##### */ + 0x30, 0x00, /* --##------ */ + 0x3f, 0xc0, /* --######## */ + 0x3f, 0xc0, /* --######## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 201 0xc9 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0xc0, /* --######## */ + 0x3f, 0xc0, /* --######## */ + 0x30, 0x00, /* --##------ */ + 0x37, 0xc0, /* --##-##### */ + 0x37, 0xc0, /* --##-##### */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 202 0xca '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0xf7, 0xc0, /* ####-##### */ + 0xf7, 0xc0, /* ####-##### */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 203 0xcb '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0xf7, 0xc0, /* ####-##### */ + 0xf7, 0xc0, /* ####-##### */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 204 0xcc '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x37, 0xc0, /* --##-##### */ + 0x37, 0xc0, /* --##-##### */ + 0x30, 0x00, /* --##------ */ + 0x37, 0xc0, /* --##-##### */ + 0x37, 0xc0, /* --##-##### */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 205 0xcd '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 206 0xce '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0xf7, 0xc0, /* ####-##### */ + 0xf7, 0xc0, /* ####-##### */ + 0x00, 0x00, /* ---------- */ + 0xf7, 0xc0, /* ####-##### */ + 0xf7, 0xc0, /* ####-##### */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 207 0xcf '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 208 0xd0 '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 209 0xd1 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 210 0xd2 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 211 0xd3 '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x3f, 0xc0, /* --######## */ + 0x3f, 0xc0, /* --######## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 212 0xd4 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0f, 0xc0, /* ----###### */ + 0x0f, 0xc0, /* ----###### */ + 0x0c, 0x00, /* ----##---- */ + 0x0f, 0xc0, /* ----###### */ + 0x0f, 0xc0, /* ----###### */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 213 0xd5 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0f, 0xc0, /* ----###### */ + 0x0f, 0xc0, /* ----###### */ + 0x0c, 0x00, /* ----##---- */ + 0x0f, 0xc0, /* ----###### */ + 0x0f, 0xc0, /* ----###### */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 214 0xd6 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0xc0, /* --######## */ + 0x3f, 0xc0, /* --######## */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 215 0xd7 '' */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + 0x36, 0x00, /* --##-##--- */ + + /* 216 0xd8 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x0c, 0x00, /* ----##---- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 217 0xd9 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0xfc, 0x00, /* ######---- */ + 0xfc, 0x00, /* ######---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 218 0xda '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0f, 0xc0, /* ----###### */ + 0x0f, 0xc0, /* ----###### */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 219 0xdb '' */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + + /* 220 0xdc '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + + /* 221 0xdd '' */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + 0xf8, 0x00, /* #####----- */ + + /* 222 0xde '' */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + 0x07, 0xc0, /* -----##### */ + + /* 223 0xdf '' */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0xff, 0xc0, /* ########## */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 224 0xe0 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7d, 0x80, /* -#####-##- */ + 0xc7, 0x00, /* ##---###-- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc6, 0x00, /* ##---##--- */ + 0xc7, 0x00, /* ##---###-- */ + 0x7d, 0x80, /* -#####-##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 225 0xe1 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3e, 0x00, /* --#####--- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x62, 0x00, /* -##---#--- */ + 0x7f, 0x00, /* -#######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x7f, 0x00, /* -#######-- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + + /* 226 0xe2 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 227 0xe3 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 228 0xe4 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x30, 0x00, /* --##------ */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x06, 0x00, /* -----##--- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 229 0xe5 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1f, 0xc0, /* ---####### */ + 0x33, 0x00, /* --##--##-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 230 0xe6 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x63, 0x80, /* -##---###- */ + 0x7d, 0x80, /* -#####-##- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + + /* 231 0xe7 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x07, 0x00, /* -----###-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 232 0xe8 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x3f, 0x00, /* --######-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3f, 0x00, /* --######-- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 233 0xe9 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 234 0xea '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x73, 0x80, /* -###--###- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 235 0xeb '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 236 0xec '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 237 0xed '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x01, 0x80, /* -------##- */ + 0x03, 0x00, /* ------##-- */ + 0x3f, 0x00, /* --######-- */ + 0x67, 0x80, /* -##--####- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x79, 0x80, /* -####--##- */ + 0x3f, 0x00, /* --######-- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 238 0xee '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1f, 0x80, /* ---######- */ + 0x30, 0x00, /* --##------ */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x7f, 0x80, /* -########- */ + 0x60, 0x00, /* -##------- */ + 0x60, 0x00, /* -##------- */ + 0x30, 0x00, /* --##------ */ + 0x1f, 0x80, /* ---######- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 239 0xef '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x61, 0x80, /* -##----##- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 240 0xf0 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 241 0xf1 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x7f, 0x80, /* -########- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 242 0xf2 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x30, 0x00, /* --##------ */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x06, 0x00, /* -----##--- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 243 0xf3 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x03, 0x00, /* ------##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x30, 0x00, /* --##------ */ + 0x30, 0x00, /* --##------ */ + 0x18, 0x00, /* ---##----- */ + 0x0c, 0x00, /* ----##---- */ + 0x06, 0x00, /* -----##--- */ + 0x03, 0x00, /* ------##-- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 244 0xf4 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x07, 0x00, /* -----###-- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0d, 0x80, /* ----##-##- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + + /* 245 0xf5 '' */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x6c, 0x00, /* -##-##---- */ + 0x38, 0x00, /* --###----- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 246 0xf6 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x7f, 0x80, /* -########- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 247 0xf7 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x39, 0x80, /* --###--##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x67, 0x00, /* -##--###-- */ + 0x00, 0x00, /* ---------- */ + 0x39, 0x80, /* --###--##- */ + 0x6d, 0x80, /* -##-##-##- */ + 0x67, 0x00, /* -##--###-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 248 0xf8 '' */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1e, 0x00, /* ---####--- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 249 0xf9 '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x1c, 0x00, /* ---###---- */ + 0x1c, 0x00, /* ---###---- */ + 0x1c, 0x00, /* ---###---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 250 0xfa '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x0c, 0x00, /* ----##---- */ + 0x0c, 0x00, /* ----##---- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 251 0xfb '' */ + 0x00, 0x00, /* ---------- */ + 0x03, 0x80, /* ------###- */ + 0x03, 0x80, /* ------###- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x03, 0x00, /* ------##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x63, 0x00, /* -##---##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x1b, 0x00, /* ---##-##-- */ + 0x0f, 0x00, /* ----####-- */ + 0x07, 0x00, /* -----###-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 252 0xfc '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3e, 0x00, /* --#####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 253 0xfd '' */ + 0x00, 0x00, /* ---------- */ + 0x1e, 0x00, /* ---####--- */ + 0x33, 0x00, /* --##--##-- */ + 0x33, 0x00, /* --##--##-- */ + 0x06, 0x00, /* -----##--- */ + 0x0c, 0x00, /* ----##---- */ + 0x18, 0x00, /* ---##----- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 254 0xfe '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x3f, 0x00, /* --######-- */ + 0x3f, 0x00, /* --######-- */ + 0x3f, 0x00, /* --######-- */ + 0x3f, 0x00, /* --######-- */ + 0x3f, 0x00, /* --######-- */ + 0x3f, 0x00, /* --######-- */ + 0x3f, 0x00, /* --######-- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + + /* 255 0xff '' */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ + 0x00, 0x00, /* ---------- */ +} }; + + +const struct font_desc font_ter_10x18 = { + .idx = TER10x18_IDX, + .name = "TER10x18", + .width = 10, + .height = 18, + .charcount = 256, + .data = fontdata_ter10x18.data, +#ifdef __sparc__ + .pref = 5, +#else + .pref = -1, +#endif +}; diff --git a/lib/fonts/fonts.c b/lib/fonts/fonts.c index 47e34950b665..a7f118b30171 100644 --- a/lib/fonts/fonts.c +++ b/lib/fonts/fonts.c @@ -54,6 +54,9 @@ static const struct font_desc *fonts[] = { #ifdef CONFIG_FONT_6x10 &font_6x10, #endif +#ifdef CONFIG_FONT_TER10x18 + &font_ter_10x18, +#endif #ifdef CONFIG_FONT_TER16x32 &font_ter_16x32, #endif diff --git a/lib/hweight.c b/lib/hweight.c index c94586b62551..0dfcafc3fd39 100644 --- a/lib/hweight.c +++ b/lib/hweight.c @@ -4,8 +4,8 @@ #include <asm/types.h> /** - * hweightN - returns the hamming weight of a N-bit word - * @x: the word to weigh + * DOC: __sw_hweightN - returns the hamming weight of a N-bit word + * @w: the word to weigh * * The Hamming Weight of a number is the total number of bits set in it. */ diff --git a/lib/math/div64.c b/lib/math/div64.c index bf77b9843175..d1e92ea24fce 100644 --- a/lib/math/div64.c +++ b/lib/math/div64.c @@ -177,94 +177,157 @@ EXPORT_SYMBOL(div64_s64); * Iterative div/mod for use when dividend is not expected to be much * bigger than divisor. */ +#ifndef iter_div_u64_rem u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder) { return __iter_div_u64_rem(dividend, divisor, remainder); } EXPORT_SYMBOL(iter_div_u64_rem); +#endif -#ifndef mul_u64_u64_div_u64 -u64 mul_u64_u64_div_u64(u64 a, u64 b, u64 c) -{ - if (ilog2(a) + ilog2(b) <= 62) - return div64_u64(a * b, c); +#if !defined(mul_u64_add_u64_div_u64) || defined(test_mul_u64_add_u64_div_u64) -#if defined(__SIZEOF_INT128__) +#define mul_add(a, b, c) add_u64_u32(mul_u32_u32(a, b), c) +#if defined(__SIZEOF_INT128__) && !defined(test_mul_u64_add_u64_div_u64) +static inline u64 mul_u64_u64_add_u64(u64 *p_lo, u64 a, u64 b, u64 c) +{ /* native 64x64=128 bits multiplication */ - u128 prod = (u128)a * b; - u64 n_lo = prod, n_hi = prod >> 64; + u128 prod = (u128)a * b + c; + *p_lo = prod; + return prod >> 64; +} #else - - /* perform a 64x64=128 bits multiplication manually */ - u32 a_lo = a, a_hi = a >> 32, b_lo = b, b_hi = b >> 32; +static inline u64 mul_u64_u64_add_u64(u64 *p_lo, u64 a, u64 b, u64 c) +{ + /* perform a 64x64=128 bits multiplication in 32bit chunks */ u64 x, y, z; - x = (u64)a_lo * b_lo; - y = (u64)a_lo * b_hi + (u32)(x >> 32); - z = (u64)a_hi * b_hi + (u32)(y >> 32); - y = (u64)a_hi * b_lo + (u32)y; - z += (u32)(y >> 32); - x = (y << 32) + (u32)x; - - u64 n_lo = x, n_hi = z; + /* Since (x-1)(x-1) + 2(x-1) == x.x - 1 two u32 can be added to a u64 */ + x = mul_add(a, b, c); + y = mul_add(a, b >> 32, c >> 32); + y = add_u64_u32(y, x >> 32); + z = mul_add(a >> 32, b >> 32, y >> 32); + y = mul_add(a >> 32, b, y); + *p_lo = (y << 32) + (u32)x; + return add_u64_u32(z, y >> 32); +} +#endif +#ifndef BITS_PER_ITER +#define BITS_PER_ITER (__LONG_WIDTH__ >= 64 ? 32 : 16) #endif - /* make sure c is not zero, trigger runtime exception otherwise */ - if (unlikely(c == 0)) { - unsigned long zero = 0; +#if BITS_PER_ITER == 32 +#define mul_u64_long_add_u64(p_lo, a, b, c) mul_u64_u64_add_u64(p_lo, a, b, c) +#define add_u64_long(a, b) ((a) + (b)) +#else +#undef BITS_PER_ITER +#define BITS_PER_ITER 16 +static inline u32 mul_u64_long_add_u64(u64 *p_lo, u64 a, u32 b, u64 c) +{ + u64 n_lo = mul_add(a, b, c); + u64 n_med = mul_add(a >> 32, b, c >> 32); - OPTIMIZER_HIDE_VAR(zero); - return ~0UL/zero; - } + n_med = add_u64_u32(n_med, n_lo >> 32); + *p_lo = n_med << 32 | (u32)n_lo; + return n_med >> 32; +} + +#define add_u64_long(a, b) add_u64_u32(a, b) +#endif + +u64 mul_u64_add_u64_div_u64(u64 a, u64 b, u64 c, u64 d) +{ + unsigned long d_msig, q_digit; + unsigned int reps, d_z_hi; + u64 quotient, n_lo, n_hi; + u32 overflow; - int shift = __builtin_ctzll(c); + n_hi = mul_u64_u64_add_u64(&n_lo, a, b, c); - /* try reducing the fraction in case the dividend becomes <= 64 bits */ - if ((n_hi >> shift) == 0) { - u64 n = shift ? (n_lo >> shift) | (n_hi << (64 - shift)) : n_lo; + if (!n_hi) + return div64_u64(n_lo, d); - return div64_u64(n, c >> shift); - /* - * The remainder value if needed would be: - * res = div64_u64_rem(n, c >> shift, &rem); - * rem = (rem << shift) + (n_lo - (n << shift)); - */ - } + if (unlikely(n_hi >= d)) { + /* trigger runtime exception if divisor is zero */ + if (d == 0) { + unsigned long zero = 0; - if (n_hi >= c) { + OPTIMIZER_HIDE_VAR(zero); + return ~0UL/zero; + } /* overflow: result is unrepresentable in a u64 */ - return -1; + return ~0ULL; } - /* Do the full 128 by 64 bits division */ - - shift = __builtin_clzll(c); - c <<= shift; + /* Left align the divisor, shifting the dividend to match */ + d_z_hi = __builtin_clzll(d); + if (d_z_hi) { + d <<= d_z_hi; + n_hi = n_hi << d_z_hi | n_lo >> (64 - d_z_hi); + n_lo <<= d_z_hi; + } - int p = 64 + shift; - u64 res = 0; - bool carry; + reps = 64 / BITS_PER_ITER; + /* Optimise loop count for small dividends */ + if (!(u32)(n_hi >> 32)) { + reps -= 32 / BITS_PER_ITER; + n_hi = n_hi << 32 | n_lo >> 32; + n_lo <<= 32; + } +#if BITS_PER_ITER == 16 + if (!(u32)(n_hi >> 48)) { + reps--; + n_hi = add_u64_u32(n_hi << 16, n_lo >> 48); + n_lo <<= 16; + } +#endif - do { - carry = n_hi >> 63; - shift = carry ? 1 : __builtin_clzll(n_hi); - if (p < shift) - break; - p -= shift; - n_hi <<= shift; - n_hi |= n_lo >> (64 - shift); - n_lo <<= shift; - if (carry || (n_hi >= c)) { - n_hi -= c; - res |= 1ULL << p; + /* Invert the dividend so we can use add instead of subtract. */ + n_lo = ~n_lo; + n_hi = ~n_hi; + + /* + * Get the most significant BITS_PER_ITER bits of the divisor. + * This is used to get a low 'guestimate' of the quotient digit. + */ + d_msig = (d >> (64 - BITS_PER_ITER)) + 1; + + /* + * Now do a 'long division' with BITS_PER_ITER bit 'digits'. + * The 'guess' quotient digit can be low and BITS_PER_ITER+1 bits. + * The worst case is dividing ~0 by 0x8000 which requires two subtracts. + */ + quotient = 0; + while (reps--) { + q_digit = (unsigned long)(~n_hi >> (64 - 2 * BITS_PER_ITER)) / d_msig; + /* Shift 'n' left to align with the product q_digit * d */ + overflow = n_hi >> (64 - BITS_PER_ITER); + n_hi = add_u64_u32(n_hi << BITS_PER_ITER, n_lo >> (64 - BITS_PER_ITER)); + n_lo <<= BITS_PER_ITER; + /* Add product to negated divisor */ + overflow += mul_u64_long_add_u64(&n_hi, d, q_digit, n_hi); + /* Adjust for the q_digit 'guestimate' being low */ + while (overflow < 0xffffffff >> (32 - BITS_PER_ITER)) { + q_digit++; + n_hi += d; + overflow += n_hi < d; } - } while (n_hi); - /* The remainder value if needed would be n_hi << p */ + quotient = add_u64_long(quotient << BITS_PER_ITER, q_digit); + } - return res; + /* + * The above only ensures the remainder doesn't overflow, + * it can still be possible to add (aka subtract) another copy + * of the divisor. + */ + if ((n_hi + d) > n_hi) + quotient++; + return quotient; } -EXPORT_SYMBOL(mul_u64_u64_div_u64); +#if !defined(test_mul_u64_add_u64_div_u64) +EXPORT_SYMBOL(mul_u64_add_u64_div_u64); +#endif #endif diff --git a/lib/math/test_mul_u64_u64_div_u64.c b/lib/math/test_mul_u64_u64_div_u64.c index 58d058de4e73..338d014f0c73 100644 --- a/lib/math/test_mul_u64_u64_div_u64.c +++ b/lib/math/test_mul_u64_u64_div_u64.c @@ -10,80 +10,141 @@ #include <linux/printk.h> #include <linux/math64.h> -typedef struct { u64 a; u64 b; u64 c; u64 result; } test_params; +typedef struct { u64 a; u64 b; u64 d; u64 result; uint round_up;} test_params; static test_params test_values[] = { /* this contains many edge values followed by a couple random values */ -{ 0xb, 0x7, 0x3, 0x19 }, -{ 0xffff0000, 0xffff0000, 0xf, 0x1110eeef00000000 }, -{ 0xffffffff, 0xffffffff, 0x1, 0xfffffffe00000001 }, -{ 0xffffffff, 0xffffffff, 0x2, 0x7fffffff00000000 }, -{ 0x1ffffffff, 0xffffffff, 0x2, 0xfffffffe80000000 }, -{ 0x1ffffffff, 0xffffffff, 0x3, 0xaaaaaaa9aaaaaaab }, -{ 0x1ffffffff, 0x1ffffffff, 0x4, 0xffffffff00000000 }, -{ 0xffff000000000000, 0xffff000000000000, 0xffff000000000001, 0xfffeffffffffffff }, -{ 0x3333333333333333, 0x3333333333333333, 0x5555555555555555, 0x1eb851eb851eb851 }, -{ 0x7fffffffffffffff, 0x2, 0x3, 0x5555555555555554 }, -{ 0xffffffffffffffff, 0x2, 0x8000000000000000, 0x3 }, -{ 0xffffffffffffffff, 0x2, 0xc000000000000000, 0x2 }, -{ 0xffffffffffffffff, 0x4000000000000004, 0x8000000000000000, 0x8000000000000007 }, -{ 0xffffffffffffffff, 0x4000000000000001, 0x8000000000000000, 0x8000000000000001 }, -{ 0xffffffffffffffff, 0x8000000000000001, 0xffffffffffffffff, 0x8000000000000001 }, -{ 0xfffffffffffffffe, 0x8000000000000001, 0xffffffffffffffff, 0x8000000000000000 }, -{ 0xffffffffffffffff, 0x8000000000000001, 0xfffffffffffffffe, 0x8000000000000001 }, -{ 0xffffffffffffffff, 0x8000000000000001, 0xfffffffffffffffd, 0x8000000000000002 }, -{ 0x7fffffffffffffff, 0xffffffffffffffff, 0xc000000000000000, 0xaaaaaaaaaaaaaaa8 }, -{ 0xffffffffffffffff, 0x7fffffffffffffff, 0xa000000000000000, 0xccccccccccccccca }, -{ 0xffffffffffffffff, 0x7fffffffffffffff, 0x9000000000000000, 0xe38e38e38e38e38b }, -{ 0x7fffffffffffffff, 0x7fffffffffffffff, 0x5000000000000000, 0xccccccccccccccc9 }, -{ 0xffffffffffffffff, 0xfffffffffffffffe, 0xffffffffffffffff, 0xfffffffffffffffe }, -{ 0xe6102d256d7ea3ae, 0x70a77d0be4c31201, 0xd63ec35ab3220357, 0x78f8bf8cc86c6e18 }, -{ 0xf53bae05cb86c6e1, 0x3847b32d2f8d32e0, 0xcfd4f55a647f403c, 0x42687f79d8998d35 }, -{ 0x9951c5498f941092, 0x1f8c8bfdf287a251, 0xa3c8dc5f81ea3fe2, 0x1d887cb25900091f }, -{ 0x374fee9daa1bb2bb, 0x0d0bfbff7b8ae3ef, 0xc169337bd42d5179, 0x03bb2dbaffcbb961 }, -{ 0xeac0d03ac10eeaf0, 0x89be05dfa162ed9b, 0x92bb1679a41f0e4b, 0xdc5f5cc9e270d216 }, +{ 0xb, 0x7, 0x3, 0x19, 1 }, +{ 0xffff0000, 0xffff0000, 0xf, 0x1110eeef00000000, 0 }, +{ 0xffffffff, 0xffffffff, 0x1, 0xfffffffe00000001, 0 }, +{ 0xffffffff, 0xffffffff, 0x2, 0x7fffffff00000000, 1 }, +{ 0x1ffffffff, 0xffffffff, 0x2, 0xfffffffe80000000, 1 }, +{ 0x1ffffffff, 0xffffffff, 0x3, 0xaaaaaaa9aaaaaaab, 0 }, +{ 0x1ffffffff, 0x1ffffffff, 0x4, 0xffffffff00000000, 1 }, +{ 0xffff000000000000, 0xffff000000000000, 0xffff000000000001, 0xfffeffffffffffff, 1 }, +{ 0x3333333333333333, 0x3333333333333333, 0x5555555555555555, 0x1eb851eb851eb851, 1 }, +{ 0x7fffffffffffffff, 0x2, 0x3, 0x5555555555555554, 1 }, +{ 0xffffffffffffffff, 0x2, 0x8000000000000000, 0x3, 1 }, +{ 0xffffffffffffffff, 0x2, 0xc000000000000000, 0x2, 1 }, +{ 0xffffffffffffffff, 0x4000000000000004, 0x8000000000000000, 0x8000000000000007, 1 }, +{ 0xffffffffffffffff, 0x4000000000000001, 0x8000000000000000, 0x8000000000000001, 1 }, +{ 0xffffffffffffffff, 0x8000000000000001, 0xffffffffffffffff, 0x8000000000000001, 0 }, +{ 0xfffffffffffffffe, 0x8000000000000001, 0xffffffffffffffff, 0x8000000000000000, 1 }, +{ 0xffffffffffffffff, 0x8000000000000001, 0xfffffffffffffffe, 0x8000000000000001, 1 }, +{ 0xffffffffffffffff, 0x8000000000000001, 0xfffffffffffffffd, 0x8000000000000002, 1 }, +{ 0x7fffffffffffffff, 0xffffffffffffffff, 0xc000000000000000, 0xaaaaaaaaaaaaaaa8, 1 }, +{ 0xffffffffffffffff, 0x7fffffffffffffff, 0xa000000000000000, 0xccccccccccccccca, 1 }, +{ 0xffffffffffffffff, 0x7fffffffffffffff, 0x9000000000000000, 0xe38e38e38e38e38b, 1 }, +{ 0x7fffffffffffffff, 0x7fffffffffffffff, 0x5000000000000000, 0xccccccccccccccc9, 1 }, +{ 0xffffffffffffffff, 0xfffffffffffffffe, 0xffffffffffffffff, 0xfffffffffffffffe, 0 }, +{ 0xe6102d256d7ea3ae, 0x70a77d0be4c31201, 0xd63ec35ab3220357, 0x78f8bf8cc86c6e18, 1 }, +{ 0xf53bae05cb86c6e1, 0x3847b32d2f8d32e0, 0xcfd4f55a647f403c, 0x42687f79d8998d35, 1 }, +{ 0x9951c5498f941092, 0x1f8c8bfdf287a251, 0xa3c8dc5f81ea3fe2, 0x1d887cb25900091f, 1 }, +{ 0x374fee9daa1bb2bb, 0x0d0bfbff7b8ae3ef, 0xc169337bd42d5179, 0x03bb2dbaffcbb961, 1 }, +{ 0xeac0d03ac10eeaf0, 0x89be05dfa162ed9b, 0x92bb1679a41f0e4b, 0xdc5f5cc9e270d216, 1 }, }; /* * The above table can be verified with the following shell script: - * - * #!/bin/sh - * sed -ne 's/^{ \+\(.*\), \+\(.*\), \+\(.*\), \+\(.*\) },$/\1 \2 \3 \4/p' \ - * lib/math/test_mul_u64_u64_div_u64.c | - * while read a b c r; do - * expected=$( printf "obase=16; ibase=16; %X * %X / %X\n" $a $b $c | bc ) - * given=$( printf "%X\n" $r ) - * if [ "$expected" = "$given" ]; then - * echo "$a * $b / $c = $r OK" - * else - * echo "$a * $b / $c = $r is wrong" >&2 - * echo "should be equivalent to 0x$expected" >&2 - * exit 1 - * fi - * done + +#!/bin/sh +sed -ne 's/^{ \+\(.*\), \+\(.*\), \+\(.*\), \+\(.*\), \+\(.*\) },$/\1 \2 \3 \4 \5/p' \ + lib/math/test_mul_u64_u64_div_u64.c | +while read a b d r e; do + expected=$( printf "obase=16; ibase=16; %X * %X / %X\n" $a $b $d | bc ) + given=$( printf "%X\n" $r ) + if [ "$expected" = "$given" ]; then + echo "$a * $b / $d = $r OK" + else + echo "$a * $b / $d = $r is wrong" >&2 + echo "should be equivalent to 0x$expected" >&2 + exit 1 + fi + expected=$( printf "obase=16; ibase=16; (%X * %X + %X) / %X\n" $a $b $((d-1)) $d | bc ) + given=$( printf "%X\n" $((r + e)) ) + if [ "$expected" = "$given" ]; then + echo "$a * $b +/ $d = $(printf '%#x' $((r + e))) OK" + else + echo "$a * $b +/ $d = $(printf '%#x' $((r + e))) is wrong" >&2 + echo "should be equivalent to 0x$expected" >&2 + exit 1 + fi +done + */ -static int __init test_init(void) +static u64 test_mul_u64_add_u64_div_u64(u64 a, u64 b, u64 c, u64 d); +#if __LONG_WIDTH__ >= 64 +#define TEST_32BIT_DIV +static u64 test_mul_u64_add_u64_div_u64_32bit(u64 a, u64 b, u64 c, u64 d); +#endif + +static int __init test_run(unsigned int fn_no, const char *fn_name) { + u64 start_time; + int errors = 0; + int tests = 0; int i; - pr_info("Starting mul_u64_u64_div_u64() test\n"); + start_time = ktime_get_ns(); for (i = 0; i < ARRAY_SIZE(test_values); i++) { u64 a = test_values[i].a; u64 b = test_values[i].b; - u64 c = test_values[i].c; + u64 d = test_values[i].d; u64 expected_result = test_values[i].result; - u64 result = mul_u64_u64_div_u64(a, b, c); + u64 result, result_up; + + switch (fn_no) { + default: + result = mul_u64_u64_div_u64(a, b, d); + result_up = mul_u64_u64_div_u64_roundup(a, b, d); + break; + case 1: + result = test_mul_u64_add_u64_div_u64(a, b, 0, d); + result_up = test_mul_u64_add_u64_div_u64(a, b, d - 1, d); + break; +#ifdef TEST_32BIT_DIV + case 2: + result = test_mul_u64_add_u64_div_u64_32bit(a, b, 0, d); + result_up = test_mul_u64_add_u64_div_u64_32bit(a, b, d - 1, d); + break; +#endif + } + + tests += 2; if (result != expected_result) { - pr_err("ERROR: 0x%016llx * 0x%016llx / 0x%016llx\n", a, b, c); + pr_err("ERROR: 0x%016llx * 0x%016llx / 0x%016llx\n", a, b, d); pr_err("ERROR: expected result: %016llx\n", expected_result); pr_err("ERROR: obtained result: %016llx\n", result); + errors++; + } + expected_result += test_values[i].round_up; + if (result_up != expected_result) { + pr_err("ERROR: 0x%016llx * 0x%016llx +/ 0x%016llx\n", a, b, d); + pr_err("ERROR: expected result: %016llx\n", expected_result); + pr_err("ERROR: obtained result: %016llx\n", result_up); + errors++; } } - pr_info("Completed mul_u64_u64_div_u64() test\n"); + pr_info("Completed %s() test, %d tests, %d errors, %llu ns\n", + fn_name, tests, errors, ktime_get_ns() - start_time); + return errors; +} + +static int __init test_init(void) +{ + pr_info("Starting mul_u64_u64_div_u64() test\n"); + if (test_run(0, "mul_u64_u64_div_u64")) + return -EINVAL; + if (test_run(1, "test_mul_u64_u64_div_u64")) + return -EINVAL; +#ifdef TEST_32BIT_DIV + if (test_run(2, "test_mul_u64_u64_div_u64_32bit")) + return -EINVAL; +#endif return 0; } @@ -91,6 +152,36 @@ static void __exit test_exit(void) { } +/* Compile the generic mul_u64_add_u64_div_u64() code */ +#undef __div64_32 +#define __div64_32 __div64_32 +#define div_s64_rem div_s64_rem +#define div64_u64_rem div64_u64_rem +#define div64_u64 div64_u64 +#define div64_s64 div64_s64 +#define iter_div_u64_rem iter_div_u64_rem + +#undef mul_u64_add_u64_div_u64 +#define mul_u64_add_u64_div_u64 test_mul_u64_add_u64_div_u64 +#define test_mul_u64_add_u64_div_u64 test_mul_u64_add_u64_div_u64 + +#include "div64.c" + +#ifdef TEST_32BIT_DIV +/* Recompile the generic code for 32bit long */ +#undef test_mul_u64_add_u64_div_u64 +#define test_mul_u64_add_u64_div_u64 test_mul_u64_add_u64_div_u64_32bit +#undef BITS_PER_ITER +#define BITS_PER_ITER 16 + +#define mul_u64_u64_add_u64 mul_u64_u64_add_u64_32bit +#undef mul_u64_long_add_u64 +#undef add_u64_long +#undef mul_add + +#include "div64.c" +#endif + module_init(test_init); module_exit(test_exit); diff --git a/lib/plist.c b/lib/plist.c index 330febb4bd7d..ba677c31e8f3 100644 --- a/lib/plist.c +++ b/lib/plist.c @@ -47,8 +47,8 @@ static void plist_check_list(struct list_head *top) plist_check_prev_next(top, prev, next); while (next != top) { - WRITE_ONCE(prev, next); - WRITE_ONCE(next, prev->next); + prev = next; + next = prev->next; plist_check_prev_next(top, prev, next); } } diff --git a/lib/ratelimit.c b/lib/ratelimit.c index 859c251b23ce..e2d65d3b1c35 100644 --- a/lib/ratelimit.c +++ b/lib/ratelimit.c @@ -27,7 +27,7 @@ int ___ratelimit(struct ratelimit_state *rs, const char *func) { /* Paired with WRITE_ONCE() in .proc_handler(). - * Changing two values seperately could be inconsistent + * Changing two values separately could be inconsistent * and some message could be lost. (See: net_ratelimit_state). */ int interval = READ_ONCE(rs->interval); diff --git a/lib/rbtree.c b/lib/rbtree.c index 5114eda6309c..18d42bcf4ec9 100644 --- a/lib/rbtree.c +++ b/lib/rbtree.c @@ -460,35 +460,6 @@ void __rb_insert_augmented(struct rb_node *node, struct rb_root *root, } EXPORT_SYMBOL(__rb_insert_augmented); -/* - * This function returns the first node (in sort order) of the tree. - */ -struct rb_node *rb_first(const struct rb_root *root) -{ - struct rb_node *n; - - n = root->rb_node; - if (!n) - return NULL; - while (n->rb_left) - n = n->rb_left; - return n; -} -EXPORT_SYMBOL(rb_first); - -struct rb_node *rb_last(const struct rb_root *root) -{ - struct rb_node *n; - - n = root->rb_node; - if (!n) - return NULL; - while (n->rb_right) - n = n->rb_right; - return n; -} -EXPORT_SYMBOL(rb_last); - struct rb_node *rb_next(const struct rb_node *node) { struct rb_node *parent; diff --git a/lib/sys_info.c b/lib/sys_info.c index 496f9151c9b6..f32a06ec9ed4 100644 --- a/lib/sys_info.c +++ b/lib/sys_info.c @@ -1,31 +1,35 @@ // SPDX-License-Identifier: GPL-2.0-only -#include <linux/sched/debug.h> +#include <linux/array_size.h> +#include <linux/bitops.h> +#include <linux/cleanup.h> #include <linux/console.h> +#include <linux/log2.h> #include <linux/kernel.h> #include <linux/ftrace.h> -#include <linux/sysctl.h> #include <linux/nmi.h> +#include <linux/sched/debug.h> +#include <linux/string.h> +#include <linux/sysctl.h> #include <linux/sys_info.h> -struct sys_info_name { - unsigned long bit; - const char *name; +static const char * const si_names[] = { + [ilog2(SYS_INFO_TASKS)] = "tasks", + [ilog2(SYS_INFO_MEM)] = "mem", + [ilog2(SYS_INFO_TIMERS)] = "timers", + [ilog2(SYS_INFO_LOCKS)] = "locks", + [ilog2(SYS_INFO_FTRACE)] = "ftrace", + [ilog2(SYS_INFO_PANIC_CONSOLE_REPLAY)] = "", + [ilog2(SYS_INFO_ALL_BT)] = "all_bt", + [ilog2(SYS_INFO_BLOCKED_TASKS)] = "blocked_tasks", }; /* - * When 'si_names' gets updated, please make sure the 'sys_info_avail' - * below is updated accordingly. + * Default kernel sys_info mask. + * If a kernel module calls sys_info() with "parameter == 0", then + * this mask will be used. */ -static const struct sys_info_name si_names[] = { - { SYS_INFO_TASKS, "tasks" }, - { SYS_INFO_MEM, "mem" }, - { SYS_INFO_TIMERS, "timers" }, - { SYS_INFO_LOCKS, "locks" }, - { SYS_INFO_FTRACE, "ftrace" }, - { SYS_INFO_ALL_CPU_BT, "all_bt" }, - { SYS_INFO_BLOCKED_TASKS, "blocked_tasks" }, -}; +static unsigned long kernel_si_mask; /* Expecting string like "xxx_sys_info=tasks,mem,timers,locks,ftrace,..." */ unsigned long sys_info_parse_param(char *str) @@ -36,12 +40,9 @@ unsigned long sys_info_parse_param(char *str) s = str; while ((name = strsep(&s, ",")) && *name) { - for (i = 0; i < ARRAY_SIZE(si_names); i++) { - if (!strcmp(name, si_names[i].name)) { - si_bits |= si_names[i].bit; - break; - } - } + i = match_string(si_names, ARRAY_SIZE(si_names), name); + if (i >= 0) + __set_bit(i, &si_bits); } return si_bits; @@ -49,56 +50,93 @@ unsigned long sys_info_parse_param(char *str) #ifdef CONFIG_SYSCTL -static const char sys_info_avail[] __maybe_unused = "tasks,mem,timers,locks,ftrace,all_bt,blocked_tasks"; +static int sys_info_write_handler(const struct ctl_table *table, + void *buffer, size_t *lenp, loff_t *ppos, + unsigned long *si_bits_global) +{ + unsigned long si_bits; + int ret; + + ret = proc_dostring(table, 1, buffer, lenp, ppos); + if (ret) + return ret; + + si_bits = sys_info_parse_param(table->data); + + /* The access to the global value is not synchronized. */ + WRITE_ONCE(*si_bits_global, si_bits); + + return 0; +} + +static int sys_info_read_handler(const struct ctl_table *table, + void *buffer, size_t *lenp, loff_t *ppos, + unsigned long *si_bits_global) +{ + unsigned long si_bits; + unsigned int len = 0; + char *delim = ""; + unsigned int i; + + /* The access to the global value is not synchronized. */ + si_bits = READ_ONCE(*si_bits_global); + + for_each_set_bit(i, &si_bits, ARRAY_SIZE(si_names)) { + if (*si_names[i]) { + len += scnprintf(table->data + len, table->maxlen - len, + "%s%s", delim, si_names[i]); + delim = ","; + } + } + + return proc_dostring(table, 0, buffer, lenp, ppos); +} int sysctl_sys_info_handler(const struct ctl_table *ro_table, int write, void *buffer, size_t *lenp, loff_t *ppos) { - char names[sizeof(sys_info_avail)]; struct ctl_table table; - unsigned long *si_bits_global; - - si_bits_global = ro_table->data; - - if (write) { - unsigned long si_bits; - int ret; - - table = *ro_table; - table.data = names; - table.maxlen = sizeof(names); - ret = proc_dostring(&table, write, buffer, lenp, ppos); - if (ret) - return ret; - - si_bits = sys_info_parse_param(names); - /* The access to the global value is not synchronized. */ - WRITE_ONCE(*si_bits_global, si_bits); - return 0; - } else { - /* for 'read' operation */ - char *delim = ""; - int i, len = 0; - - names[0] = '\0'; - for (i = 0; i < ARRAY_SIZE(si_names); i++) { - if (*si_bits_global & si_names[i].bit) { - len += scnprintf(names + len, sizeof(names) - len, - "%s%s", delim, si_names[i].name); - delim = ","; - } - } + unsigned int i; + size_t maxlen; - table = *ro_table; - table.data = names; - table.maxlen = sizeof(names); - return proc_dostring(&table, write, buffer, lenp, ppos); - } + maxlen = 0; + for (i = 0; i < ARRAY_SIZE(si_names); i++) + maxlen += strlen(si_names[i]) + 1; + + char *names __free(kfree) = kzalloc(maxlen, GFP_KERNEL); + if (!names) + return -ENOMEM; + + table = *ro_table; + table.data = names; + table.maxlen = maxlen; + + if (write) + return sys_info_write_handler(&table, buffer, lenp, ppos, ro_table->data); + else + return sys_info_read_handler(&table, buffer, lenp, ppos, ro_table->data); +} + +static const struct ctl_table sys_info_sysctls[] = { + { + .procname = "kernel_sys_info", + .data = &kernel_si_mask, + .maxlen = sizeof(kernel_si_mask), + .mode = 0644, + .proc_handler = sysctl_sys_info_handler, + }, +}; + +static int __init sys_info_sysctl_init(void) +{ + register_sysctl_init("kernel", sys_info_sysctls); + return 0; } +subsys_initcall(sys_info_sysctl_init); #endif -void sys_info(unsigned long si_mask) +static void __sys_info(unsigned long si_mask) { if (si_mask & SYS_INFO_TASKS) show_state(); @@ -115,9 +153,14 @@ void sys_info(unsigned long si_mask) if (si_mask & SYS_INFO_FTRACE) ftrace_dump(DUMP_ALL); - if (si_mask & SYS_INFO_ALL_CPU_BT) + if (si_mask & SYS_INFO_ALL_BT) trigger_all_cpu_backtrace(); if (si_mask & SYS_INFO_BLOCKED_TASKS) show_state_filter(TASK_UNINTERRUPTIBLE); } + +void sys_info(unsigned long si_mask) +{ + __sys_info(si_mask ? : kernel_si_mask); +} diff --git a/lib/test_kho.c b/lib/test_kho.c index fff018e5548d..47de56280795 100644 --- a/lib/test_kho.c +++ b/lib/test_kho.c @@ -33,44 +33,28 @@ struct kho_test_state { unsigned int nr_folios; struct folio **folios; phys_addr_t *folios_info; + struct kho_vmalloc folios_info_phys; + int nr_folios_preserved; struct folio *fdt; __wsum csum; }; static struct kho_test_state kho_test_state; -static int kho_test_notifier(struct notifier_block *self, unsigned long cmd, - void *v) +static void kho_test_unpreserve_data(struct kho_test_state *state) { - struct kho_test_state *state = &kho_test_state; - struct kho_serialization *ser = v; - int err = 0; - - switch (cmd) { - case KEXEC_KHO_ABORT: - return NOTIFY_DONE; - case KEXEC_KHO_FINALIZE: - /* Handled below */ - break; - default: - return NOTIFY_BAD; - } - - err |= kho_preserve_folio(state->fdt); - err |= kho_add_subtree(ser, KHO_TEST_FDT, folio_address(state->fdt)); + for (int i = 0; i < state->nr_folios_preserved; i++) + kho_unpreserve_folio(state->folios[i]); - return err ? NOTIFY_BAD : NOTIFY_DONE; + kho_unpreserve_vmalloc(&state->folios_info_phys); + vfree(state->folios_info); } -static struct notifier_block kho_test_nb = { - .notifier_call = kho_test_notifier, -}; - -static int kho_test_save_data(struct kho_test_state *state, void *fdt) +static int kho_test_preserve_data(struct kho_test_state *state) { - phys_addr_t *folios_info __free(kvfree) = NULL; struct kho_vmalloc folios_info_phys; - int err = 0; + phys_addr_t *folios_info; + int err; folios_info = vmalloc_array(state->nr_folios, sizeof(*folios_info)); if (!folios_info) @@ -78,62 +62,98 @@ static int kho_test_save_data(struct kho_test_state *state, void *fdt) err = kho_preserve_vmalloc(folios_info, &folios_info_phys); if (err) - return err; + goto err_free_info; + + state->folios_info_phys = folios_info_phys; + state->folios_info = folios_info; for (int i = 0; i < state->nr_folios; i++) { struct folio *folio = state->folios[i]; unsigned int order = folio_order(folio); folios_info[i] = virt_to_phys(folio_address(folio)) | order; - err = kho_preserve_folio(folio); if (err) - break; + goto err_unpreserve; + state->nr_folios_preserved++; } + return 0; + +err_unpreserve: + /* + * kho_test_unpreserve_data frees folio_info, bail out immediately to + * avoid double free + */ + kho_test_unpreserve_data(state); + return err; + +err_free_info: + vfree(folios_info); + return err; +} + +static int kho_test_prepare_fdt(struct kho_test_state *state, ssize_t fdt_size) +{ + const char compatible[] = KHO_TEST_COMPAT; + unsigned int magic = KHO_TEST_MAGIC; + void *fdt = folio_address(state->fdt); + int err; + + err = fdt_create(fdt, fdt_size); + err |= fdt_finish_reservemap(fdt); + err |= fdt_begin_node(fdt, ""); + err |= fdt_property(fdt, "compatible", compatible, sizeof(compatible)); + err |= fdt_property(fdt, "magic", &magic, sizeof(magic)); + err |= fdt_begin_node(fdt, "data"); err |= fdt_property(fdt, "nr_folios", &state->nr_folios, sizeof(state->nr_folios)); - err |= fdt_property(fdt, "folios_info", &folios_info_phys, - sizeof(folios_info_phys)); + err |= fdt_property(fdt, "folios_info", &state->folios_info_phys, + sizeof(state->folios_info_phys)); err |= fdt_property(fdt, "csum", &state->csum, sizeof(state->csum)); err |= fdt_end_node(fdt); - if (!err) - state->folios_info = no_free_ptr(folios_info); + err |= fdt_end_node(fdt); + err |= fdt_finish(fdt); return err; } -static int kho_test_prepare_fdt(struct kho_test_state *state) +static int kho_test_preserve(struct kho_test_state *state) { - const char compatible[] = KHO_TEST_COMPAT; - unsigned int magic = KHO_TEST_MAGIC; ssize_t fdt_size; - int err = 0; - void *fdt; + int err; fdt_size = state->nr_folios * sizeof(phys_addr_t) + PAGE_SIZE; state->fdt = folio_alloc(GFP_KERNEL, get_order(fdt_size)); if (!state->fdt) return -ENOMEM; - fdt = folio_address(state->fdt); - - err |= fdt_create(fdt, fdt_size); - err |= fdt_finish_reservemap(fdt); + err = kho_preserve_folio(state->fdt); + if (err) + goto err_free_fdt; - err |= fdt_begin_node(fdt, ""); - err |= fdt_property(fdt, "compatible", compatible, sizeof(compatible)); - err |= fdt_property(fdt, "magic", &magic, sizeof(magic)); - err |= kho_test_save_data(state, fdt); - err |= fdt_end_node(fdt); + err = kho_test_preserve_data(state); + if (err) + goto err_unpreserve_fdt; - err |= fdt_finish(fdt); + err = kho_test_prepare_fdt(state, fdt_size); + if (err) + goto err_unpreserve_data; + err = kho_add_subtree(KHO_TEST_FDT, folio_address(state->fdt)); if (err) - folio_put(state->fdt); + goto err_unpreserve_data; + + return 0; +err_unpreserve_data: + kho_test_unpreserve_data(state); +err_unpreserve_fdt: + kho_unpreserve_folio(state->fdt); +err_free_fdt: + folio_put(state->fdt); return err; } @@ -199,18 +219,12 @@ static int kho_test_save(void) if (err) goto err_free_folios; - err = kho_test_prepare_fdt(state); + err = kho_test_preserve(state); if (err) goto err_free_folios; - err = register_kho_notifier(&kho_test_nb); - if (err) - goto err_free_fdt; - return 0; -err_free_fdt: - folio_put(state->fdt); err_free_folios: kvfree(folios); return err; @@ -292,7 +306,6 @@ static int kho_test_restore(phys_addr_t fdt_phys) if (err) return err; - pr_info("KHO restore succeeded\n"); return 0; } @@ -305,8 +318,15 @@ static int __init kho_test_init(void) return 0; err = kho_retrieve_subtree(KHO_TEST_FDT, &fdt_phys); - if (!err) - return kho_test_restore(fdt_phys); + if (!err) { + err = kho_test_restore(fdt_phys); + if (err) + pr_err("KHO restore failed\n"); + else + pr_info("KHO restore succeeded\n"); + + return err; + } if (err != -ENOENT) { pr_warn("failed to retrieve %s FDT: %d\n", KHO_TEST_FDT, err); @@ -329,7 +349,7 @@ static void kho_test_cleanup(void) static void __exit kho_test_exit(void) { - unregister_kho_notifier(&kho_test_nb); + kho_remove_subtree(folio_address(kho_test_state.fdt)); kho_test_cleanup(); } module_exit(kho_test_exit); diff --git a/lib/tests/Makefile b/lib/tests/Makefile index f7460831cfdd..601dba4b7d96 100644 --- a/lib/tests/Makefile +++ b/lib/tests/Makefile @@ -4,6 +4,7 @@ # KUnit tests CFLAGS_bitfield_kunit.o := $(DISABLE_STRUCTLEAK_PLUGIN) +obj-$(CONFIG_BASE64_KUNIT) += base64_kunit.o obj-$(CONFIG_BITFIELD_KUNIT) += bitfield_kunit.o obj-$(CONFIG_BITS_TEST) += test_bits.o obj-$(CONFIG_BLACKHOLE_DEV_KUNIT_TEST) += blackhole_dev_kunit.o diff --git a/lib/tests/base64_kunit.c b/lib/tests/base64_kunit.c new file mode 100644 index 000000000000..f7252070c359 --- /dev/null +++ b/lib/tests/base64_kunit.c @@ -0,0 +1,294 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * base64_kunit_test.c - KUnit tests for base64 encoding and decoding functions + * + * Copyright (c) 2025, Guan-Chun Wu <409411716@gms.tku.edu.tw> + */ + +#include <kunit/test.h> +#include <linux/base64.h> + +/* ---------- Benchmark helpers ---------- */ +static u64 bench_encode_ns(const u8 *data, int len, char *dst, int reps, + enum base64_variant variant) +{ + u64 t0, t1; + + t0 = ktime_get_ns(); + for (int i = 0; i < reps; i++) + base64_encode(data, len, dst, true, variant); + t1 = ktime_get_ns(); + + return div64_u64(t1 - t0, (u64)reps); +} + +static u64 bench_decode_ns(const char *data, int len, u8 *dst, int reps, + enum base64_variant variant) +{ + u64 t0, t1; + + t0 = ktime_get_ns(); + for (int i = 0; i < reps; i++) + base64_decode(data, len, dst, true, variant); + t1 = ktime_get_ns(); + + return div64_u64(t1 - t0, (u64)reps); +} + +static void run_perf_and_check(struct kunit *test, const char *label, int size, + enum base64_variant variant) +{ + const int reps = 1000; + size_t outlen = DIV_ROUND_UP(size, 3) * 4; + u8 *in = kmalloc(size, GFP_KERNEL); + char *enc = kmalloc(outlen, GFP_KERNEL); + u8 *decoded = kmalloc(size, GFP_KERNEL); + + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, in); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, enc); + KUNIT_ASSERT_NOT_ERR_OR_NULL(test, decoded); + + get_random_bytes(in, size); + int enc_len = base64_encode(in, size, enc, true, variant); + int dec_len = base64_decode(enc, enc_len, decoded, true, variant); + + /* correctness sanity check */ + KUNIT_EXPECT_EQ(test, dec_len, size); + KUNIT_EXPECT_MEMEQ(test, decoded, in, size); + + /* benchmark encode */ + + u64 t1 = bench_encode_ns(in, size, enc, reps, variant); + + kunit_info(test, "[%s] encode run : %lluns", label, t1); + + u64 t2 = bench_decode_ns(enc, enc_len, decoded, reps, variant); + + kunit_info(test, "[%s] decode run : %lluns", label, t2); + + kfree(in); + kfree(enc); + kfree(decoded); +} + +static void base64_performance_tests(struct kunit *test) +{ + /* run on STD variant only */ + run_perf_and_check(test, "64B", 64, BASE64_STD); + run_perf_and_check(test, "1KB", 1024, BASE64_STD); +} + +/* ---------- Helpers for encode ---------- */ +static void expect_encode_ok(struct kunit *test, const u8 *src, int srclen, + const char *expected, bool padding, + enum base64_variant variant) +{ + char buf[128]; + int encoded_len = base64_encode(src, srclen, buf, padding, variant); + + buf[encoded_len] = '\0'; + + KUNIT_EXPECT_EQ(test, encoded_len, strlen(expected)); + KUNIT_EXPECT_STREQ(test, buf, expected); +} + +/* ---------- Helpers for decode ---------- */ +static void expect_decode_ok(struct kunit *test, const char *src, + const u8 *expected, int expected_len, bool padding, + enum base64_variant variant) +{ + u8 buf[128]; + int decoded_len = base64_decode(src, strlen(src), buf, padding, variant); + + KUNIT_EXPECT_EQ(test, decoded_len, expected_len); + KUNIT_EXPECT_MEMEQ(test, buf, expected, expected_len); +} + +static void expect_decode_err(struct kunit *test, const char *src, + int srclen, bool padding, + enum base64_variant variant) +{ + u8 buf[64]; + int decoded_len = base64_decode(src, srclen, buf, padding, variant); + + KUNIT_EXPECT_EQ(test, decoded_len, -1); +} + +/* ---------- Encode Tests ---------- */ +static void base64_std_encode_tests(struct kunit *test) +{ + /* With padding */ + expect_encode_ok(test, (const u8 *)"", 0, "", true, BASE64_STD); + expect_encode_ok(test, (const u8 *)"f", 1, "Zg==", true, BASE64_STD); + expect_encode_ok(test, (const u8 *)"fo", 2, "Zm8=", true, BASE64_STD); + expect_encode_ok(test, (const u8 *)"foo", 3, "Zm9v", true, BASE64_STD); + expect_encode_ok(test, (const u8 *)"foob", 4, "Zm9vYg==", true, BASE64_STD); + expect_encode_ok(test, (const u8 *)"fooba", 5, "Zm9vYmE=", true, BASE64_STD); + expect_encode_ok(test, (const u8 *)"foobar", 6, "Zm9vYmFy", true, BASE64_STD); + + /* Extra cases with padding */ + expect_encode_ok(test, (const u8 *)"Hello, world!", 13, "SGVsbG8sIHdvcmxkIQ==", + true, BASE64_STD); + expect_encode_ok(test, (const u8 *)"ABCDEFGHIJKLMNOPQRSTUVWXYZ", 26, + "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo=", true, BASE64_STD); + expect_encode_ok(test, (const u8 *)"abcdefghijklmnopqrstuvwxyz", 26, + "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=", true, BASE64_STD); + expect_encode_ok(test, (const u8 *)"0123456789+/", 12, "MDEyMzQ1Njc4OSsv", + true, BASE64_STD); + + /* Without padding */ + expect_encode_ok(test, (const u8 *)"", 0, "", false, BASE64_STD); + expect_encode_ok(test, (const u8 *)"f", 1, "Zg", false, BASE64_STD); + expect_encode_ok(test, (const u8 *)"fo", 2, "Zm8", false, BASE64_STD); + expect_encode_ok(test, (const u8 *)"foo", 3, "Zm9v", false, BASE64_STD); + expect_encode_ok(test, (const u8 *)"foob", 4, "Zm9vYg", false, BASE64_STD); + expect_encode_ok(test, (const u8 *)"fooba", 5, "Zm9vYmE", false, BASE64_STD); + expect_encode_ok(test, (const u8 *)"foobar", 6, "Zm9vYmFy", false, BASE64_STD); + + /* Extra cases without padding */ + expect_encode_ok(test, (const u8 *)"Hello, world!", 13, "SGVsbG8sIHdvcmxkIQ", + false, BASE64_STD); + expect_encode_ok(test, (const u8 *)"ABCDEFGHIJKLMNOPQRSTUVWXYZ", 26, + "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo", false, BASE64_STD); + expect_encode_ok(test, (const u8 *)"abcdefghijklmnopqrstuvwxyz", 26, + "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo", false, BASE64_STD); + expect_encode_ok(test, (const u8 *)"0123456789+/", 12, "MDEyMzQ1Njc4OSsv", + false, BASE64_STD); +} + +/* ---------- Decode Tests ---------- */ +static void base64_std_decode_tests(struct kunit *test) +{ + /* -------- With padding --------*/ + expect_decode_ok(test, "", (const u8 *)"", 0, true, BASE64_STD); + expect_decode_ok(test, "Zg==", (const u8 *)"f", 1, true, BASE64_STD); + expect_decode_ok(test, "Zm8=", (const u8 *)"fo", 2, true, BASE64_STD); + expect_decode_ok(test, "Zm9v", (const u8 *)"foo", 3, true, BASE64_STD); + expect_decode_ok(test, "Zm9vYg==", (const u8 *)"foob", 4, true, BASE64_STD); + expect_decode_ok(test, "Zm9vYmE=", (const u8 *)"fooba", 5, true, BASE64_STD); + expect_decode_ok(test, "Zm9vYmFy", (const u8 *)"foobar", 6, true, BASE64_STD); + expect_decode_ok(test, "SGVsbG8sIHdvcmxkIQ==", (const u8 *)"Hello, world!", 13, + true, BASE64_STD); + expect_decode_ok(test, "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo=", + (const u8 *)"ABCDEFGHIJKLMNOPQRSTUVWXYZ", 26, true, BASE64_STD); + expect_decode_ok(test, "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo=", + (const u8 *)"abcdefghijklmnopqrstuvwxyz", 26, true, BASE64_STD); + + /* Error cases */ + expect_decode_err(test, "Zg=!", 4, true, BASE64_STD); + expect_decode_err(test, "Zm$=", 4, true, BASE64_STD); + expect_decode_err(test, "Z===", 4, true, BASE64_STD); + expect_decode_err(test, "Zg", 2, true, BASE64_STD); + expect_decode_err(test, "Zm9v====", 8, true, BASE64_STD); + expect_decode_err(test, "Zm==A", 5, true, BASE64_STD); + + { + char with_nul[4] = { 'Z', 'g', '\0', '=' }; + + expect_decode_err(test, with_nul, 4, true, BASE64_STD); + } + + /* -------- Without padding --------*/ + expect_decode_ok(test, "", (const u8 *)"", 0, false, BASE64_STD); + expect_decode_ok(test, "Zg", (const u8 *)"f", 1, false, BASE64_STD); + expect_decode_ok(test, "Zm8", (const u8 *)"fo", 2, false, BASE64_STD); + expect_decode_ok(test, "Zm9v", (const u8 *)"foo", 3, false, BASE64_STD); + expect_decode_ok(test, "Zm9vYg", (const u8 *)"foob", 4, false, BASE64_STD); + expect_decode_ok(test, "Zm9vYmE", (const u8 *)"fooba", 5, false, BASE64_STD); + expect_decode_ok(test, "Zm9vYmFy", (const u8 *)"foobar", 6, false, BASE64_STD); + expect_decode_ok(test, "TWFu", (const u8 *)"Man", 3, false, BASE64_STD); + expect_decode_ok(test, "SGVsbG8sIHdvcmxkIQ", (const u8 *)"Hello, world!", 13, + false, BASE64_STD); + expect_decode_ok(test, "QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVo", + (const u8 *)"ABCDEFGHIJKLMNOPQRSTUVWXYZ", 26, false, BASE64_STD); + expect_decode_ok(test, "YWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXo", + (const u8 *)"abcdefghijklmnopqrstuvwxyz", 26, false, BASE64_STD); + expect_decode_ok(test, "MDEyMzQ1Njc4OSsv", (const u8 *)"0123456789+/", 12, + false, BASE64_STD); + + /* Error cases */ + expect_decode_err(test, "Zg=!", 4, false, BASE64_STD); + expect_decode_err(test, "Zm$=", 4, false, BASE64_STD); + expect_decode_err(test, "Z===", 4, false, BASE64_STD); + expect_decode_err(test, "Zg=", 3, false, BASE64_STD); + expect_decode_err(test, "Zm9v====", 8, false, BASE64_STD); + expect_decode_err(test, "Zm==v", 4, false, BASE64_STD); + + { + char with_nul[4] = { 'Z', 'g', '\0', '=' }; + + expect_decode_err(test, with_nul, 4, false, BASE64_STD); + } +} + +/* ---------- Variant tests (URLSAFE / IMAP) ---------- */ +static void base64_variant_tests(struct kunit *test) +{ + const u8 sample1[] = { 0x00, 0xfb, 0xff, 0x7f, 0x80 }; + char std_buf[128], url_buf[128], imap_buf[128]; + u8 back[128]; + int n_std, n_url, n_imap, m; + int i; + + n_std = base64_encode(sample1, sizeof(sample1), std_buf, false, BASE64_STD); + n_url = base64_encode(sample1, sizeof(sample1), url_buf, false, BASE64_URLSAFE); + std_buf[n_std] = '\0'; + url_buf[n_url] = '\0'; + + for (i = 0; i < n_std; i++) { + if (std_buf[i] == '+') + std_buf[i] = '-'; + else if (std_buf[i] == '/') + std_buf[i] = '_'; + } + KUNIT_EXPECT_STREQ(test, std_buf, url_buf); + + m = base64_decode(url_buf, n_url, back, false, BASE64_URLSAFE); + KUNIT_EXPECT_EQ(test, m, (int)sizeof(sample1)); + KUNIT_EXPECT_MEMEQ(test, back, sample1, sizeof(sample1)); + + n_std = base64_encode(sample1, sizeof(sample1), std_buf, false, BASE64_STD); + n_imap = base64_encode(sample1, sizeof(sample1), imap_buf, false, BASE64_IMAP); + std_buf[n_std] = '\0'; + imap_buf[n_imap] = '\0'; + + for (i = 0; i < n_std; i++) + if (std_buf[i] == '/') + std_buf[i] = ','; + KUNIT_EXPECT_STREQ(test, std_buf, imap_buf); + + m = base64_decode(imap_buf, n_imap, back, false, BASE64_IMAP); + KUNIT_EXPECT_EQ(test, m, (int)sizeof(sample1)); + KUNIT_EXPECT_MEMEQ(test, back, sample1, sizeof(sample1)); + + { + const char *bad = "Zg=="; + u8 tmp[8]; + + m = base64_decode(bad, strlen(bad), tmp, false, BASE64_URLSAFE); + KUNIT_EXPECT_EQ(test, m, -1); + + m = base64_decode(bad, strlen(bad), tmp, false, BASE64_IMAP); + KUNIT_EXPECT_EQ(test, m, -1); + } +} + +/* ---------- Test registration ---------- */ +static struct kunit_case base64_test_cases[] = { + KUNIT_CASE(base64_performance_tests), + KUNIT_CASE(base64_std_encode_tests), + KUNIT_CASE(base64_std_decode_tests), + KUNIT_CASE(base64_variant_tests), + {} +}; + +static struct kunit_suite base64_test_suite = { + .name = "base64", + .test_cases = base64_test_cases, +}; + +kunit_test_suite(base64_test_suite); + +MODULE_AUTHOR("Guan-Chun Wu <409411716@gms.tku.edu.tw>"); +MODULE_DESCRIPTION("KUnit tests for Base64 encoding/decoding, including performance checks"); +MODULE_LICENSE("GPL"); diff --git a/lib/usercopy.c b/lib/usercopy.c index 7b17b83c8042..b00a3a957de6 100644 --- a/lib/usercopy.c +++ b/lib/usercopy.c @@ -12,7 +12,7 @@ /* out-of-line parts */ -#if !defined(INLINE_COPY_FROM_USER) || defined(CONFIG_RUST) +#if !defined(INLINE_COPY_FROM_USER) unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n) { return _inline_copy_from_user(to, from, n); @@ -20,7 +20,7 @@ unsigned long _copy_from_user(void *to, const void __user *from, unsigned long n EXPORT_SYMBOL(_copy_from_user); #endif -#if !defined(INLINE_COPY_TO_USER) || defined(CONFIG_RUST) +#if !defined(INLINE_COPY_TO_USER) unsigned long _copy_to_user(void __user *to, const void *from, unsigned long n) { return _inline_copy_to_user(to, from, n); diff --git a/lib/xxhash.c b/lib/xxhash.c index cf629766f376..4125b3e3cf7f 100644 --- a/lib/xxhash.c +++ b/lib/xxhash.c @@ -73,21 +73,6 @@ static const uint64_t PRIME64_3 = 1609587929392839161ULL; static const uint64_t PRIME64_4 = 9650029242287828579ULL; static const uint64_t PRIME64_5 = 2870177450012600261ULL; -/*-************************** - * Utils - ***************************/ -void xxh32_copy_state(struct xxh32_state *dst, const struct xxh32_state *src) -{ - memcpy(dst, src, sizeof(*dst)); -} -EXPORT_SYMBOL(xxh32_copy_state); - -void xxh64_copy_state(struct xxh64_state *dst, const struct xxh64_state *src) -{ - memcpy(dst, src, sizeof(*dst)); -} -EXPORT_SYMBOL(xxh64_copy_state); - /*-*************************** * Simple Hash Functions ****************************/ @@ -239,20 +224,6 @@ EXPORT_SYMBOL(xxh64); /*-************************************************** * Advanced Hash Functions ***************************************************/ -void xxh32_reset(struct xxh32_state *statePtr, const uint32_t seed) -{ - /* use a local state for memcpy() to avoid strict-aliasing warnings */ - struct xxh32_state state; - - memset(&state, 0, sizeof(state)); - state.v1 = seed + PRIME32_1 + PRIME32_2; - state.v2 = seed + PRIME32_2; - state.v3 = seed + 0; - state.v4 = seed - PRIME32_1; - memcpy(statePtr, &state, sizeof(state)); -} -EXPORT_SYMBOL(xxh32_reset); - void xxh64_reset(struct xxh64_state *statePtr, const uint64_t seed) { /* use a local state for memcpy() to avoid strict-aliasing warnings */ diff --git a/lib/xz/xz_dec_bcj.c b/lib/xz/xz_dec_bcj.c index 8237db17eee3..610d58d947ab 100644 --- a/lib/xz/xz_dec_bcj.c +++ b/lib/xz/xz_dec_bcj.c @@ -20,7 +20,6 @@ struct xz_dec_bcj { enum { BCJ_X86 = 4, /* x86 or x86-64 */ BCJ_POWERPC = 5, /* Big endian only */ - BCJ_IA64 = 6, /* Big or little endian */ BCJ_ARM = 7, /* Little endian only */ BCJ_ARMTHUMB = 8, /* Little endian only */ BCJ_SPARC = 9, /* Big or little endian */ @@ -180,92 +179,6 @@ static size_t bcj_powerpc(struct xz_dec_bcj *s, uint8_t *buf, size_t size) } #endif -#ifdef XZ_DEC_IA64 -static size_t bcj_ia64(struct xz_dec_bcj *s, uint8_t *buf, size_t size) -{ - static const uint8_t branch_table[32] = { - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 4, 4, 6, 6, 0, 0, 7, 7, - 4, 4, 0, 0, 4, 4, 0, 0 - }; - - /* - * The local variables take a little bit stack space, but it's less - * than what LZMA2 decoder takes, so it doesn't make sense to reduce - * stack usage here without doing that for the LZMA2 decoder too. - */ - - /* Loop counters */ - size_t i; - size_t j; - - /* Instruction slot (0, 1, or 2) in the 128-bit instruction word */ - uint32_t slot; - - /* Bitwise offset of the instruction indicated by slot */ - uint32_t bit_pos; - - /* bit_pos split into byte and bit parts */ - uint32_t byte_pos; - uint32_t bit_res; - - /* Address part of an instruction */ - uint32_t addr; - - /* Mask used to detect which instructions to convert */ - uint32_t mask; - - /* 41-bit instruction stored somewhere in the lowest 48 bits */ - uint64_t instr; - - /* Instruction normalized with bit_res for easier manipulation */ - uint64_t norm; - - size &= ~(size_t)15; - - for (i = 0; i < size; i += 16) { - mask = branch_table[buf[i] & 0x1F]; - for (slot = 0, bit_pos = 5; slot < 3; ++slot, bit_pos += 41) { - if (((mask >> slot) & 1) == 0) - continue; - - byte_pos = bit_pos >> 3; - bit_res = bit_pos & 7; - instr = 0; - for (j = 0; j < 6; ++j) - instr |= (uint64_t)(buf[i + j + byte_pos]) - << (8 * j); - - norm = instr >> bit_res; - - if (((norm >> 37) & 0x0F) == 0x05 - && ((norm >> 9) & 0x07) == 0) { - addr = (norm >> 13) & 0x0FFFFF; - addr |= ((uint32_t)(norm >> 36) & 1) << 20; - addr <<= 4; - addr -= s->pos + (uint32_t)i; - addr >>= 4; - - norm &= ~((uint64_t)0x8FFFFF << 13); - norm |= (uint64_t)(addr & 0x0FFFFF) << 13; - norm |= (uint64_t)(addr & 0x100000) - << (36 - 20); - - instr &= (1 << bit_res) - 1; - instr |= norm << bit_res; - - for (j = 0; j < 6; j++) - buf[i + j + byte_pos] - = (uint8_t)(instr >> (8 * j)); - } - } - } - - return i; -} -#endif - #ifdef XZ_DEC_ARM static size_t bcj_arm(struct xz_dec_bcj *s, uint8_t *buf, size_t size) { @@ -509,11 +422,6 @@ static void bcj_apply(struct xz_dec_bcj *s, filtered = bcj_powerpc(s, buf, size); break; #endif -#ifdef XZ_DEC_IA64 - case BCJ_IA64: - filtered = bcj_ia64(s, buf, size); - break; -#endif #ifdef XZ_DEC_ARM case BCJ_ARM: filtered = bcj_arm(s, buf, size); @@ -699,9 +607,6 @@ enum xz_ret xz_dec_bcj_reset(struct xz_dec_bcj *s, uint8_t id) #ifdef XZ_DEC_POWERPC case BCJ_POWERPC: #endif -#ifdef XZ_DEC_IA64 - case BCJ_IA64: -#endif #ifdef XZ_DEC_ARM case BCJ_ARM: #endif diff --git a/lib/xz/xz_private.h b/lib/xz/xz_private.h index 8409784b1639..6775078f3cce 100644 --- a/lib/xz/xz_private.h +++ b/lib/xz/xz_private.h @@ -24,9 +24,6 @@ # ifdef CONFIG_XZ_DEC_POWERPC # define XZ_DEC_POWERPC # endif -# ifdef CONFIG_XZ_DEC_IA64 -# define XZ_DEC_IA64 -# endif # ifdef CONFIG_XZ_DEC_ARM # define XZ_DEC_ARM # endif @@ -103,7 +100,6 @@ */ #ifndef XZ_DEC_BCJ # if defined(XZ_DEC_X86) || defined(XZ_DEC_POWERPC) \ - || defined(XZ_DEC_IA64) \ || defined(XZ_DEC_ARM) || defined(XZ_DEC_ARMTHUMB) \ || defined(XZ_DEC_SPARC) || defined(XZ_DEC_ARM64) \ || defined(XZ_DEC_RISCV) diff --git a/mm/Makefile b/mm/Makefile index 00ceb2418b64..2d0570a16e5b 100644 --- a/mm/Makefile +++ b/mm/Makefile @@ -100,6 +100,7 @@ obj-$(CONFIG_NUMA) += memory-tiers.o obj-$(CONFIG_DEVICE_MIGRATION) += migrate_device.o obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o obj-$(CONFIG_PAGE_COUNTER) += page_counter.o +obj-$(CONFIG_LIVEUPDATE) += memfd_luo.o obj-$(CONFIG_MEMCG_V1) += memcontrol-v1.o obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o ifdef CONFIG_SWAP diff --git a/mm/internal.h b/mm/internal.h index 89790def1bae..e430da900430 100644 --- a/mm/internal.h +++ b/mm/internal.h @@ -1582,6 +1582,12 @@ void __meminit __init_page_from_nid(unsigned long pfn, int nid); unsigned long shrink_slab(gfp_t gfp_mask, int nid, struct mem_cgroup *memcg, int priority); +int shmem_add_to_page_cache(struct folio *folio, + struct address_space *mapping, + pgoff_t index, void *expected, gfp_t gfp); +int shmem_inode_acct_blocks(struct inode *inode, long pages); +bool shmem_recalc_inode(struct inode *inode, long alloced, long swapped); + #ifdef CONFIG_SHRINKER_DEBUG static inline __printf(2, 0) int shrinker_debugfs_name_alloc( struct shrinker *shrinker, const char *fmt, va_list ap) diff --git a/mm/memblock.c b/mm/memblock.c index f0f2dc66e9a2..905d06b16348 100644 --- a/mm/memblock.c +++ b/mm/memblock.c @@ -2445,60 +2445,59 @@ int reserve_mem_release_by_name(const char *name) #define MEMBLOCK_KHO_FDT "memblock" #define MEMBLOCK_KHO_NODE_COMPATIBLE "memblock-v1" #define RESERVE_MEM_KHO_NODE_COMPATIBLE "reserve-mem-v1" -static struct page *kho_fdt; -static int reserve_mem_kho_finalize(struct kho_serialization *ser) +static int __init reserved_mem_preserve(void) { - int err = 0, i; + unsigned int nr_preserved = 0; + int err; - for (i = 0; i < reserved_mem_count; i++) { + for (unsigned int i = 0; i < reserved_mem_count; i++, nr_preserved++) { struct reserve_mem_table *map = &reserved_mem_table[i]; struct page *page = phys_to_page(map->start); unsigned int nr_pages = map->size >> PAGE_SHIFT; - err |= kho_preserve_pages(page, nr_pages); + err = kho_preserve_pages(page, nr_pages); + if (err) + goto err_unpreserve; } - err |= kho_preserve_folio(page_folio(kho_fdt)); - err |= kho_add_subtree(ser, MEMBLOCK_KHO_FDT, page_to_virt(kho_fdt)); + return 0; - return notifier_from_errno(err); -} +err_unpreserve: + for (unsigned int i = 0; i < nr_preserved; i++) { + struct reserve_mem_table *map = &reserved_mem_table[i]; + struct page *page = phys_to_page(map->start); + unsigned int nr_pages = map->size >> PAGE_SHIFT; -static int reserve_mem_kho_notifier(struct notifier_block *self, - unsigned long cmd, void *v) -{ - switch (cmd) { - case KEXEC_KHO_FINALIZE: - return reserve_mem_kho_finalize((struct kho_serialization *)v); - case KEXEC_KHO_ABORT: - return NOTIFY_DONE; - default: - return NOTIFY_BAD; + kho_unpreserve_pages(page, nr_pages); } -} -static struct notifier_block reserve_mem_kho_nb = { - .notifier_call = reserve_mem_kho_notifier, -}; + return err; +} static int __init prepare_kho_fdt(void) { - int err = 0, i; + struct page *fdt_page; void *fdt; + int err; - kho_fdt = alloc_page(GFP_KERNEL); - if (!kho_fdt) - return -ENOMEM; + fdt_page = alloc_page(GFP_KERNEL); + if (!fdt_page) { + err = -ENOMEM; + goto err_report; + } - fdt = page_to_virt(kho_fdt); + fdt = page_to_virt(fdt_page); + err = kho_preserve_pages(fdt_page, 1); + if (err) + goto err_free_fdt; err |= fdt_create(fdt, PAGE_SIZE); err |= fdt_finish_reservemap(fdt); - err |= fdt_begin_node(fdt, ""); err |= fdt_property_string(fdt, "compatible", MEMBLOCK_KHO_NODE_COMPATIBLE); - for (i = 0; i < reserved_mem_count; i++) { + + for (unsigned int i = 0; !err && i < reserved_mem_count; i++) { struct reserve_mem_table *map = &reserved_mem_table[i]; err |= fdt_begin_node(fdt, map->name); @@ -2508,14 +2507,29 @@ static int __init prepare_kho_fdt(void) err |= fdt_end_node(fdt); } err |= fdt_end_node(fdt); - err |= fdt_finish(fdt); - if (err) { - pr_err("failed to prepare memblock FDT for KHO: %d\n", err); - put_page(kho_fdt); - kho_fdt = NULL; - } + if (err) + goto err_unpreserve_fdt; + + err = kho_add_subtree(MEMBLOCK_KHO_FDT, fdt); + if (err) + goto err_unpreserve_fdt; + + err = reserved_mem_preserve(); + if (err) + goto err_remove_subtree; + + return 0; + +err_remove_subtree: + kho_remove_subtree(fdt); +err_unpreserve_fdt: + kho_unpreserve_pages(fdt_page, 1); +err_free_fdt: + put_page(fdt_page); +err_report: + pr_err("failed to prepare memblock FDT for KHO: %d\n", err); return err; } @@ -2530,13 +2544,6 @@ static int __init reserve_mem_init(void) err = prepare_kho_fdt(); if (err) return err; - - err = register_kho_notifier(&reserve_mem_kho_nb); - if (err) { - put_page(kho_fdt); - kho_fdt = NULL; - } - return err; } late_initcall(reserve_mem_init); diff --git a/mm/memfd_luo.c b/mm/memfd_luo.c new file mode 100644 index 000000000000..4f6ba63b4310 --- /dev/null +++ b/mm/memfd_luo.c @@ -0,0 +1,516 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + * + * Copyright (C) 2025 Amazon.com Inc. or its affiliates. + * Pratyush Yadav <ptyadav@amazon.de> + */ + +/** + * DOC: Memfd Preservation via LUO + * + * Overview + * ======== + * + * Memory file descriptors (memfd) can be preserved over a kexec using the Live + * Update Orchestrator (LUO) file preservation. This allows userspace to + * transfer its memory contents to the next kernel after a kexec. + * + * The preservation is not intended to be transparent. Only select properties of + * the file are preserved. All others are reset to default. The preserved + * properties are described below. + * + * .. note:: + * The LUO API is not stabilized yet, so the preserved properties of a memfd + * are also not stable and are subject to backwards incompatible changes. + * + * .. note:: + * Currently a memfd backed by Hugetlb is not supported. Memfds created + * with ``MFD_HUGETLB`` will be rejected. + * + * Preserved Properties + * ==================== + * + * The following properties of the memfd are preserved across kexec: + * + * File Contents + * All data stored in the file is preserved. + * + * File Size + * The size of the file is preserved. Holes in the file are filled by + * allocating pages for them during preservation. + * + * File Position + * The current file position is preserved, allowing applications to continue + * reading/writing from their last position. + * + * File Status Flags + * memfds are always opened with ``O_RDWR`` and ``O_LARGEFILE``. This property + * is maintained. + * + * Non-Preserved Properties + * ======================== + * + * All properties which are not preserved must be assumed to be reset to + * default. This section describes some of those properties which may be more of + * note. + * + * ``FD_CLOEXEC`` flag + * A memfd can be created with the ``MFD_CLOEXEC`` flag that sets the + * ``FD_CLOEXEC`` on the file. This flag is not preserved and must be set + * again after restore via ``fcntl()``. + * + * Seals + * File seals are not preserved. The file is unsealed on restore and if + * needed, must be sealed again via ``fcntl()``. + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/bits.h> +#include <linux/err.h> +#include <linux/file.h> +#include <linux/io.h> +#include <linux/kexec_handover.h> +#include <linux/kho/abi/memfd.h> +#include <linux/liveupdate.h> +#include <linux/shmem_fs.h> +#include <linux/vmalloc.h> +#include "internal.h" + +static int memfd_luo_preserve_folios(struct file *file, + struct kho_vmalloc *kho_vmalloc, + struct memfd_luo_folio_ser **out_folios_ser, + u64 *nr_foliosp) +{ + struct inode *inode = file_inode(file); + struct memfd_luo_folio_ser *folios_ser; + unsigned int max_folios; + long i, size, nr_pinned; + struct folio **folios; + int err = -EINVAL; + pgoff_t offset; + u64 nr_folios; + + size = i_size_read(inode); + /* + * If the file has zero size, then the folios and nr_folios properties + * are not set. + */ + if (!size) { + *nr_foliosp = 0; + *out_folios_ser = NULL; + memset(kho_vmalloc, 0, sizeof(*kho_vmalloc)); + return 0; + } + + /* + * Guess the number of folios based on inode size. Real number might end + * up being smaller if there are higher order folios. + */ + max_folios = PAGE_ALIGN(size) / PAGE_SIZE; + folios = kvmalloc_array(max_folios, sizeof(*folios), GFP_KERNEL); + if (!folios) + return -ENOMEM; + + /* + * Pin the folios so they don't move around behind our back. This also + * ensures none of the folios are in CMA -- which ensures they don't + * fall in KHO scratch memory. It also moves swapped out folios back to + * memory. + * + * A side effect of doing this is that it allocates a folio for all + * indices in the file. This might waste memory on sparse memfds. If + * that is really a problem in the future, we can have a + * memfd_pin_folios() variant that does not allocate a page on empty + * slots. + */ + nr_pinned = memfd_pin_folios(file, 0, size - 1, folios, max_folios, + &offset); + if (nr_pinned < 0) { + err = nr_pinned; + pr_err("failed to pin folios: %d\n", err); + goto err_free_folios; + } + nr_folios = nr_pinned; + + folios_ser = vcalloc(nr_folios, sizeof(*folios_ser)); + if (!folios_ser) { + err = -ENOMEM; + goto err_unpin; + } + + for (i = 0; i < nr_folios; i++) { + struct memfd_luo_folio_ser *pfolio = &folios_ser[i]; + struct folio *folio = folios[i]; + unsigned int flags = 0; + + err = kho_preserve_folio(folio); + if (err) + goto err_unpreserve; + + if (folio_test_dirty(folio)) + flags |= MEMFD_LUO_FOLIO_DIRTY; + if (folio_test_uptodate(folio)) + flags |= MEMFD_LUO_FOLIO_UPTODATE; + + pfolio->pfn = folio_pfn(folio); + pfolio->flags = flags; + pfolio->index = folio->index; + } + + err = kho_preserve_vmalloc(folios_ser, kho_vmalloc); + if (err) + goto err_unpreserve; + + kvfree(folios); + *nr_foliosp = nr_folios; + *out_folios_ser = folios_ser; + + /* + * Note: folios_ser is purposely not freed here. It is preserved + * memory (via KHO). In the 'unpreserve' path, we use the vmap pointer + * that is passed via private_data. + */ + return 0; + +err_unpreserve: + for (i = i - 1; i >= 0; i--) + kho_unpreserve_folio(folios[i]); + vfree(folios_ser); +err_unpin: + unpin_folios(folios, nr_folios); +err_free_folios: + kvfree(folios); + + return err; +} + +static void memfd_luo_unpreserve_folios(struct kho_vmalloc *kho_vmalloc, + struct memfd_luo_folio_ser *folios_ser, + u64 nr_folios) +{ + long i; + + if (!nr_folios) + return; + + kho_unpreserve_vmalloc(kho_vmalloc); + + for (i = 0; i < nr_folios; i++) { + const struct memfd_luo_folio_ser *pfolio = &folios_ser[i]; + struct folio *folio; + + if (!pfolio->pfn) + continue; + + folio = pfn_folio(pfolio->pfn); + + kho_unpreserve_folio(folio); + unpin_folio(folio); + } + + vfree(folios_ser); +} + +static int memfd_luo_preserve(struct liveupdate_file_op_args *args) +{ + struct inode *inode = file_inode(args->file); + struct memfd_luo_folio_ser *folios_ser; + struct memfd_luo_ser *ser; + u64 nr_folios; + int err = 0; + + inode_lock(inode); + shmem_freeze(inode, true); + + /* Allocate the main serialization structure in preserved memory */ + ser = kho_alloc_preserve(sizeof(*ser)); + if (IS_ERR(ser)) { + err = PTR_ERR(ser); + goto err_unlock; + } + + ser->pos = args->file->f_pos; + ser->size = i_size_read(inode); + + err = memfd_luo_preserve_folios(args->file, &ser->folios, + &folios_ser, &nr_folios); + if (err) + goto err_free_ser; + + ser->nr_folios = nr_folios; + inode_unlock(inode); + + args->private_data = folios_ser; + args->serialized_data = virt_to_phys(ser); + + return 0; + +err_free_ser: + kho_unpreserve_free(ser); +err_unlock: + shmem_freeze(inode, false); + inode_unlock(inode); + return err; +} + +static int memfd_luo_freeze(struct liveupdate_file_op_args *args) +{ + struct memfd_luo_ser *ser; + + if (WARN_ON_ONCE(!args->serialized_data)) + return -EINVAL; + + ser = phys_to_virt(args->serialized_data); + + /* + * The pos might have changed since prepare. Everything else stays the + * same. + */ + ser->pos = args->file->f_pos; + + return 0; +} + +static void memfd_luo_unpreserve(struct liveupdate_file_op_args *args) +{ + struct inode *inode = file_inode(args->file); + struct memfd_luo_ser *ser; + + if (WARN_ON_ONCE(!args->serialized_data)) + return; + + inode_lock(inode); + shmem_freeze(inode, false); + + ser = phys_to_virt(args->serialized_data); + + memfd_luo_unpreserve_folios(&ser->folios, args->private_data, + ser->nr_folios); + + kho_unpreserve_free(ser); + inode_unlock(inode); +} + +static void memfd_luo_discard_folios(const struct memfd_luo_folio_ser *folios_ser, + u64 nr_folios) +{ + u64 i; + + for (i = 0; i < nr_folios; i++) { + const struct memfd_luo_folio_ser *pfolio = &folios_ser[i]; + struct folio *folio; + phys_addr_t phys; + + if (!pfolio->pfn) + continue; + + phys = PFN_PHYS(pfolio->pfn); + folio = kho_restore_folio(phys); + if (!folio) { + pr_warn_ratelimited("Unable to restore folio at physical address: %llx\n", + phys); + continue; + } + + folio_put(folio); + } +} + +static void memfd_luo_finish(struct liveupdate_file_op_args *args) +{ + struct memfd_luo_folio_ser *folios_ser; + struct memfd_luo_ser *ser; + + if (args->retrieved) + return; + + ser = phys_to_virt(args->serialized_data); + if (!ser) + return; + + if (ser->nr_folios) { + folios_ser = kho_restore_vmalloc(&ser->folios); + if (!folios_ser) + goto out; + + memfd_luo_discard_folios(folios_ser, ser->nr_folios); + vfree(folios_ser); + } + +out: + kho_restore_free(ser); +} + +static int memfd_luo_retrieve_folios(struct file *file, + struct memfd_luo_folio_ser *folios_ser, + u64 nr_folios) +{ + struct inode *inode = file_inode(file); + struct address_space *mapping = inode->i_mapping; + struct folio *folio; + int err = -EIO; + long i; + + for (i = 0; i < nr_folios; i++) { + const struct memfd_luo_folio_ser *pfolio = &folios_ser[i]; + phys_addr_t phys; + u64 index; + int flags; + + if (!pfolio->pfn) + continue; + + phys = PFN_PHYS(pfolio->pfn); + folio = kho_restore_folio(phys); + if (!folio) { + pr_err("Unable to restore folio at physical address: %llx\n", + phys); + goto put_folios; + } + index = pfolio->index; + flags = pfolio->flags; + + /* Set up the folio for insertion. */ + __folio_set_locked(folio); + __folio_set_swapbacked(folio); + + err = mem_cgroup_charge(folio, NULL, mapping_gfp_mask(mapping)); + if (err) { + pr_err("shmem: failed to charge folio index %ld: %d\n", + i, err); + goto unlock_folio; + } + + err = shmem_add_to_page_cache(folio, mapping, index, NULL, + mapping_gfp_mask(mapping)); + if (err) { + pr_err("shmem: failed to add to page cache folio index %ld: %d\n", + i, err); + goto unlock_folio; + } + + if (flags & MEMFD_LUO_FOLIO_UPTODATE) + folio_mark_uptodate(folio); + if (flags & MEMFD_LUO_FOLIO_DIRTY) + folio_mark_dirty(folio); + + err = shmem_inode_acct_blocks(inode, 1); + if (err) { + pr_err("shmem: failed to account folio index %ld: %d\n", + i, err); + goto unlock_folio; + } + + shmem_recalc_inode(inode, 1, 0); + folio_add_lru(folio); + folio_unlock(folio); + folio_put(folio); + } + + return 0; + +unlock_folio: + folio_unlock(folio); + folio_put(folio); +put_folios: + /* + * Note: don't free the folios already added to the file. They will be + * freed when the file is freed. Free the ones not added yet here. + */ + for (long j = i + 1; j < nr_folios; j++) { + const struct memfd_luo_folio_ser *pfolio = &folios_ser[j]; + + folio = kho_restore_folio(pfolio->pfn); + if (folio) + folio_put(folio); + } + + return err; +} + +static int memfd_luo_retrieve(struct liveupdate_file_op_args *args) +{ + struct memfd_luo_folio_ser *folios_ser; + struct memfd_luo_ser *ser; + struct file *file; + int err; + + ser = phys_to_virt(args->serialized_data); + if (!ser) + return -EINVAL; + + file = shmem_file_setup("", 0, VM_NORESERVE); + + if (IS_ERR(file)) { + pr_err("failed to setup file: %pe\n", file); + return PTR_ERR(file); + } + + vfs_setpos(file, ser->pos, MAX_LFS_FILESIZE); + file->f_inode->i_size = ser->size; + + if (ser->nr_folios) { + folios_ser = kho_restore_vmalloc(&ser->folios); + if (!folios_ser) { + err = -EINVAL; + goto put_file; + } + + err = memfd_luo_retrieve_folios(file, folios_ser, ser->nr_folios); + vfree(folios_ser); + if (err) + goto put_file; + } + + args->file = file; + kho_restore_free(ser); + + return 0; + +put_file: + fput(file); + + return err; +} + +static bool memfd_luo_can_preserve(struct liveupdate_file_handler *handler, + struct file *file) +{ + struct inode *inode = file_inode(file); + + return shmem_file(file) && !inode->i_nlink; +} + +static const struct liveupdate_file_ops memfd_luo_file_ops = { + .freeze = memfd_luo_freeze, + .finish = memfd_luo_finish, + .retrieve = memfd_luo_retrieve, + .preserve = memfd_luo_preserve, + .unpreserve = memfd_luo_unpreserve, + .can_preserve = memfd_luo_can_preserve, + .owner = THIS_MODULE, +}; + +static struct liveupdate_file_handler memfd_luo_handler = { + .ops = &memfd_luo_file_ops, + .compatible = MEMFD_LUO_FH_COMPATIBLE, +}; + +static int __init memfd_luo_init(void) +{ + int err = liveupdate_register_file_handler(&memfd_luo_handler); + + if (err && err != -EOPNOTSUPP) { + pr_err("Could not register luo filesystem handler: %pe\n", + ERR_PTR(err)); + + return err; + } + + return 0; +} +late_initcall(memfd_luo_init); diff --git a/mm/shmem.c b/mm/shmem.c index d578d8e765d7..3f194c9842a8 100644 --- a/mm/shmem.c +++ b/mm/shmem.c @@ -174,20 +174,20 @@ static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb) */ static inline int shmem_acct_size(unsigned long flags, loff_t size) { - return (flags & VM_NORESERVE) ? + return (flags & SHMEM_F_NORESERVE) ? 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size)); } static inline void shmem_unacct_size(unsigned long flags, loff_t size) { - if (!(flags & VM_NORESERVE)) + if (!(flags & SHMEM_F_NORESERVE)) vm_unacct_memory(VM_ACCT(size)); } static inline int shmem_reacct_size(unsigned long flags, loff_t oldsize, loff_t newsize) { - if (!(flags & VM_NORESERVE)) { + if (!(flags & SHMEM_F_NORESERVE)) { if (VM_ACCT(newsize) > VM_ACCT(oldsize)) return security_vm_enough_memory_mm(current->mm, VM_ACCT(newsize) - VM_ACCT(oldsize)); @@ -205,7 +205,7 @@ static inline int shmem_reacct_size(unsigned long flags, */ static inline int shmem_acct_blocks(unsigned long flags, long pages) { - if (!(flags & VM_NORESERVE)) + if (!(flags & SHMEM_F_NORESERVE)) return 0; return security_vm_enough_memory_mm(current->mm, @@ -214,11 +214,11 @@ static inline int shmem_acct_blocks(unsigned long flags, long pages) static inline void shmem_unacct_blocks(unsigned long flags, long pages) { - if (flags & VM_NORESERVE) + if (flags & SHMEM_F_NORESERVE) vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE)); } -static int shmem_inode_acct_blocks(struct inode *inode, long pages) +int shmem_inode_acct_blocks(struct inode *inode, long pages) { struct shmem_inode_info *info = SHMEM_I(inode); struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb); @@ -434,7 +434,7 @@ static void shmem_free_inode(struct super_block *sb, size_t freed_ispace) * * Return: true if swapped was incremented from 0, for shmem_writeout(). */ -static bool shmem_recalc_inode(struct inode *inode, long alloced, long swapped) +bool shmem_recalc_inode(struct inode *inode, long alloced, long swapped) { struct shmem_inode_info *info = SHMEM_I(inode); bool first_swapped = false; @@ -878,9 +878,9 @@ static void shmem_update_stats(struct folio *folio, int nr_pages) /* * Somewhat like filemap_add_folio, but error if expected item has gone. */ -static int shmem_add_to_page_cache(struct folio *folio, - struct address_space *mapping, - pgoff_t index, void *expected, gfp_t gfp) +int shmem_add_to_page_cache(struct folio *folio, + struct address_space *mapping, + pgoff_t index, void *expected, gfp_t gfp) { XA_STATE_ORDER(xas, &mapping->i_pages, index, folio_order(folio)); unsigned long nr = folio_nr_pages(folio); @@ -1314,6 +1314,8 @@ static int shmem_setattr(struct mnt_idmap *idmap, return -EPERM; if (newsize != oldsize) { + if (info->flags & SHMEM_F_MAPPING_FROZEN) + return -EPERM; error = shmem_reacct_size(SHMEM_I(inode)->flags, oldsize, newsize); if (error) @@ -1568,7 +1570,7 @@ int shmem_writeout(struct folio *folio, struct swap_iocb **plug, int nr_pages; bool split = false; - if ((info->flags & VM_LOCKED) || sbinfo->noswap) + if ((info->flags & SHMEM_F_LOCKED) || sbinfo->noswap) goto redirty; if (!total_swap_pages) @@ -2926,15 +2928,15 @@ int shmem_lock(struct file *file, int lock, struct ucounts *ucounts) * ipc_lock_object() when called from shmctl_do_lock(), * no serialization needed when called from shm_destroy(). */ - if (lock && !(info->flags & VM_LOCKED)) { + if (lock && !(info->flags & SHMEM_F_LOCKED)) { if (!user_shm_lock(inode->i_size, ucounts)) goto out_nomem; - info->flags |= VM_LOCKED; + info->flags |= SHMEM_F_LOCKED; mapping_set_unevictable(file->f_mapping); } - if (!lock && (info->flags & VM_LOCKED) && ucounts) { + if (!lock && (info->flags & SHMEM_F_LOCKED) && ucounts) { user_shm_unlock(inode->i_size, ucounts); - info->flags &= ~VM_LOCKED; + info->flags &= ~SHMEM_F_LOCKED; mapping_clear_unevictable(file->f_mapping); } retval = 0; @@ -3079,7 +3081,7 @@ static struct inode *__shmem_get_inode(struct mnt_idmap *idmap, spin_lock_init(&info->lock); atomic_set(&info->stop_eviction, 0); info->seals = F_SEAL_SEAL; - info->flags = flags & VM_NORESERVE; + info->flags = (flags & VM_NORESERVE) ? SHMEM_F_NORESERVE : 0; info->i_crtime = inode_get_mtime(inode); info->fsflags = (dir == NULL) ? 0 : SHMEM_I(dir)->fsflags & SHMEM_FL_INHERITED; @@ -3306,6 +3308,10 @@ shmem_write_begin(const struct kiocb *iocb, struct address_space *mapping, return -EPERM; } + if (unlikely((info->flags & SHMEM_F_MAPPING_FROZEN) && + pos + len > inode->i_size)) + return -EPERM; + ret = shmem_get_folio(inode, index, pos + len, &folio, SGP_WRITE); if (ret) return ret; @@ -3679,6 +3685,11 @@ static long shmem_fallocate(struct file *file, int mode, loff_t offset, inode_lock(inode); + if (info->flags & SHMEM_F_MAPPING_FROZEN) { + error = -EPERM; + goto out; + } + if (mode & FALLOC_FL_PUNCH_HOLE) { struct address_space *mapping = file->f_mapping; loff_t unmap_start = round_up(offset, PAGE_SIZE); @@ -5799,8 +5810,10 @@ static inline struct inode *shmem_get_inode(struct mnt_idmap *idmap, /* common code */ static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, - loff_t size, unsigned long flags, unsigned int i_flags) + loff_t size, unsigned long vm_flags, + unsigned int i_flags) { + unsigned long flags = (vm_flags & VM_NORESERVE) ? SHMEM_F_NORESERVE : 0; struct inode *inode; struct file *res; @@ -5817,7 +5830,7 @@ static struct file *__shmem_file_setup(struct vfsmount *mnt, const char *name, return ERR_PTR(-ENOMEM); inode = shmem_get_inode(&nop_mnt_idmap, mnt->mnt_sb, NULL, - S_IFREG | S_IRWXUGO, 0, flags); + S_IFREG | S_IRWXUGO, 0, vm_flags); if (IS_ERR(inode)) { shmem_unacct_size(flags, size); return ERR_CAST(inode); diff --git a/net/sunrpc/svcsock.c b/net/sunrpc/svcsock.c index 16ff6c100821..d61cd9b40491 100644 --- a/net/sunrpc/svcsock.c +++ b/net/sunrpc/svcsock.c @@ -68,6 +68,17 @@ #define RPCDBG_FACILITY RPCDBG_SVCXPRT +/* + * For UDP: + * 1 for header page + * enough pages for RPCSVC_MAXPAYLOAD_UDP + * 1 in case payload is not aligned + * 1 for tail page + */ +enum { + SUNRPC_MAX_UDP_SENDPAGES = 1 + RPCSVC_MAXPAYLOAD_UDP / PAGE_SIZE + 1 + 1 +}; + /* To-do: to avoid tying up an nfsd thread while waiting for a * handshake request, the request could instead be deferred. */ @@ -740,14 +751,14 @@ static int svc_udp_sendto(struct svc_rqst *rqstp) if (svc_xprt_is_dead(xprt)) goto out_notconn; - count = xdr_buf_to_bvec(rqstp->rq_bvec, rqstp->rq_maxpages, xdr); + count = xdr_buf_to_bvec(svsk->sk_bvec, SUNRPC_MAX_UDP_SENDPAGES, xdr); - iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, svsk->sk_bvec, count, rqstp->rq_res.len); err = sock_sendmsg(svsk->sk_sock, &msg); if (err == -ECONNREFUSED) { /* ICMP error on earlier request. */ - iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, svsk->sk_bvec, count, rqstp->rq_res.len); err = sock_sendmsg(svsk->sk_sock, &msg); } @@ -1062,9 +1073,10 @@ static ssize_t svc_tcp_read_marker(struct svc_sock *svsk, return svc_sock_reclen(svsk); err_too_large: - net_notice_ratelimited("svc: %s %s RPC fragment too large: %d\n", - __func__, svsk->sk_xprt.xpt_server->sv_name, - svc_sock_reclen(svsk)); + net_notice_ratelimited("svc: %s oversized RPC fragment (%u octets) from %pISpc\n", + svsk->sk_xprt.xpt_server->sv_name, + svc_sock_reclen(svsk), + (struct sockaddr *)&svsk->sk_xprt.xpt_remote); svc_xprt_deferred_close(&svsk->sk_xprt); err_short: return -EAGAIN; @@ -1235,19 +1247,19 @@ static int svc_tcp_sendmsg(struct svc_sock *svsk, struct svc_rqst *rqstp, int ret; /* The stream record marker is copied into a temporary page - * fragment buffer so that it can be included in rq_bvec. + * fragment buffer so that it can be included in sk_bvec. */ buf = page_frag_alloc(&svsk->sk_frag_cache, sizeof(marker), GFP_KERNEL); if (!buf) return -ENOMEM; memcpy(buf, &marker, sizeof(marker)); - bvec_set_virt(rqstp->rq_bvec, buf, sizeof(marker)); + bvec_set_virt(svsk->sk_bvec, buf, sizeof(marker)); - count = xdr_buf_to_bvec(rqstp->rq_bvec + 1, rqstp->rq_maxpages, + count = xdr_buf_to_bvec(svsk->sk_bvec + 1, rqstp->rq_maxpages, &rqstp->rq_res); - iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, rqstp->rq_bvec, + iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, svsk->sk_bvec, 1 + count, sizeof(marker) + rqstp->rq_res.len); ret = sock_sendmsg(svsk->sk_sock, &msg); page_frag_free(buf); @@ -1392,6 +1404,20 @@ void svc_sock_update_bufs(struct svc_serv *serv) spin_unlock_bh(&serv->sv_lock); } +static int svc_sock_sendpages(struct svc_serv *serv, struct socket *sock, int flags) +{ + switch (sock->type) { + case SOCK_STREAM: + /* +1 for TCP record marker */ + if (flags & SVC_SOCK_TEMPORARY) + return svc_serv_maxpages(serv) + 1; + return 0; + case SOCK_DGRAM: + return SUNRPC_MAX_UDP_SENDPAGES; + } + return -EINVAL; +} + /* * Initialize socket for RPC use and create svc_sock struct */ @@ -1402,12 +1428,26 @@ static struct svc_sock *svc_setup_socket(struct svc_serv *serv, struct svc_sock *svsk; struct sock *inet; int pmap_register = !(flags & SVC_SOCK_ANONYMOUS); + int sendpages; unsigned long pages; + sendpages = svc_sock_sendpages(serv, sock, flags); + if (sendpages < 0) + return ERR_PTR(sendpages); + pages = svc_serv_maxpages(serv); svsk = kzalloc(struct_size(svsk, sk_pages, pages), GFP_KERNEL); if (!svsk) return ERR_PTR(-ENOMEM); + + if (sendpages) { + svsk->sk_bvec = kcalloc(sendpages, sizeof(*svsk->sk_bvec), GFP_KERNEL); + if (!svsk->sk_bvec) { + kfree(svsk); + return ERR_PTR(-ENOMEM); + } + } + svsk->sk_maxpages = pages; inet = sock->sk; @@ -1419,6 +1459,7 @@ static struct svc_sock *svc_setup_socket(struct svc_serv *serv, inet->sk_protocol, ntohs(inet_sk(inet)->inet_sport)); if (err < 0) { + kfree(svsk->sk_bvec); kfree(svsk); return ERR_PTR(err); } @@ -1636,5 +1677,6 @@ static void svc_sock_free(struct svc_xprt *xprt) sock_release(sock); page_frag_cache_drain(&svsk->sk_frag_cache); + kfree(svsk->sk_bvec); kfree(svsk); } diff --git a/net/sunrpc/xprtrdma/svc_rdma_transport.c b/net/sunrpc/xprtrdma/svc_rdma_transport.c index 3d7f1413df02..b7b318ad25c4 100644 --- a/net/sunrpc/xprtrdma/svc_rdma_transport.c +++ b/net/sunrpc/xprtrdma/svc_rdma_transport.c @@ -591,12 +591,18 @@ static void svc_rdma_detach(struct svc_xprt *xprt) rdma_disconnect(rdma->sc_cm_id); } -static void __svc_rdma_free(struct work_struct *work) +/** + * svc_rdma_free - Release class-specific transport resources + * @xprt: Generic svc transport object + */ +static void svc_rdma_free(struct svc_xprt *xprt) { struct svcxprt_rdma *rdma = - container_of(work, struct svcxprt_rdma, sc_work); + container_of(xprt, struct svcxprt_rdma, sc_xprt); struct ib_device *device = rdma->sc_cm_id->device; + might_sleep(); + /* This blocks until the Completion Queues are empty */ if (rdma->sc_qp && !IS_ERR(rdma->sc_qp)) ib_drain_qp(rdma->sc_qp); @@ -629,15 +635,6 @@ static void __svc_rdma_free(struct work_struct *work) kfree(rdma); } -static void svc_rdma_free(struct svc_xprt *xprt) -{ - struct svcxprt_rdma *rdma = - container_of(xprt, struct svcxprt_rdma, sc_xprt); - - INIT_WORK(&rdma->sc_work, __svc_rdma_free); - schedule_work(&rdma->sc_work); -} - static int svc_rdma_has_wspace(struct svc_xprt *xprt) { struct svcxprt_rdma *rdma = diff --git a/rust/helpers/rbtree.c b/rust/helpers/rbtree.c index 6d404b84a9b5..2a0eabbb4160 100644 --- a/rust/helpers/rbtree.c +++ b/rust/helpers/rbtree.c @@ -7,3 +7,13 @@ void rust_helper_rb_link_node(struct rb_node *node, struct rb_node *parent, { rb_link_node(node, parent, rb_link); } + +struct rb_node *rust_helper_rb_first(const struct rb_root *root) +{ + return rb_first(root); +} + +struct rb_node *rust_helper_rb_last(const struct rb_root *root) +{ + return rb_last(root); +} diff --git a/rust/helpers/uaccess.c b/rust/helpers/uaccess.c index f49076f813cd..4629b2d15529 100644 --- a/rust/helpers/uaccess.c +++ b/rust/helpers/uaccess.c @@ -13,3 +13,15 @@ unsigned long rust_helper_copy_to_user(void __user *to, const void *from, { return copy_to_user(to, from, n); } + +#ifdef INLINE_COPY_FROM_USER +unsigned long rust_helper__copy_from_user(void *to, const void __user *from, unsigned long n) +{ + return _inline_copy_from_user(to, from, n); +} + +unsigned long rust_helper__copy_to_user(void __user *to, const void *from, unsigned long n) +{ + return _inline_copy_to_user(to, from, n); +} +#endif diff --git a/rust/kernel/bitmap.rs b/rust/kernel/bitmap.rs index aa8fc7bf06fc..83d7dea99137 100644 --- a/rust/kernel/bitmap.rs +++ b/rust/kernel/bitmap.rs @@ -12,8 +12,6 @@ use crate::bindings; use crate::pr_err; use core::ptr::NonNull; -const BITS_PER_LONG: usize = bindings::BITS_PER_LONG as usize; - /// Represents a C bitmap. Wraps underlying C bitmap API. /// /// # Invariants @@ -149,14 +147,14 @@ macro_rules! bitmap_assert_return { /// /// # Invariants /// -/// * `nbits` is `<= i32::MAX` and never changes. -/// * if `nbits <= bindings::BITS_PER_LONG`, then `repr` is a `usize`. +/// * `nbits` is `<= MAX_LEN`. +/// * if `nbits <= MAX_INLINE_LEN`, then `repr` is a `usize`. /// * otherwise, `repr` holds a non-null pointer to an initialized /// array of `unsigned long` that is large enough to hold `nbits` bits. pub struct BitmapVec { /// Representation of bitmap. repr: BitmapRepr, - /// Length of this bitmap. Must be `<= i32::MAX`. + /// Length of this bitmap. Must be `<= MAX_LEN`. nbits: usize, } @@ -164,7 +162,7 @@ impl core::ops::Deref for BitmapVec { type Target = Bitmap; fn deref(&self) -> &Bitmap { - let ptr = if self.nbits <= BITS_PER_LONG { + let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN { // SAFETY: Bitmap is represented inline. #[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")] unsafe { @@ -183,7 +181,7 @@ impl core::ops::Deref for BitmapVec { impl core::ops::DerefMut for BitmapVec { fn deref_mut(&mut self) -> &mut Bitmap { - let ptr = if self.nbits <= BITS_PER_LONG { + let ptr = if self.nbits <= BitmapVec::MAX_INLINE_LEN { // SAFETY: Bitmap is represented inline. #[allow(unused_unsafe, reason = "Safe since Rust 1.92.0")] unsafe { @@ -213,7 +211,7 @@ unsafe impl Sync for BitmapVec {} impl Drop for BitmapVec { fn drop(&mut self) { - if self.nbits <= BITS_PER_LONG { + if self.nbits <= BitmapVec::MAX_INLINE_LEN { return; } // SAFETY: `self.ptr` was returned by the C `bitmap_zalloc`. @@ -226,23 +224,39 @@ impl Drop for BitmapVec { } impl BitmapVec { + /// The maximum possible length of a `BitmapVec`. + pub const MAX_LEN: usize = i32::MAX as usize; + + /// The maximum length that uses the inline representation. + pub const MAX_INLINE_LEN: usize = usize::BITS as usize; + + /// Construct a longest possible inline [`BitmapVec`]. + #[inline] + pub fn new_inline() -> Self { + // INVARIANT: `nbits <= MAX_INLINE_LEN`, so an inline bitmap is the right repr. + BitmapVec { + repr: BitmapRepr { bitmap: 0 }, + nbits: BitmapVec::MAX_INLINE_LEN, + } + } + /// Constructs a new [`BitmapVec`]. /// /// Fails with [`AllocError`] when the [`BitmapVec`] could not be allocated. This - /// includes the case when `nbits` is greater than `i32::MAX`. + /// includes the case when `nbits` is greater than `MAX_LEN`. #[inline] pub fn new(nbits: usize, flags: Flags) -> Result<Self, AllocError> { - if nbits <= BITS_PER_LONG { + if nbits <= BitmapVec::MAX_INLINE_LEN { return Ok(BitmapVec { repr: BitmapRepr { bitmap: 0 }, nbits, }); } - if nbits > i32::MAX.try_into().unwrap() { + if nbits > Self::MAX_LEN { return Err(AllocError); } let nbits_u32 = u32::try_from(nbits).unwrap(); - // SAFETY: `BITS_PER_LONG < nbits` and `nbits <= i32::MAX`. + // SAFETY: `MAX_INLINE_LEN < nbits` and `nbits <= MAX_LEN`. let ptr = unsafe { bindings::bitmap_zalloc(nbits_u32, flags.as_raw()) }; let ptr = NonNull::new(ptr).ok_or(AllocError)?; // INVARIANT: `ptr` returned by C `bitmap_zalloc` and `nbits` checked. @@ -495,9 +509,10 @@ mod tests { #[test] fn bitmap_borrow() { let fake_bitmap: [usize; 2] = [0, 0]; + let fake_bitmap_len = 2 * usize::BITS as usize; // SAFETY: `fake_c_bitmap` is an array of expected length. - let b = unsafe { Bitmap::from_raw(fake_bitmap.as_ptr(), 2 * BITS_PER_LONG) }; - assert_eq!(2 * BITS_PER_LONG, b.len()); + let b = unsafe { Bitmap::from_raw(fake_bitmap.as_ptr(), fake_bitmap_len) }; + assert_eq!(fake_bitmap_len, b.len()); assert_eq!(None, b.next_bit(0)); } diff --git a/rust/kernel/id_pool.rs b/rust/kernel/id_pool.rs index a41a3404213c..384753fe0e44 100644 --- a/rust/kernel/id_pool.rs +++ b/rust/kernel/id_pool.rs @@ -7,8 +7,6 @@ use crate::alloc::{AllocError, Flags}; use crate::bitmap::BitmapVec; -const BITS_PER_LONG: usize = bindings::BITS_PER_LONG as usize; - /// Represents a dynamic ID pool backed by a [`BitmapVec`]. /// /// Clients acquire and release IDs from unset bits in a bitmap. @@ -25,22 +23,22 @@ const BITS_PER_LONG: usize = bindings::BITS_PER_LONG as usize; /// Basic usage /// /// ``` -/// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; -/// use kernel::id_pool::IdPool; +/// use kernel::alloc::AllocError; +/// use kernel::id_pool::{IdPool, UnusedId}; /// -/// let mut pool = IdPool::new(64, GFP_KERNEL)?; +/// let mut pool = IdPool::with_capacity(64, GFP_KERNEL)?; /// for i in 0..64 { -/// assert_eq!(i, pool.acquire_next_id(i).ok_or(ENOSPC)?); +/// assert_eq!(i, pool.find_unused_id(i).ok_or(ENOSPC)?.acquire()); /// } /// /// pool.release_id(23); -/// assert_eq!(23, pool.acquire_next_id(0).ok_or(ENOSPC)?); +/// assert_eq!(23, pool.find_unused_id(0).ok_or(ENOSPC)?.acquire()); /// -/// assert_eq!(None, pool.acquire_next_id(0)); // time to realloc. +/// assert!(pool.find_unused_id(0).is_none()); // time to realloc. /// let resizer = pool.grow_request().ok_or(ENOSPC)?.realloc(GFP_KERNEL)?; /// pool.grow(resizer); /// -/// assert_eq!(pool.acquire_next_id(0), Some(64)); +/// assert_eq!(pool.find_unused_id(0).ok_or(ENOSPC)?.acquire(), 64); /// # Ok::<(), Error>(()) /// ``` /// @@ -54,8 +52,8 @@ const BITS_PER_LONG: usize = bindings::BITS_PER_LONG as usize; /// fn get_id_maybe_realloc(guarded_pool: &SpinLock<IdPool>) -> Result<usize, AllocError> { /// let mut pool = guarded_pool.lock(); /// loop { -/// match pool.acquire_next_id(0) { -/// Some(index) => return Ok(index), +/// match pool.find_unused_id(0) { +/// Some(index) => return Ok(index.acquire()), /// None => { /// let alloc_request = pool.grow_request(); /// drop(pool); @@ -97,13 +95,24 @@ impl ReallocRequest { impl IdPool { /// Constructs a new [`IdPool`]. /// - /// A capacity below [`BITS_PER_LONG`] is adjusted to - /// [`BITS_PER_LONG`]. + /// The pool will have a capacity of [`MAX_INLINE_LEN`]. + /// + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN + #[inline] + pub fn new() -> Self { + Self { + map: BitmapVec::new_inline(), + } + } + + /// Constructs a new [`IdPool`] with space for a specific number of bits. + /// + /// A capacity below [`MAX_INLINE_LEN`] is adjusted to [`MAX_INLINE_LEN`]. /// - /// [`BITS_PER_LONG`]: srctree/include/asm-generic/bitsperlong.h + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN #[inline] - pub fn new(num_ids: usize, flags: Flags) -> Result<Self, AllocError> { - let num_ids = core::cmp::max(num_ids, BITS_PER_LONG); + pub fn with_capacity(num_ids: usize, flags: Flags) -> Result<Self, AllocError> { + let num_ids = usize::max(num_ids, BitmapVec::MAX_INLINE_LEN); let map = BitmapVec::new(num_ids, flags)?; Ok(Self { map }) } @@ -116,28 +125,34 @@ impl IdPool { /// Returns a [`ReallocRequest`] if the [`IdPool`] can be shrunk, [`None`] otherwise. /// - /// The capacity of an [`IdPool`] cannot be shrunk below [`BITS_PER_LONG`]. + /// The capacity of an [`IdPool`] cannot be shrunk below [`MAX_INLINE_LEN`]. /// - /// [`BITS_PER_LONG`]: srctree/include/asm-generic/bitsperlong.h + /// [`MAX_INLINE_LEN`]: BitmapVec::MAX_INLINE_LEN /// /// # Examples /// /// ``` - /// use kernel::alloc::{AllocError, flags::GFP_KERNEL}; - /// use kernel::id_pool::{ReallocRequest, IdPool}; + /// use kernel::{ + /// alloc::AllocError, + /// bitmap::BitmapVec, + /// id_pool::{ + /// IdPool, + /// ReallocRequest, + /// }, + /// }; /// - /// let mut pool = IdPool::new(1024, GFP_KERNEL)?; + /// let mut pool = IdPool::with_capacity(1024, GFP_KERNEL)?; /// let alloc_request = pool.shrink_request().ok_or(AllocError)?; /// let resizer = alloc_request.realloc(GFP_KERNEL)?; /// pool.shrink(resizer); - /// assert_eq!(pool.capacity(), kernel::bindings::BITS_PER_LONG as usize); + /// assert_eq!(pool.capacity(), BitmapVec::MAX_INLINE_LEN); /// # Ok::<(), AllocError>(()) /// ``` #[inline] pub fn shrink_request(&self) -> Option<ReallocRequest> { let cap = self.capacity(); - // Shrinking below [`BITS_PER_LONG`] is never possible. - if cap <= BITS_PER_LONG { + // Shrinking below `MAX_INLINE_LEN` is never possible. + if cap <= BitmapVec::MAX_INLINE_LEN { return None; } // Determine if the bitmap can shrink based on the position of @@ -146,13 +161,13 @@ impl IdPool { // bitmap should shrink to half its current size. let Some(bit) = self.map.last_bit() else { return Some(ReallocRequest { - num_ids: BITS_PER_LONG, + num_ids: BitmapVec::MAX_INLINE_LEN, }); }; if bit >= (cap / 4) { return None; } - let num_ids = usize::max(BITS_PER_LONG, cap / 2); + let num_ids = usize::max(BitmapVec::MAX_INLINE_LEN, cap / 2); Some(ReallocRequest { num_ids }) } @@ -177,11 +192,13 @@ impl IdPool { /// Returns a [`ReallocRequest`] for growing this [`IdPool`], if possible. /// - /// The capacity of an [`IdPool`] cannot be grown above [`i32::MAX`]. + /// The capacity of an [`IdPool`] cannot be grown above [`MAX_LEN`]. + /// + /// [`MAX_LEN`]: BitmapVec::MAX_LEN #[inline] pub fn grow_request(&self) -> Option<ReallocRequest> { let num_ids = self.capacity() * 2; - if num_ids > i32::MAX.try_into().unwrap() { + if num_ids > BitmapVec::MAX_LEN { return None; } Some(ReallocRequest { num_ids }) @@ -204,18 +221,18 @@ impl IdPool { self.map = resizer.new; } - /// Acquires a new ID by finding and setting the next zero bit in the - /// bitmap. + /// Finds an unused ID in the bitmap. /// /// Upon success, returns its index. Otherwise, returns [`None`] /// to indicate that a [`Self::grow_request`] is needed. #[inline] - pub fn acquire_next_id(&mut self, offset: usize) -> Option<usize> { - let next_zero_bit = self.map.next_zero_bit(offset); - if let Some(nr) = next_zero_bit { - self.map.set_bit(nr); - } - next_zero_bit + #[must_use] + pub fn find_unused_id(&mut self, offset: usize) -> Option<UnusedId<'_>> { + // INVARIANT: `next_zero_bit()` returns None or an integer less than `map.len()` + Some(UnusedId { + id: self.map.next_zero_bit(offset)?, + pool: self, + }) } /// Releases an ID. @@ -224,3 +241,55 @@ impl IdPool { self.map.clear_bit(id); } } + +/// Represents an unused id in an [`IdPool`]. +/// +/// # Invariants +/// +/// The value of `id` is less than `pool.map.len()`. +pub struct UnusedId<'pool> { + id: usize, + pool: &'pool mut IdPool, +} + +impl<'pool> UnusedId<'pool> { + /// Get the unused id as an usize. + /// + /// Be aware that the id has not yet been acquired in the pool. The + /// [`acquire`] method must be called to prevent others from taking the id. + /// + /// [`acquire`]: UnusedId::acquire() + #[inline] + #[must_use] + pub fn as_usize(&self) -> usize { + self.id + } + + /// Get the unused id as an u32. + /// + /// Be aware that the id has not yet been acquired in the pool. The + /// [`acquire`] method must be called to prevent others from taking the id. + /// + /// [`acquire`]: UnusedId::acquire() + #[inline] + #[must_use] + pub fn as_u32(&self) -> u32 { + // CAST: By the type invariants: + // `self.id < pool.map.len() <= BitmapVec::MAX_LEN = i32::MAX`. + self.id as u32 + } + + /// Acquire the unused id. + #[inline] + pub fn acquire(self) -> usize { + self.pool.map.set_bit(self.id); + self.id + } +} + +impl Default for IdPool { + #[inline] + fn default() -> Self { + Self::new() + } +} diff --git a/samples/Kconfig b/samples/Kconfig index 6e072a5f1ed8..5bc7c9e5a59e 100644 --- a/samples/Kconfig +++ b/samples/Kconfig @@ -23,11 +23,11 @@ config SAMPLE_TRACE_CUSTOM_EVENTS This builds the custom trace event example module. config SAMPLE_TRACE_PRINTK - tristate "Build trace_printk module - tests various trace_printk formats" + tristate "Build trace_printk module - tests various trace_printk formats" depends on EVENT_TRACING && m help - This builds a module that calls trace_printk() and can be used to - test various trace_printk() calls from a module. + This builds a module that calls trace_printk() and can be used to + test various trace_printk() calls from a module. config SAMPLE_FTRACE_DIRECT tristate "Build register_ftrace_direct() example" @@ -54,11 +54,11 @@ config SAMPLE_FTRACE_OPS measures the time taken to invoke one function a number of times. config SAMPLE_TRACE_ARRAY - tristate "Build sample module for kernel access to Ftrace instances" + tristate "Build sample module for kernel access to Ftrace instances" depends on EVENT_TRACING && m help - This builds a module that demonstrates the use of various APIs to - access Ftrace instances from within the kernel. + This builds a module that demonstrates the use of various APIs to + access Ftrace instances from within the kernel. config SAMPLE_KOBJECT tristate "Build kobject examples" @@ -290,11 +290,11 @@ config SAMPLE_CORESIGHT_SYSCFG configurations and easily load them into the system at runtime. config SAMPLE_KMEMLEAK - tristate "Simple test for the kernel memory leak detector" - depends on DEBUG_KMEMLEAK && m - help - Build a sample program which have explicitly leaks memory to test - kmemleak + tristate "Simple test for the kernel memory leak detector" + depends on DEBUG_KMEMLEAK && m + help + Build a sample program which have explicitly leaks memory to test + kmemleak. config SAMPLE_CGROUP bool "Build cgroup sample code" diff --git a/samples/vfs/Makefile b/samples/vfs/Makefile index 6554b73a75c8..9256ca5d762b 100644 --- a/samples/vfs/Makefile +++ b/samples/vfs/Makefile @@ -1,4 +1,5 @@ # SPDX-License-Identifier: GPL-2.0-only userprogs-always-y += test-fsmount test-statx mountinfo test-list-all-mounts +userccflags += -I $(srctree)/tools/testing/selftests/ userccflags += -I usr/include diff --git a/scripts/checkpatch.pl b/scripts/checkpatch.pl index 92669904eecc..d58ca9655ab7 100755 --- a/scripts/checkpatch.pl +++ b/scripts/checkpatch.pl @@ -860,6 +860,10 @@ our %deprecated_apis = ( "kunmap" => "kunmap_local", "kmap_atomic" => "kmap_local_page", "kunmap_atomic" => "kunmap_local", + #These should be enough to drive away new IDR users + "DEFINE_IDR" => "DEFINE_XARRAY", + "idr_init" => "xa_init", + "idr_init_base" => "xa_init_flags" ); #Create a search pattern for all these strings to speed up a loop below @@ -3345,6 +3349,13 @@ sub process { } } +# Check for auto-generated unhandled placeholder text (mostly for cover letters) + if (($in_commit_log || $in_header_lines) && + $rawline =~ /(?:SUBJECT|BLURB) HERE/) { + ERROR("PLACEHOLDER_USE", + "Placeholder text detected\n" . $herecurr); + } + # Check for git id commit length and improperly formed commit descriptions # A correctly formed commit description is: # commit <SHA-1 hash length 12+ chars> ("Complete commit subject") diff --git a/scripts/gdb/linux/bpf.py b/scripts/gdb/linux/bpf.py new file mode 100644 index 000000000000..1870534ef6f9 --- /dev/null +++ b/scripts/gdb/linux/bpf.py @@ -0,0 +1,253 @@ +# SPDX-License-Identifier: GPL-2.0 + +import json +import subprocess +import tempfile + +import gdb + +from linux import constants, lists, radixtree, utils + + +if constants.LX_CONFIG_BPF and constants.LX_CONFIG_BPF_JIT: + bpf_ksym_type = utils.CachedType("struct bpf_ksym") +if constants.LX_CONFIG_BPF_SYSCALL: + bpf_prog_type = utils.CachedType("struct bpf_prog") + + +def get_ksym_name(ksym): + name = ksym["name"].bytes + end = name.find(b"\x00") + if end != -1: + name = name[:end] + return name.decode() + + +def list_ksyms(): + if not (constants.LX_CONFIG_BPF and constants.LX_CONFIG_BPF_JIT): + return [] + bpf_kallsyms = gdb.parse_and_eval("&bpf_kallsyms") + bpf_ksym_ptr_type = bpf_ksym_type.get_type().pointer() + return list(lists.list_for_each_entry(bpf_kallsyms, + bpf_ksym_ptr_type, + "lnode")) + + +class KsymAddBreakpoint(gdb.Breakpoint): + def __init__(self, monitor): + super(KsymAddBreakpoint, self).__init__("bpf_ksym_add", internal=True) + self.silent = True + self.monitor = monitor + + def stop(self): + self.monitor.add(gdb.parse_and_eval("ksym")) + return False + + +class KsymRemoveBreakpoint(gdb.Breakpoint): + def __init__(self, monitor): + super(KsymRemoveBreakpoint, self).__init__("bpf_ksym_del", + internal=True) + self.silent = True + self.monitor = monitor + + def stop(self): + self.monitor.remove(gdb.parse_and_eval("ksym")) + return False + + +class KsymMonitor: + def __init__(self, add, remove): + self.add = add + self.remove = remove + + self.add_bp = KsymAddBreakpoint(self) + self.remove_bp = KsymRemoveBreakpoint(self) + + self.notify_initial() + + def notify_initial(self): + for ksym in list_ksyms(): + self.add(ksym) + + def delete(self): + self.add_bp.delete() + self.remove_bp.delete() + + +def list_progs(): + if not constants.LX_CONFIG_BPF_SYSCALL: + return [] + idr_rt = gdb.parse_and_eval("&prog_idr.idr_rt") + bpf_prog_ptr_type = bpf_prog_type.get_type().pointer() + progs = [] + for _, slot in radixtree.for_each_slot(idr_rt): + prog = slot.dereference().cast(bpf_prog_ptr_type) + progs.append(prog) + # Subprogs are not registered in prog_idr, fetch them manually. + # func[0] is the current prog. + aux = prog["aux"] + func = aux["func"] + real_func_cnt = int(aux["real_func_cnt"]) + for i in range(1, real_func_cnt): + progs.append(func[i]) + return progs + + +class ProgAddBreakpoint(gdb.Breakpoint): + def __init__(self, monitor): + super(ProgAddBreakpoint, self).__init__("bpf_prog_kallsyms_add", + internal=True) + self.silent = True + self.monitor = monitor + + def stop(self): + self.monitor.add(gdb.parse_and_eval("fp")) + return False + + +class ProgRemoveBreakpoint(gdb.Breakpoint): + def __init__(self, monitor): + super(ProgRemoveBreakpoint, self).__init__("bpf_prog_free_id", + internal=True) + self.silent = True + self.monitor = monitor + + def stop(self): + self.monitor.remove(gdb.parse_and_eval("prog")) + return False + + +class ProgMonitor: + def __init__(self, add, remove): + self.add = add + self.remove = remove + + self.add_bp = ProgAddBreakpoint(self) + self.remove_bp = ProgRemoveBreakpoint(self) + + self.notify_initial() + + def notify_initial(self): + for prog in list_progs(): + self.add(prog) + + def delete(self): + self.add_bp.delete() + self.remove_bp.delete() + + +def btf_str_by_offset(btf, offset): + while offset < btf["start_str_off"]: + btf = btf["base_btf"] + + offset -= btf["start_str_off"] + if offset < btf["hdr"]["str_len"]: + return (btf["strings"] + offset).string() + + return None + + +def bpf_line_info_line_num(line_col): + return line_col >> 10 + + +def bpf_line_info_line_col(line_col): + return line_col & 0x3ff + + +class LInfoIter: + def __init__(self, prog): + # See bpf_prog_get_file_line() for details. + self.pos = 0 + self.nr_linfo = 0 + + if prog is None: + return + + self.bpf_func = int(prog["bpf_func"]) + aux = prog["aux"] + self.btf = aux["btf"] + linfo_idx = aux["linfo_idx"] + self.nr_linfo = int(aux["nr_linfo"]) - linfo_idx + if self.nr_linfo == 0: + return + + linfo_ptr = aux["linfo"] + tpe = linfo_ptr.type.target().array(self.nr_linfo).pointer() + self.linfo = (linfo_ptr + linfo_idx).cast(tpe).dereference() + jited_linfo_ptr = aux["jited_linfo"] + tpe = jited_linfo_ptr.type.target().array(self.nr_linfo).pointer() + self.jited_linfo = (jited_linfo_ptr + linfo_idx).cast(tpe).dereference() + + self.filenos = {} + + def get_code_off(self): + if self.pos >= self.nr_linfo: + return -1 + return self.jited_linfo[self.pos] - self.bpf_func + + def advance(self): + self.pos += 1 + + def get_fileno(self): + file_name_off = int(self.linfo[self.pos]["file_name_off"]) + fileno = self.filenos.get(file_name_off) + if fileno is not None: + return fileno, None + file_name = btf_str_by_offset(self.btf, file_name_off) + fileno = len(self.filenos) + 1 + self.filenos[file_name_off] = fileno + return fileno, file_name + + def get_line_col(self): + line_col = int(self.linfo[self.pos]["line_col"]) + return bpf_line_info_line_num(line_col), \ + bpf_line_info_line_col(line_col) + + +def generate_debug_obj(ksym, prog): + name = get_ksym_name(ksym) + # Avoid read_memory(); it throws bogus gdb.MemoryError in some contexts. + start = ksym["start"] + code = start.cast(gdb.lookup_type("unsigned char") + .array(int(ksym["end"]) - int(start)) + .pointer()).dereference().bytes + linfo_iter = LInfoIter(prog) + + result = tempfile.NamedTemporaryFile(suffix=".o", mode="wb") + try: + with tempfile.NamedTemporaryFile(suffix=".s", mode="w") as src: + # ".loc" does not apply to ".byte"s, only to ".insn"s, but since + # this needs to work for all architectures, the latter are not an + # option. Ask the assembler to apply ".loc"s to labels as well, + # and generate dummy labels after each ".loc". + src.write(".loc_mark_labels 1\n") + + src.write(".globl {}\n".format(name)) + src.write(".type {},@function\n".format(name)) + src.write("{}:\n".format(name)) + for code_off, code_byte in enumerate(code): + if linfo_iter.get_code_off() == code_off: + fileno, file_name = linfo_iter.get_fileno() + if file_name is not None: + src.write(".file {} {}\n".format( + fileno, json.dumps(file_name))) + line, col = linfo_iter.get_line_col() + src.write(".loc {} {} {}\n".format(fileno, line, col)) + src.write("0:\n") + linfo_iter.advance() + src.write(".byte {}\n".format(code_byte)) + src.write(".size {},{}\n".format(name, len(code))) + src.flush() + + try: + subprocess.check_call(["as", "-c", src.name, "-o", result.name]) + except FileNotFoundError: + # "as" is not installed. + result.close() + return None + return result + except: + result.close() + raise diff --git a/scripts/gdb/linux/constants.py.in b/scripts/gdb/linux/constants.py.in index c3886739a028..6d475540c6ba 100644 --- a/scripts/gdb/linux/constants.py.in +++ b/scripts/gdb/linux/constants.py.in @@ -170,3 +170,6 @@ LX_CONFIG(CONFIG_PAGE_OWNER) LX_CONFIG(CONFIG_SLUB_DEBUG) LX_CONFIG(CONFIG_SLAB_FREELIST_HARDENED) LX_CONFIG(CONFIG_MMU) +LX_CONFIG(CONFIG_BPF) +LX_CONFIG(CONFIG_BPF_JIT) +LX_CONFIG(CONFIG_BPF_SYSCALL) diff --git a/scripts/gdb/linux/radixtree.py b/scripts/gdb/linux/radixtree.py index 074543ac763d..bc2954e45c32 100644 --- a/scripts/gdb/linux/radixtree.py +++ b/scripts/gdb/linux/radixtree.py @@ -30,13 +30,16 @@ def entry_to_node(node): def node_maxindex(node): return (constants.LX_RADIX_TREE_MAP_SIZE << node['shift']) - 1 -def lookup(root, index): +def resolve_root(root): + if root.type == radix_tree_root_type.get_type(): + return root if root.type == radix_tree_root_type.get_type().pointer(): - node = root.dereference() - elif root.type != radix_tree_root_type.get_type(): - raise gdb.GdbError("must be {} not {}" - .format(radix_tree_root_type.get_type(), root.type)) + return root.dereference() + raise gdb.GdbError("must be {} not {}" + .format(radix_tree_root_type.get_type(), root.type)) +def lookup(root, index): + root = resolve_root(root) node = root['xa_head'] if node == 0: return None @@ -71,14 +74,120 @@ def lookup(root, index): return node -class LxRadixTree(gdb.Function): +def descend(parent, index): + offset = (index >> int(parent["shift"])) & constants.LX_RADIX_TREE_MAP_MASK + return offset, parent["slots"][offset] + +def load_root(root): + node = root["xa_head"] + nodep = node + + if is_internal_node(node): + node = entry_to_node(node) + maxindex = node_maxindex(node) + return int(node["shift"]) + constants.LX_RADIX_TREE_MAP_SHIFT, \ + nodep, maxindex + + return 0, nodep, 0 + +class RadixTreeIter: + def __init__(self, start): + self.index = 0 + self.next_index = start + self.node = None + +def xa_mk_internal(v): + return (v << 2) | 2 + +LX_XA_RETRY_ENTRY = xa_mk_internal(256) +LX_RADIX_TREE_RETRY = LX_XA_RETRY_ENTRY + +def next_chunk(root, iter): + mask = (1 << (utils.get_ulong_type().sizeof * 8)) - 1 + + index = iter.next_index + if index == 0 and iter.index != 0: + return None + + restart = True + while restart: + restart = False + + _, child, maxindex = load_root(root) + if index > maxindex: + return None + if not child: + return None + + if not is_internal_node(child): + iter.index = index + iter.next_index = (maxindex + 1) & mask + iter.node = None + return root["xa_head"].address + + while True: + node = entry_to_node(child) + offset, child = descend(node, index) + + if not child: + while True: + offset += 1 + if offset >= constants.LX_RADIX_TREE_MAP_SIZE: + break + slot = node["slots"][offset] + if slot: + break + index &= ~node_maxindex(node) + index = (index + (offset << int(node["shift"]))) & mask + if index == 0: + return None + if offset == constants.LX_RADIX_TREE_MAP_SIZE: + restart = True + break + child = node["slots"][offset] + + if not child: + restart = True + break + if child == LX_XA_RETRY_ENTRY: + break + if not node["shift"] or not is_internal_node(child): + break + + iter.index = (index & ~node_maxindex(node)) | offset + iter.next_index = ((index | node_maxindex(node)) + 1) & mask + iter.node = node + + return node["slots"][offset].address + +def next_slot(slot, iter): + mask = (1 << (utils.get_ulong_type().sizeof * 8)) - 1 + for _ in range(iter.next_index - iter.index - 1): + slot += 1 + iter.index = (iter.index + 1) & mask + if slot.dereference(): + return slot + return None + +def for_each_slot(root, start=0): + iter = RadixTreeIter(start) + slot = None + while True: + if not slot: + slot = next_chunk(root, iter) + if not slot: + break + yield iter.index, slot + slot = next_slot(slot, iter) + +class LxRadixTreeLookup(gdb.Function): """ Lookup and return a node from a RadixTree. $lx_radix_tree_lookup(root_node [, index]): Return the node at the given index. If index is omitted, the root node is dereference and returned.""" def __init__(self): - super(LxRadixTree, self).__init__("lx_radix_tree_lookup") + super(LxRadixTreeLookup, self).__init__("lx_radix_tree_lookup") def invoke(self, root, index=0): result = lookup(root, index) @@ -87,4 +196,20 @@ If index is omitted, the root node is dereference and returned.""" return result +class LxRadixTree(gdb.Command): + """Show all values stored in a RadixTree.""" + + def __init__(self): + super(LxRadixTree, self).__init__("lx-radix-tree", gdb.COMMAND_DATA, + gdb.COMPLETE_NONE) + + def invoke(self, argument, from_tty): + args = gdb.string_to_argv(argument) + if len(args) != 1: + raise gdb.GdbError("Usage: lx-radix-tree ROOT") + root = gdb.parse_and_eval(args[0]) + for index, slot in for_each_slot(root): + gdb.write("[{}] = {}\n".format(index, slot.dereference())) + LxRadixTree() +LxRadixTreeLookup() diff --git a/scripts/gdb/linux/symbols.py b/scripts/gdb/linux/symbols.py index 6edb99221675..d4308b726183 100644 --- a/scripts/gdb/linux/symbols.py +++ b/scripts/gdb/linux/symbols.py @@ -11,13 +11,14 @@ # This work is licensed under the terms of the GNU GPL version 2. # +import atexit import gdb import os import re import struct from itertools import count -from linux import modules, utils, constants +from linux import bpf, constants, modules, utils if hasattr(gdb, 'Breakpoint'): @@ -114,17 +115,27 @@ class LxSymbols(gdb.Command): The kernel (vmlinux) is taken from the current working directly. Modules (.ko) are scanned recursively, starting in the same directory. Optionally, the module search path can be extended by a space separated list of paths passed to the -lx-symbols command.""" +lx-symbols command. + +When the -bpf flag is specified, symbols from the currently loaded BPF programs +are loaded as well.""" module_paths = [] module_files = [] module_files_updated = False loaded_modules = [] breakpoint = None + bpf_prog_monitor = None + bpf_ksym_monitor = None + bpf_progs = {} + # The remove-symbol-file command, even when invoked with -a, requires the + # respective object file to exist, so keep them around. + bpf_debug_objs = {} def __init__(self): super(LxSymbols, self).__init__("lx-symbols", gdb.COMMAND_FILES, gdb.COMPLETE_FILENAME) + atexit.register(self.cleanup_bpf) def _update_module_files(self): self.module_files = [] @@ -197,6 +208,51 @@ lx-symbols command.""" else: gdb.write("no module object found for '{0}'\n".format(module_name)) + def add_bpf_prog(self, prog): + if prog["jited"]: + self.bpf_progs[int(prog["bpf_func"])] = prog + + def remove_bpf_prog(self, prog): + self.bpf_progs.pop(int(prog["bpf_func"]), None) + + def add_bpf_ksym(self, ksym): + addr = int(ksym["start"]) + name = bpf.get_ksym_name(ksym) + with utils.pagination_off(): + gdb.write("loading @{addr}: {name}\n".format( + addr=hex(addr), name=name)) + debug_obj = bpf.generate_debug_obj(ksym, self.bpf_progs.get(addr)) + if debug_obj is None: + return + try: + cmdline = "add-symbol-file {obj} {addr}".format( + obj=debug_obj.name, addr=hex(addr)) + gdb.execute(cmdline, to_string=True) + except: + debug_obj.close() + raise + self.bpf_debug_objs[addr] = debug_obj + + def remove_bpf_ksym(self, ksym): + addr = int(ksym["start"]) + debug_obj = self.bpf_debug_objs.pop(addr, None) + if debug_obj is None: + return + try: + name = bpf.get_ksym_name(ksym) + gdb.write("unloading @{addr}: {name}\n".format( + addr=hex(addr), name=name)) + cmdline = "remove-symbol-file {path}".format(path=debug_obj.name) + gdb.execute(cmdline, to_string=True) + finally: + debug_obj.close() + + def cleanup_bpf(self): + self.bpf_progs = {} + while len(self.bpf_debug_objs) > 0: + self.bpf_debug_objs.popitem()[1].close() + + def load_all_symbols(self): gdb.write("loading vmlinux\n") @@ -224,34 +280,59 @@ lx-symbols command.""" else: [self.load_module_symbols(module) for module in module_list] + self.cleanup_bpf() + if self.bpf_prog_monitor is not None: + self.bpf_prog_monitor.notify_initial() + if self.bpf_ksym_monitor is not None: + self.bpf_ksym_monitor.notify_initial() + for saved_state in saved_states: saved_state['breakpoint'].enabled = saved_state['enabled'] def invoke(self, arg, from_tty): skip_decompressor() - self.module_paths = [os.path.abspath(os.path.expanduser(p)) - for p in arg.split()] + monitor_bpf = False + self.module_paths = [] + for p in arg.split(): + if p == "-bpf": + monitor_bpf = True + else: + p.append(os.path.abspath(os.path.expanduser(p))) self.module_paths.append(os.getcwd()) + if self.breakpoint is not None: + self.breakpoint.delete() + self.breakpoint = None + if self.bpf_prog_monitor is not None: + self.bpf_prog_monitor.delete() + self.bpf_prog_monitor = None + if self.bpf_ksym_monitor is not None: + self.bpf_ksym_monitor.delete() + self.bpf_ksym_monitor = None + # enforce update self.module_files = [] self.module_files_updated = False self.load_all_symbols() - if not modules.has_modules(): + if not hasattr(gdb, 'Breakpoint'): + gdb.write("Note: symbol update on module and BPF loading not " + "supported with this gdb version\n") return - if hasattr(gdb, 'Breakpoint'): - if self.breakpoint is not None: - self.breakpoint.delete() - self.breakpoint = None + if modules.has_modules(): self.breakpoint = LoadModuleBreakpoint( "kernel/module/main.c:do_init_module", self) - else: - gdb.write("Note: symbol update on module loading not supported " - "with this gdb version\n") + + if monitor_bpf: + if constants.LX_CONFIG_BPF_SYSCALL: + self.bpf_prog_monitor = bpf.ProgMonitor(self.add_bpf_prog, + self.remove_bpf_prog) + if constants.LX_CONFIG_BPF and constants.LX_CONFIG_BPF_JIT: + self.bpf_ksym_monitor = bpf.KsymMonitor(self.add_bpf_ksym, + self.remove_bpf_ksym) LxSymbols() diff --git a/scripts/mod/devicetable-offsets.c b/scripts/mod/devicetable-offsets.c index d3d00e85edf7..ef2ffb68f69d 100644 --- a/scripts/mod/devicetable-offsets.c +++ b/scripts/mod/devicetable-offsets.c @@ -1,4 +1,5 @@ // SPDX-License-Identifier: GPL-2.0 +#define COMPILE_OFFSETS #include <linux/kbuild.h> #include <linux/mod_devicetable.h> diff --git a/security/landlock/errata/abi-1.h b/security/landlock/errata/abi-1.h new file mode 100644 index 000000000000..e8a2bff2e5b6 --- /dev/null +++ b/security/landlock/errata/abi-1.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ + +/** + * DOC: erratum_3 + * + * Erratum 3: Disconnected directory handling + * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + * + * This fix addresses an issue with disconnected directories that occur when a + * directory is moved outside the scope of a bind mount. The change ensures + * that evaluated access rights include both those from the disconnected file + * hierarchy down to its filesystem root and those from the related mount point + * hierarchy. This prevents access right widening through rename or link + * actions. + */ +LANDLOCK_ERRATUM(3) diff --git a/security/landlock/fs.c b/security/landlock/fs.c index cee2b6f22c83..fe794875ad46 100644 --- a/security/landlock/fs.c +++ b/security/landlock/fs.c @@ -714,7 +714,8 @@ static void test_is_eacces_with_write(struct kunit *const test) * is_access_to_paths_allowed - Check accesses for requests with a common path * * @domain: Domain to check against. - * @path: File hierarchy to walk through. + * @path: File hierarchy to walk through. For refer checks, this would be + * the common mountpoint. * @access_request_parent1: Accesses to check, once @layer_masks_parent1 is * equal to @layer_masks_parent2 (if any). This is tied to the unique * requested path for most actions, or the source in case of a refer action @@ -837,7 +838,6 @@ static bool is_access_to_paths_allowed( * restriction. */ while (true) { - struct dentry *parent_dentry; const struct landlock_rule *rule; /* @@ -909,21 +909,33 @@ jump_up: break; } } + if (unlikely(IS_ROOT(walker_path.dentry))) { - /* - * Stops at disconnected root directories. Only allows - * access to internal filesystems (e.g. nsfs, which is - * reachable through /proc/<pid>/ns/<namespace>). - */ - if (walker_path.mnt->mnt_flags & MNT_INTERNAL) { + if (likely(walker_path.mnt->mnt_flags & MNT_INTERNAL)) { + /* + * Stops and allows access when reaching disconnected root + * directories that are part of internal filesystems (e.g. nsfs, + * which is reachable through /proc/<pid>/ns/<namespace>). + */ allowed_parent1 = true; allowed_parent2 = true; + break; } - break; + + /* + * We reached a disconnected root directory from a bind mount. + * Let's continue the walk with the mount point we missed. + */ + dput(walker_path.dentry); + walker_path.dentry = walker_path.mnt->mnt_root; + dget(walker_path.dentry); + } else { + struct dentry *const parent_dentry = + dget_parent(walker_path.dentry); + + dput(walker_path.dentry); + walker_path.dentry = parent_dentry; } - parent_dentry = dget_parent(walker_path.dentry); - dput(walker_path.dentry); - walker_path.dentry = parent_dentry; } path_put(&walker_path); @@ -1021,6 +1033,9 @@ static access_mask_t maybe_remove(const struct dentry *const dentry) * file. While walking from @dir to @mnt_root, we record all the domain's * allowed accesses in @layer_masks_dom. * + * Because of disconnected directories, this walk may not reach @mnt_dir. In + * this case, the walk will continue to @mnt_dir after this call. + * * This is similar to is_access_to_paths_allowed() but much simpler because it * only handles walking on the same mount point and only checks one set of * accesses. @@ -1062,8 +1077,11 @@ static bool collect_domain_accesses( break; } - /* We should not reach a root other than @mnt_root. */ - if (dir == mnt_root || WARN_ON_ONCE(IS_ROOT(dir))) + /* + * Stops at the mount point or the filesystem root for a disconnected + * directory. + */ + if (dir == mnt_root || unlikely(IS_ROOT(dir))) break; parent_dentry = dget_parent(dir); diff --git a/security/landlock/ruleset.c b/security/landlock/ruleset.c index ce7940efea51..dfcdc19ea268 100644 --- a/security/landlock/ruleset.c +++ b/security/landlock/ruleset.c @@ -83,6 +83,10 @@ static void build_check_rule(void) .num_layers = ~0, }; + /* + * Checks that .num_layers is large enough for at least + * LANDLOCK_MAX_NUM_LAYERS layers. + */ BUILD_BUG_ON(rule.num_layers < LANDLOCK_MAX_NUM_LAYERS); } @@ -290,6 +294,10 @@ static void build_check_layer(void) .access = ~0, }; + /* + * Checks that .level and .access are large enough to contain their expected + * maximum values. + */ BUILD_BUG_ON(layer.level < LANDLOCK_MAX_NUM_LAYERS); BUILD_BUG_ON(layer.access < LANDLOCK_MASK_ACCESS_FS); } @@ -644,8 +652,8 @@ bool landlock_unmask_layers(const struct landlock_rule *const rule, bool is_empty; /* - * Records in @layer_masks which layer grants access to each - * requested access. + * Records in @layer_masks which layer grants access to each requested + * access: bit cleared if the related layer grants access. */ is_empty = true; for_each_set_bit(access_bit, &access_req, masks_array_size) { diff --git a/security/landlock/ruleset.h b/security/landlock/ruleset.h index 5da9a64f5af7..1a78cba662b2 100644 --- a/security/landlock/ruleset.h +++ b/security/landlock/ruleset.h @@ -27,7 +27,7 @@ struct landlock_hierarchy; */ struct landlock_layer { /** - * @level: Position of this layer in the layer stack. + * @level: Position of this layer in the layer stack. Starts from 1. */ u16 level; /** diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c index fe6c2cebc7f0..f7189990dd6a 100644 --- a/sound/usb/mixer_quirks.c +++ b/sound/usb/mixer_quirks.c @@ -3416,10 +3416,6 @@ static int snd_bbfpro_controls_create(struct usb_mixer_interface *mixer) #define RME_DIGIFACE_REGISTER(reg, mask) (((reg) << 16) | (mask)) #define RME_DIGIFACE_INVERT BIT(31) -/* Nonconst helpers */ -#define field_get(_mask, _reg) (((_reg) & (_mask)) >> (ffs(_mask) - 1)) -#define field_prep(_mask, _val) (((_val) << (ffs(_mask) - 1)) & (_mask)) - static int snd_rme_digiface_write_reg(struct snd_kcontrol *kcontrol, int item, u16 mask, u16 val) { struct usb_mixer_elem_list *list = snd_kcontrol_chip(kcontrol); diff --git a/tools/Makefile b/tools/Makefile index c31cbbd12c45..cb40961a740f 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -14,6 +14,7 @@ help: @echo ' counter - counter tools' @echo ' cpupower - a tool for all things x86 CPU power' @echo ' debugging - tools for debugging' + @echo ' dma - tools for DMA mapping' @echo ' firewire - the userspace part of nosy, an IEEE-1394 traffic sniffer' @echo ' firmware - Firmware tools' @echo ' freefall - laptop accelerometer program for disk protection' @@ -69,7 +70,7 @@ acpi: FORCE cpupower: FORCE $(call descend,power/$@) -counter firewire hv guest bootconfig spi usb virtio mm bpf iio gpio objtool leds wmi firmware debugging tracing: FORCE +counter dma firewire hv guest bootconfig spi usb virtio mm bpf iio gpio objtool leds wmi firmware debugging tracing: FORCE $(call descend,$@) bpf/%: FORCE @@ -122,7 +123,7 @@ kvm_stat: FORCE ynl: FORCE $(call descend,net/ynl) -all: acpi counter cpupower gpio hv firewire \ +all: acpi counter cpupower dma gpio hv firewire \ perf selftests bootconfig spi turbostat usb \ virtio mm bpf x86_energy_perf_policy \ tmon freefall iio objtool kvm_stat wmi \ @@ -134,7 +135,7 @@ acpi_install: cpupower_install: $(call descend,power/$(@:_install=),install) -counter_install firewire_install gpio_install hv_install iio_install perf_install bootconfig_install spi_install usb_install virtio_install mm_install bpf_install objtool_install wmi_install debugging_install tracing_install: +counter_install dma_install firewire_install gpio_install hv_install iio_install perf_install bootconfig_install spi_install usb_install virtio_install mm_install bpf_install objtool_install wmi_install debugging_install tracing_install: $(call descend,$(@:_install=),install) selftests_install: @@ -164,7 +165,7 @@ kvm_stat_install: ynl_install: $(call descend,net/$(@:_install=),install) -install: acpi_install counter_install cpupower_install gpio_install \ +install: acpi_install counter_install cpupower_install dma_install gpio_install \ hv_install firewire_install iio_install \ perf_install selftests_install turbostat_install usb_install \ virtio_install mm_install bpf_install x86_energy_perf_policy_install \ @@ -178,7 +179,7 @@ acpi_clean: cpupower_clean: $(call descend,power/cpupower,clean) -counter_clean hv_clean firewire_clean bootconfig_clean spi_clean usb_clean virtio_clean mm_clean wmi_clean bpf_clean iio_clean gpio_clean objtool_clean leds_clean firmware_clean debugging_clean tracing_clean: +counter_clean dma_clean hv_clean firewire_clean bootconfig_clean spi_clean usb_clean virtio_clean mm_clean wmi_clean bpf_clean iio_clean gpio_clean objtool_clean leds_clean firmware_clean debugging_clean tracing_clean: $(call descend,$(@:_clean=),clean) libapi_clean: @@ -224,7 +225,7 @@ build_clean: ynl_clean: $(call descend,net/$(@:_clean=),clean) -clean: acpi_clean counter_clean cpupower_clean hv_clean firewire_clean \ +clean: acpi_clean counter_clean cpupower_clean dma_clean hv_clean firewire_clean \ perf_clean selftests_clean turbostat_clean bootconfig_clean spi_clean usb_clean virtio_clean \ mm_clean bpf_clean iio_clean x86_energy_perf_policy_clean tmon_clean \ freefall_clean build_clean libbpf_clean libsubcmd_clean \ diff --git a/tools/dma/.gitignore b/tools/dma/.gitignore new file mode 100644 index 000000000000..94b68cf4147b --- /dev/null +++ b/tools/dma/.gitignore @@ -0,0 +1,3 @@ +# SPDX-License-Identifier: GPL-2.0-only +dma_map_benchmark +include/linux/map_benchmark.h diff --git a/tools/dma/Makefile b/tools/dma/Makefile new file mode 100644 index 000000000000..e4abf37bf020 --- /dev/null +++ b/tools/dma/Makefile @@ -0,0 +1,55 @@ +# SPDX-License-Identifier: GPL-2.0 +include ../scripts/Makefile.include + +bindir ?= /usr/bin + +# This will work when dma is built in tools env. where srctree +# isn't set and when invoked from selftests build, where srctree +# is set to ".". building_out_of_srctree is undefined for in srctree +# builds +ifndef building_out_of_srctree +srctree := $(patsubst %/,%,$(dir $(CURDIR))) +srctree := $(patsubst %/,%,$(dir $(srctree))) +endif + +# Do not use make's built-in rules +# (this improves performance and avoids hard-to-debug behaviour); +MAKEFLAGS += -r + +override CFLAGS += -O2 -Wall -g -D_GNU_SOURCE -I$(OUTPUT)include + +ALL_TARGETS := dma_map_benchmark +ALL_PROGRAMS := $(patsubst %,$(OUTPUT)%,$(ALL_TARGETS)) + +all: $(ALL_PROGRAMS) + +export srctree OUTPUT CC LD CFLAGS +include $(srctree)/tools/build/Makefile.include + +# +# We need the following to be outside of kernel tree +# +$(OUTPUT)include/linux/map_benchmark.h: ../../include/uapi/linux/map_benchmark.h + mkdir -p $(OUTPUT)include/linux 2>&1 || true + ln -sf $(CURDIR)/../../include/uapi/linux/map_benchmark.h $@ + +prepare: $(OUTPUT)include/linux/map_benchmark.h + +FORCE: + +DMA_MAP_BENCHMARK = dma_map_benchmark +$(DMA_MAP_BENCHMARK): prepare FORCE + $(CC) $(CFLAGS) $(DMA_MAP_BENCHMARK).c -o $(DMA_MAP_BENCHMARK) + +clean: + rm -f $(ALL_PROGRAMS) + rm -rf $(OUTPUT)include + find $(or $(OUTPUT),.) -name '*.o' -delete -o -name '\.*.d' -delete -o -name '\.*.cmd' -delete + +install: $(ALL_PROGRAMS) + install -d -m 755 $(DESTDIR)$(bindir); \ + for program in $(ALL_PROGRAMS); do \ + install $$program $(DESTDIR)$(bindir); \ + done + +.PHONY: all install clean prepare FORCE diff --git a/tools/testing/selftests/dma/config b/tools/dma/config index 6102ee3c43cd..6102ee3c43cd 100644 --- a/tools/testing/selftests/dma/config +++ b/tools/dma/config diff --git a/tools/testing/selftests/dma/dma_map_benchmark.c b/tools/dma/dma_map_benchmark.c index b925756373ce..dd0ed528e6df 100644 --- a/tools/testing/selftests/dma/dma_map_benchmark.c +++ b/tools/dma/dma_map_benchmark.c @@ -10,7 +10,6 @@ #include <unistd.h> #include <sys/ioctl.h> #include <sys/mman.h> -#include <linux/types.h> #include <linux/map_benchmark.h> #define NSEC_PER_MSEC 1000000L diff --git a/tools/net/sunrpc/xdrgen/generators/__init__.py b/tools/net/sunrpc/xdrgen/generators/__init__.py index b98574a36a4a..e22632cf38fb 100644 --- a/tools/net/sunrpc/xdrgen/generators/__init__.py +++ b/tools/net/sunrpc/xdrgen/generators/__init__.py @@ -2,7 +2,7 @@ """Define a base code generator class""" -import sys +from pathlib import Path from jinja2 import Environment, FileSystemLoader, Template from xdr_ast import _XdrAst, Specification, _RpcProgram, _XdrTypeSpecifier @@ -14,8 +14,11 @@ def create_jinja2_environment(language: str, xdr_type: str) -> Environment: """Open a set of templates based on output language""" match language: case "C": + templates_dir = ( + Path(__file__).parent.parent / "templates" / language / xdr_type + ) environment = Environment( - loader=FileSystemLoader(sys.path[0] + "/templates/C/" + xdr_type + "/"), + loader=FileSystemLoader(templates_dir), trim_blocks=True, lstrip_blocks=True, ) @@ -48,9 +51,7 @@ def find_xdr_program_name(root: Specification) -> str: def header_guard_infix(filename: str) -> str: """Extract the header guard infix from the specification filename""" - basename = filename.split("/")[-1] - program = basename.replace(".x", "") - return program.upper() + return Path(filename).stem.upper() def kernel_c_type(spec: _XdrTypeSpecifier) -> str: diff --git a/tools/net/sunrpc/xdrgen/generators/union.py b/tools/net/sunrpc/xdrgen/generators/union.py index 2cca00e279cd..ad1f214ef22a 100644 --- a/tools/net/sunrpc/xdrgen/generators/union.py +++ b/tools/net/sunrpc/xdrgen/generators/union.py @@ -8,7 +8,7 @@ from jinja2 import Environment from generators import SourceGenerator from generators import create_jinja2_environment, get_jinja2_template -from xdr_ast import _XdrBasic, _XdrUnion, _XdrVoid, get_header_name +from xdr_ast import _XdrBasic, _XdrUnion, _XdrVoid, _XdrString, get_header_name from xdr_ast import _XdrDeclaration, _XdrCaseSpec, public_apis, big_endian @@ -40,13 +40,20 @@ def emit_union_case_spec_definition( """Emit a definition for an XDR union's case arm""" if isinstance(node.arm, _XdrVoid): return - assert isinstance(node.arm, _XdrBasic) + if isinstance(node.arm, _XdrString): + type_name = "char *" + classifier = "" + else: + type_name = node.arm.spec.type_name + classifier = node.arm.spec.c_classifier + + assert isinstance(node.arm, (_XdrBasic, _XdrString)) template = get_jinja2_template(environment, "definition", "case_spec") print( template.render( name=node.arm.name, - type=node.arm.spec.type_name, - classifier=node.arm.spec.c_classifier, + type=type_name, + classifier=classifier, ) ) @@ -84,6 +91,12 @@ def emit_union_case_spec_decoder( if isinstance(node.arm, _XdrVoid): return + if isinstance(node.arm, _XdrString): + type_name = "char *" + classifier = "" + else: + type_name = node.arm.spec.type_name + classifier = node.arm.spec.c_classifier if big_endian_discriminant: template = get_jinja2_template(environment, "decoder", "case_spec_be") @@ -92,13 +105,13 @@ def emit_union_case_spec_decoder( for case in node.values: print(template.render(case=case)) - assert isinstance(node.arm, _XdrBasic) + assert isinstance(node.arm, (_XdrBasic, _XdrString)) template = get_jinja2_template(environment, "decoder", node.arm.template) print( template.render( name=node.arm.name, - type=node.arm.spec.type_name, - classifier=node.arm.spec.c_classifier, + type=type_name, + classifier=classifier, ) ) @@ -169,7 +182,10 @@ def emit_union_case_spec_encoder( if isinstance(node.arm, _XdrVoid): return - + if isinstance(node.arm, _XdrString): + type_name = "char *" + else: + type_name = node.arm.spec.type_name if big_endian_discriminant: template = get_jinja2_template(environment, "encoder", "case_spec_be") else: @@ -181,7 +197,7 @@ def emit_union_case_spec_encoder( print( template.render( name=node.arm.name, - type=node.arm.spec.type_name, + type=type_name, ) ) diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/close.j2 index 5bf010665f84..3dbd724d7f17 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/decoder/close.j2 @@ -1,3 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/close.j2 index 5bf010665f84..3dbd724d7f17 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/pointer/encoder/close.j2 @@ -1,3 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/close.j2 index 5bf010665f84..3dbd724d7f17 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/close.j2 @@ -1,3 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_opaque.j2 index 9a814de54ae8..65698e20d8cd 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/decoder/variable_length_opaque.j2 @@ -2,5 +2,5 @@ {% if annotate %} /* member {{ name }} (variable-length opaque) */ {% endif %} - if (!xdrgen_decode_opaque(xdr, (opaque *)ptr, {{ maxsize }})) + if (!xdrgen_decode_opaque(xdr, &ptr->{{ name }}, {{ maxsize }})) return false; diff --git a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/close.j2 index 5bf010665f84..3dbd724d7f17 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/struct/encoder/close.j2 @@ -1,3 +1,3 @@ {# SPDX-License-Identifier: GPL-2.0 #} return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/basic.j2 index da4709403dc9..b215e157dfa7 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/basic.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/basic.j2 @@ -14,4 +14,4 @@ xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ name }} *ptr) /* (basic) */ {% endif %} return xdrgen_decode_{{ type }}(xdr, ptr); -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_array.j2 index d7c80e472fe3..c8953719e626 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_array.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_array.j2 @@ -22,4 +22,4 @@ xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr return false; } return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_opaque.j2 index bdc7bd24ffb1..c854fc8c74e3 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/fixed_length_opaque.j2 @@ -14,4 +14,4 @@ xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr /* (fixed-length opaque) */ {% endif %} return xdr_stream_decode_opaque_fixed(xdr, ptr, {{ size }}) == 0; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/string.j2 index 56c5a17d6a70..bcbc1758aae9 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/string.j2 @@ -14,4 +14,4 @@ xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr /* (variable-length string) */ {% endif %} return xdrgen_decode_string(xdr, ptr, {{ maxsize }}); -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_array.j2 index e74ffdd98463..a59cc1f38eed 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_array.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_array.j2 @@ -23,4 +23,4 @@ xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr if (!xdrgen_decode_{{ type }}(xdr, &ptr->element[i])) return false; return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 index f28f8b228ad5..eb05f53e1041 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/decoder/variable_length_opaque.j2 @@ -14,4 +14,4 @@ xdrgen_decode_{{ name }}(struct xdr_stream *xdr, {{ classifier }}{{ name }} *ptr /* (variable-length opaque) */ {% endif %} return xdrgen_decode_opaque(xdr, ptr, {{ maxsize }}); -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/basic.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/basic.j2 index 35effe67e4ef..0d21dd0b723a 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/basic.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/basic.j2 @@ -18,4 +18,4 @@ xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name } /* (basic) */ {% endif %} return xdrgen_encode_{{ type }}(xdr, value); -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_array.j2 index 95202ad5ad2d..ec8cd6509514 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_array.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_array.j2 @@ -22,4 +22,4 @@ xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name } return false; } return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_opaque.j2 index 9c66a11b9912..b53fa87e1858 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/fixed_length_opaque.j2 @@ -14,4 +14,4 @@ xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name } /* (fixed-length opaque) */ {% endif %} return xdr_stream_encode_opaque_fixed(xdr, value, {{ size }}) >= 0; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/string.j2 index 3d490ff180d0..28b81f1d0bd6 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/string.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/string.j2 @@ -14,4 +14,4 @@ xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name } /* (variable-length string) */ {% endif %} return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_array.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_array.j2 index 2d2384f64918..ff093c281d51 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_array.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_array.j2 @@ -27,4 +27,4 @@ xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name } {% endif %} return false; return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_opaque.j2 b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_opaque.j2 index 8508f13c95b9..2e89592fa702 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_opaque.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/typedef/encoder/variable_length_opaque.j2 @@ -14,4 +14,4 @@ xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const {{ classifier }}{{ name } /* (variable-length opaque) */ {% endif %} return xdr_stream_encode_opaque(xdr, value.data, value.len) >= 0; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/declaration/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/declaration/close.j2 new file mode 100644 index 000000000000..816291184e8c --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/declaration/close.j2 @@ -0,0 +1,4 @@ +{# SPDX-License-Identifier: GPL-2.0 #} + +bool xdrgen_decode_{{ name }}(struct xdr_stream *xdr, struct {{ name }} *ptr); +bool xdrgen_encode_{{ name }}(struct xdr_stream *xdr, const struct {{ name }} *value); diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/close.j2 index fdc2dfd1843b..39d8d6c5094d 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/union/decoder/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/union/decoder/close.j2 @@ -1,4 +1,4 @@ {# SPDX-License-Identifier: GPL-2.0 #} } return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/close.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/close.j2 index fdc2dfd1843b..39d8d6c5094d 100644 --- a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/close.j2 +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/close.j2 @@ -1,4 +1,4 @@ {# SPDX-License-Identifier: GPL-2.0 #} } return true; -}; +} diff --git a/tools/net/sunrpc/xdrgen/templates/C/union/encoder/string.j2 b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/string.j2 new file mode 100644 index 000000000000..2f035a64f1f4 --- /dev/null +++ b/tools/net/sunrpc/xdrgen/templates/C/union/encoder/string.j2 @@ -0,0 +1,6 @@ +{# SPDX-License-Identifier: GPL-2.0 #} +{% if annotate %} + /* member {{ name }} (variable-length string) */ +{% endif %} + if (!xdrgen_encode_string(xdr, ptr->u.{{ name }}, {{ maxsize }})) + return false; diff --git a/tools/net/sunrpc/xdrgen/xdrgen b/tools/net/sunrpc/xdrgen/xdrgen index 43762be39252..3afd0547d67c 100755 --- a/tools/net/sunrpc/xdrgen/xdrgen +++ b/tools/net/sunrpc/xdrgen/xdrgen @@ -10,8 +10,13 @@ __license__ = "GPL-2.0 only" __version__ = "0.2" import sys +from pathlib import Path import argparse +_XDRGEN_DIR = Path(__file__).resolve().parent +if str(_XDRGEN_DIR) not in sys.path: + sys.path.insert(0, str(_XDRGEN_DIR)) + from subcmds import definitions from subcmds import declarations from subcmds import lint diff --git a/tools/objtool/Build b/tools/objtool/Build index 9982e665d58d..600da051af12 100644 --- a/tools/objtool/Build +++ b/tools/objtool/Build @@ -18,6 +18,7 @@ objtool-y += libstring.o objtool-y += libctype.o objtool-y += str_error_r.o objtool-y += librbtree.o +objtool-y += signal.o $(OUTPUT)libstring.o: ../lib/string.c FORCE $(call rule_mkdir) diff --git a/tools/objtool/check.c b/tools/objtool/check.c index 9ec0e07cce90..3f7999317f4d 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -3282,18 +3282,19 @@ static int propagate_alt_cfi(struct objtool_file *file, struct instruction *insn return 0; } -static int handle_insn_ops(struct instruction *insn, - struct instruction *next_insn, - struct insn_state *state) +static int noinline handle_insn_ops(struct instruction *insn, + struct instruction *next_insn, + struct insn_state *state) { + struct insn_state prev_state __maybe_unused = *state; struct stack_op *op; - int ret; + int ret = 0; for (op = insn->stack_ops; op; op = op->next) { ret = update_cfi_state(insn, next_insn, &state->cfi, op); if (ret) - return ret; + goto done; if (!opts.uaccess || !insn->alt_group) continue; @@ -3303,7 +3304,8 @@ static int handle_insn_ops(struct instruction *insn, state->uaccess_stack = 1; } else if (state->uaccess_stack >> 31) { WARN_INSN(insn, "PUSHF stack exhausted"); - return 1; + ret = 1; + goto done; } state->uaccess_stack <<= 1; state->uaccess_stack |= state->uaccess; @@ -3319,7 +3321,10 @@ static int handle_insn_ops(struct instruction *insn, } } - return 0; +done: + TRACE_INSN_STATE(insn, &prev_state, state); + + return ret; } static bool insn_cfi_match(struct instruction *insn, struct cfi_state *cfi2) @@ -3694,8 +3699,6 @@ static int validate_insn(struct objtool_file *file, struct symbol *func, struct instruction *prev_insn, struct instruction *next_insn, bool *dead_end) { - /* prev_state and alt_name are not used if there is no disassembly support */ - struct insn_state prev_state __maybe_unused; char *alt_name __maybe_unused = NULL; struct alternative *alt; u8 visited; @@ -3798,11 +3801,7 @@ static int validate_insn(struct objtool_file *file, struct symbol *func, if (skip_alt_group(insn)) return 0; - prev_state = *statep; - ret = handle_insn_ops(insn, next_insn, statep); - TRACE_INSN_STATE(insn, &prev_state, statep); - - if (ret) + if (handle_insn_ops(insn, next_insn, statep)) return 1; switch (insn->type) { diff --git a/tools/objtool/include/objtool/objtool.h b/tools/objtool/include/objtool/objtool.h index f7051bbe0bcb..6dc12a59ad00 100644 --- a/tools/objtool/include/objtool/objtool.h +++ b/tools/objtool/include/objtool/objtool.h @@ -41,6 +41,8 @@ struct objtool_file { char *top_level_dir(const char *file); +int init_signal_handler(void); + struct objtool_file *objtool_open_read(const char *_objname); int objtool_pv_add(struct objtool_file *file, int idx, struct symbol *func); diff --git a/tools/objtool/objtool.c b/tools/objtool/objtool.c index 3c26ed561c7e..1c3622117c33 100644 --- a/tools/objtool/objtool.c +++ b/tools/objtool/objtool.c @@ -104,11 +104,13 @@ char *top_level_dir(const char *file) return str; } - int main(int argc, const char **argv) { static const char *UNUSED = "OBJTOOL_NOT_IMPLEMENTED"; + if (init_signal_handler()) + return -1; + /* libsubcmd init */ exec_cmd_init("objtool", UNUSED, UNUSED, UNUSED); pager_init(UNUSED); diff --git a/tools/objtool/signal.c b/tools/objtool/signal.c new file mode 100644 index 000000000000..af5c65c0fb2d --- /dev/null +++ b/tools/objtool/signal.c @@ -0,0 +1,135 @@ +/* + * signal.c: Register a sigaltstack for objtool, to be able to + * run a signal handler on a separate stack even if + * the main process stack has overflown. Print out + * stack overflow errors when this happens. + */ +#include <stdio.h> +#include <stdlib.h> +#include <signal.h> +#include <unistd.h> +#include <sys/resource.h> +#include <string.h> + +#include <objtool/objtool.h> +#include <objtool/warn.h> + +static unsigned long stack_limit; + +static bool is_stack_overflow(void *fault_addr) +{ + unsigned long fault = (unsigned long)fault_addr; + + /* Check if fault is in the guard page just below the limit. */ + return fault < stack_limit && fault >= stack_limit - 4096; +} + +static void signal_handler(int sig_num, siginfo_t *info, void *context) +{ + struct sigaction sa_dfl = {0}; + const char *sig_name; + char msg[256]; + int msg_len; + + switch (sig_num) { + case SIGSEGV: sig_name = "SIGSEGV"; break; + case SIGBUS: sig_name = "SIGBUS"; break; + case SIGILL: sig_name = "SIGILL"; break; + case SIGABRT: sig_name = "SIGABRT"; break; + default: sig_name = "Unknown signal"; break; + } + + if (is_stack_overflow(info->si_addr)) { + msg_len = snprintf(msg, sizeof(msg), + "%s: error: %s: objtool stack overflow!\n", + objname, sig_name); + } else { + msg_len = snprintf(msg, sizeof(msg), + "%s: error: %s: objtool crash!\n", + objname, sig_name); + } + + msg_len = write(STDERR_FILENO, msg, msg_len); + + /* Re-raise the signal to trigger the core dump */ + sa_dfl.sa_handler = SIG_DFL; + sigaction(sig_num, &sa_dfl, NULL); + raise(sig_num); +} + +static int read_stack_limit(void) +{ + unsigned long stack_start, stack_end; + struct rlimit rlim; + char line[256]; + int ret = 0; + FILE *fp; + + if (getrlimit(RLIMIT_STACK, &rlim)) { + ERROR_GLIBC("getrlimit"); + return -1; + } + + fp = fopen("/proc/self/maps", "r"); + if (!fp) { + ERROR_GLIBC("fopen"); + return -1; + } + + while (fgets(line, sizeof(line), fp)) { + if (strstr(line, "[stack]")) { + if (sscanf(line, "%lx-%lx", &stack_start, &stack_end) != 2) { + ERROR_GLIBC("sscanf"); + ret = -1; + goto done; + } + stack_limit = stack_end - rlim.rlim_cur; + goto done; + } + } + + ret = -1; + ERROR("/proc/self/maps: can't find [stack]"); + +done: + fclose(fp); + + return ret; +} + +int init_signal_handler(void) +{ + int signals[] = {SIGSEGV, SIGBUS, SIGILL, SIGABRT}; + struct sigaction sa; + stack_t ss; + + if (read_stack_limit()) + return -1; + + ss.ss_sp = malloc(SIGSTKSZ); + if (!ss.ss_sp) { + ERROR_GLIBC("malloc"); + return -1; + } + ss.ss_size = SIGSTKSZ; + ss.ss_flags = 0; + + if (sigaltstack(&ss, NULL) == -1) { + ERROR_GLIBC("sigaltstack"); + return -1; + } + + sa.sa_sigaction = signal_handler; + sigemptyset(&sa.sa_mask); + + sa.sa_flags = SA_ONSTACK | SA_SIGINFO; + + for (int i = 0; i < ARRAY_SIZE(signals); i++) { + if (sigaction(signals[i], &sa, NULL) == -1) { + ERROR_GLIBC("sigaction"); + return -1; + } + } + + return 0; +} diff --git a/tools/power/x86/turbostat/turbostat.8 b/tools/power/x86/turbostat/turbostat.8 index 3340def58d01..1551fcdbfd8a 100644 --- a/tools/power/x86/turbostat/turbostat.8 +++ b/tools/power/x86/turbostat/turbostat.8 @@ -101,7 +101,7 @@ The column name "all" can be used to enable all disabled-by-default built-in cou .PP \fB--show column\fP show only the specified built-in columns. May be invoked multiple times, or with a comma-separated list of column names. .PP -\fB--show CATEGORY --hide CATEGORY\fP Show and hide also accept a single CATEGORY of columns: "all", "topology", "idle", "frequency", "power", "cpuidle", "hwidle", "swidle", "other". "idle" (enabled by default), includes "hwidle" and "pct_idle". "cpuidle" (default disabled) includes cpuidle software invocation counters. "swidle" includes "cpuidle" plus "pct_idle". "hwidle" includes only hardware based idle residency counters. Older versions of turbostat used the term "sysfs" for what is now "swidle". +\fB--show CATEGORY --hide CATEGORY\fP Show and hide also accept a comma-separated-list of CATEGORIES of columns: "all", "topology", "idle", "frequency", "power", "cpuidle", "hwidle", "swidle", "cache", "llc", "other". "idle" (enabled by default), includes "hwidle" and "pct_idle". "cpuidle" (default disabled) includes cpuidle software invocation counters. "swidle" includes "cpuidle" plus "pct_idle". "hwidle" includes only hardware based idle residency counters. Older versions of turbostat used the term "sysfs" for what is now "swidle". .PP \fB--Dump\fP displays the raw counter values. .PP @@ -159,6 +159,10 @@ The system configuration dump (if --quiet is not used) is followed by statistics .PP \fBSMI\fP The number of System Management Interrupts serviced CPU during the measurement interval. While this counter is actually per-CPU, SMI are triggered on all processors, so the number should be the same for all CPUs. .PP +\fBLLCkRPS\fP Last Level Cache Thousands of References Per Second. For CPUs with an L3 LLC, this is the number of references that CPU made to the L3 (and the number of misses that CPU made to it's L2). For CPUs with an L2 LLC, this is the number of references to the L2 (and the number of misses to the CPU's L1). The system summary row shows the sum for all CPUs. In both cases, the value displayed is the actual value divided by 1000 in the interest of usually fitting into 8 columns. +.PP +\fBLLC%hit\fP Last Level Cache Hit Rate %. Hit Rate Percent = 100.0 * (References - Misses)/References. The system summary row shows the weighted average for all CPUs (100.0 * (Sum_References - Sum_Misses)/Sum_References). +.PP \fBC1, C2, C3...\fP The number times Linux requested the C1, C2, C3 idle state during the measurement interval. The system summary line shows the sum for all CPUs. These are C-state names as exported in /sys/devices/system/cpu/cpu*/cpuidle/state*/name. While their names are generic, their attributes are processor specific. They the system description section of output shows what MWAIT sub-states they are mapped to on each system. These counters are in the "cpuidle" group, which is disabled, by default. .PP \fBC1+, C2+, C3+...\fP The idle governor idle state misprediction statistics. Inidcates the number times Linux requested the C1, C2, C3 idle state during the measurement interval, but should have requested a deeper idle state (if it exists and enabled). These statistics come from the /sys/devices/system/cpu/cpu*/cpuidle/state*/below file. These counters are in the "cpuidle" group, which is disabled, by default. @@ -410,25 +414,24 @@ CPU pCPU%c1 CPU%c1 .fi .SH ADD PERF COUNTER EXAMPLE #2 (using virtual cpu device) -Here we run on hybrid, Raptor Lake platform. -We limit turbostat to show output for just cpu0 (pcore) and cpu12 (ecore). +Here we run on hybrid, Meteor Lake platform. +We limit turbostat to show output for just cpu0 (pcore) and cpu4 (ecore). We add a counter showing number of L3 cache misses, using virtual "cpu" device, labeling it with the column header, "VCMISS". We add a counter showing number of L3 cache misses, using virtual "cpu_core" device, -labeling it with the column header, "PCMISS". This will fail on ecore cpu12. +labeling it with the column header, "PCMISS". This will fail on ecore cpu4. We add a counter showing number of L3 cache misses, using virtual "cpu_atom" device, labeling it with the column header, "ECMISS". This will fail on pcore cpu0. We display it only once, after the conclusion of 0.1 second sleep. .nf -sudo ./turbostat --quiet --cpu 0,12 --show CPU --add perf/cpu/cache-misses,cpu,delta,raw,VCMISS --add perf/cpu_core/cache-misses,cpu,delta,raw,PCMISS --add perf/cpu_atom/cache-misses,cpu,delta,raw,ECMISS sleep .1 +sudo ./turbostat --quiet --cpu 0,4 --show CPU --add perf/cpu/cache-misses,cpu,delta,VCMISS --add perf/cpu_core/cache-misses,cpu,delta,PCMISS --add perf/cpu_atom/cache-misses,cpu,delta,ECMISS sleep 5 turbostat: added_perf_counters_init_: perf/cpu_atom/cache-misses: failed to open counter on cpu0 -turbostat: added_perf_counters_init_: perf/cpu_core/cache-misses: failed to open counter on cpu12 -0.104630 sec -CPU ECMISS PCMISS VCMISS -- 0x0000000000000000 0x0000000000000000 0x0000000000000000 -0 0x0000000000000000 0x0000000000007951 0x0000000000007796 -12 0x000000000001137a 0x0000000000000000 0x0000000000011392 - +turbostat: added_perf_counters_init_: perf/cpu_core/cache-misses: failed to open counter on cpu4 +5.001207 sec +CPU ECMISS PCMISS VCMISS +- 41586506 46291219 87877749 +4 83173012 0 83173040 +0 0 92582439 92582458 .fi .SH ADD PMT COUNTER EXAMPLE diff --git a/tools/power/x86/turbostat/turbostat.c b/tools/power/x86/turbostat/turbostat.c index f2512d78bcbd..5ad45c2ac5bd 100644 --- a/tools/power/x86/turbostat/turbostat.c +++ b/tools/power/x86/turbostat/turbostat.c @@ -142,6 +142,7 @@ struct msr_counter { #define FLAGS_SHOW (1 << 1) #define SYSFS_PERCPU (1 << 1) }; +static int use_android_msr_path; struct msr_counter bic[] = { { 0x0, "usec", NULL, 0, 0, 0, NULL, 0 }, @@ -209,6 +210,8 @@ struct msr_counter bic[] = { { 0x0, "NMI", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "CPU%c1e", NULL, 0, 0, 0, NULL, 0 }, { 0x0, "pct_idle", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "LLCkRPS", NULL, 0, 0, 0, NULL, 0 }, + { 0x0, "LLC%hit", NULL, 0, 0, 0, NULL, 0 }, }; /* n.b. bic_names must match the order in bic[], above */ @@ -278,6 +281,8 @@ enum bic_names { BIC_NMI, BIC_CPU_c1e, BIC_pct_idle, + BIC_LLC_RPS, + BIC_LLC_HIT, MAX_BIC }; @@ -305,6 +310,7 @@ static cpu_set_t bic_group_frequency; static cpu_set_t bic_group_hw_idle; static cpu_set_t bic_group_sw_idle; static cpu_set_t bic_group_idle; +static cpu_set_t bic_group_cache; static cpu_set_t bic_group_other; static cpu_set_t bic_group_disabled_by_default; static cpu_set_t bic_enabled; @@ -413,9 +419,14 @@ static void bic_groups_init(void) SET_BIC(BIC_pct_idle, &bic_group_sw_idle); BIC_INIT(&bic_group_idle); + CPU_OR(&bic_group_idle, &bic_group_idle, &bic_group_hw_idle); SET_BIC(BIC_pct_idle, &bic_group_idle); + BIC_INIT(&bic_group_cache); + SET_BIC(BIC_LLC_RPS, &bic_group_cache); + SET_BIC(BIC_LLC_HIT, &bic_group_cache); + BIC_INIT(&bic_group_other); SET_BIC(BIC_IRQ, &bic_group_other); SET_BIC(BIC_NMI, &bic_group_other); @@ -466,12 +477,11 @@ static void bic_groups_init(void) #define PCL_10 14 /* PC10 */ #define PCLUNL 15 /* Unlimited */ -struct amperf_group_fd; - char *proc_stat = "/proc/stat"; FILE *outf; int *fd_percpu; int *fd_instr_count_percpu; +int *fd_llc_percpu; struct timeval interval_tv = { 5, 0 }; struct timespec interval_ts = { 5, 0 }; @@ -482,11 +492,12 @@ unsigned int quiet; unsigned int shown; unsigned int sums_need_wide_columns; unsigned int rapl_joules; +unsigned int valid_rapl_msrs; unsigned int summary_only; unsigned int list_header_only; unsigned int dump_only; unsigned int force_load; -unsigned int has_aperf; +unsigned int cpuid_has_aperf_mperf; unsigned int has_aperf_access; unsigned int has_epb; unsigned int has_turbo; @@ -552,8 +563,7 @@ static struct gfx_sysfs_info gfx_info[GFX_MAX]; int get_msr(int cpu, off_t offset, unsigned long long *msr); int add_counter(unsigned int msr_num, char *path, char *name, - unsigned int width, enum counter_scope scope, - enum counter_type type, enum counter_format format, int flags, int package_num); + unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int package_num); /* Model specific support Start */ @@ -578,7 +588,7 @@ struct platform_features { bool has_cst_prewake_bit; /* Cstate prewake bit in MSR_IA32_POWER_CTL */ int trl_msrs; /* MSR_TURBO_RATIO_LIMIT/LIMIT1/LIMIT2/SECONDARY, Atom TRL MSRs */ int plr_msrs; /* MSR_CORE/GFX/RING_PERF_LIMIT_REASONS */ - int rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */ + int plat_rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */ bool has_per_core_rapl; /* Indicates cores energy collection is per-core, not per-package. AMD specific for now */ bool has_rapl_divisor; /* Divisor for Energy unit raw value from MSR_RAPL_POWER_UNIT */ bool has_fixed_rapl_unit; /* Fixed Energy Unit used for DRAM RAPL Domain */ @@ -733,7 +743,7 @@ static const struct platform_features snb_features = { .cst_limit = CST_LIMIT_SNB, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features snx_features = { @@ -745,7 +755,7 @@ static const struct platform_features snx_features = { .cst_limit = CST_LIMIT_SNB, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, }; static const struct platform_features ivb_features = { @@ -758,7 +768,7 @@ static const struct platform_features ivb_features = { .cst_limit = CST_LIMIT_SNB, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features ivx_features = { @@ -770,7 +780,7 @@ static const struct platform_features ivx_features = { .cst_limit = CST_LIMIT_SNB, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE | TRL_LIMIT1, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL, }; static const struct platform_features hsw_features = { @@ -784,7 +794,7 @@ static const struct platform_features hsw_features = { .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features hsx_features = { @@ -798,7 +808,7 @@ static const struct platform_features hsx_features = { .has_irtl_msrs = 1, .trl_msrs = TRL_BASE | TRL_LIMIT1 | TRL_LIMIT2, .plr_msrs = PLR_CORE | PLR_RING, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, .has_fixed_rapl_unit = 1, }; @@ -813,7 +823,7 @@ static const struct platform_features hswl_features = { .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features hswg_features = { @@ -827,7 +837,7 @@ static const struct platform_features hswg_features = { .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features bdw_features = { @@ -840,7 +850,7 @@ static const struct platform_features bdw_features = { .cst_limit = CST_LIMIT_HSW, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features bdwg_features = { @@ -853,7 +863,7 @@ static const struct platform_features bdwg_features = { .cst_limit = CST_LIMIT_HSW, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO, }; static const struct platform_features bdx_features = { @@ -867,7 +877,7 @@ static const struct platform_features bdx_features = { .has_irtl_msrs = 1, .has_cst_auto_convension = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, .has_fixed_rapl_unit = 1, }; @@ -884,7 +894,7 @@ static const struct platform_features skl_features = { .has_ext_cst_msrs = 1, .trl_msrs = TRL_BASE, .tcc_offset_bits = 6, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, .enable_tsc_tweak = 1, }; @@ -901,7 +911,7 @@ static const struct platform_features cnl_features = { .has_ext_cst_msrs = 1, .trl_msrs = TRL_BASE, .tcc_offset_bits = 6, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS, .enable_tsc_tweak = 1, }; @@ -919,7 +929,7 @@ static const struct platform_features adl_features = { .has_ext_cst_msrs = cnl_features.has_ext_cst_msrs, .trl_msrs = cnl_features.trl_msrs, .tcc_offset_bits = cnl_features.tcc_offset_bits, - .rapl_msrs = cnl_features.rapl_msrs, + .plat_rapl_msrs = cnl_features.plat_rapl_msrs, .enable_tsc_tweak = cnl_features.enable_tsc_tweak, }; @@ -937,7 +947,7 @@ static const struct platform_features lnl_features = { .has_ext_cst_msrs = adl_features.has_ext_cst_msrs, .trl_msrs = adl_features.trl_msrs, .tcc_offset_bits = adl_features.tcc_offset_bits, - .rapl_msrs = adl_features.rapl_msrs, + .plat_rapl_msrs = adl_features.plat_rapl_msrs, .enable_tsc_tweak = adl_features.enable_tsc_tweak, }; @@ -952,7 +962,7 @@ static const struct platform_features skx_features = { .has_irtl_msrs = 1, .has_cst_auto_convension = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, .has_fixed_rapl_unit = 1, }; @@ -968,7 +978,7 @@ static const struct platform_features icx_features = { .has_irtl_msrs = 1, .has_cst_prewake_bit = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, .has_fixed_rapl_unit = 1, }; @@ -985,7 +995,7 @@ static const struct platform_features spr_features = { .has_cst_prewake_bit = 1, .has_fixed_rapl_psys_unit = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, }; static const struct platform_features dmr_features = { @@ -1000,7 +1010,7 @@ static const struct platform_features dmr_features = { .has_fixed_rapl_psys_unit = spr_features.has_fixed_rapl_psys_unit, .trl_msrs = spr_features.trl_msrs, .has_msr_module_c6_res_ms = 1, /* DMR has Dual-Core-Module and MC6 MSR */ - .rapl_msrs = 0, /* DMR does not have RAPL MSRs */ + .plat_rapl_msrs = 0, /* DMR does not have RAPL MSRs */ .plr_msrs = 0, /* DMR does not have PLR MSRs */ .has_irtl_msrs = 0, /* DMR does not have IRTL MSRs */ .has_config_tdp = 0, /* DMR does not have CTDP MSRs */ @@ -1019,7 +1029,7 @@ static const struct platform_features srf_features = { .has_irtl_msrs = 1, .has_cst_prewake_bit = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, }; static const struct platform_features grr_features = { @@ -1035,7 +1045,7 @@ static const struct platform_features grr_features = { .has_irtl_msrs = 1, .has_cst_prewake_bit = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS, }; static const struct platform_features slv_features = { @@ -1048,7 +1058,7 @@ static const struct platform_features slv_features = { .has_msr_c6_demotion_policy_config = 1, .has_msr_atom_pkg_c6_residency = 1, .trl_msrs = TRL_ATOM, - .rapl_msrs = RAPL_PKG | RAPL_CORE, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE, .has_rapl_divisor = 1, .rapl_quirk_tdp = 30, }; @@ -1061,7 +1071,7 @@ static const struct platform_features slvd_features = { .cst_limit = CST_LIMIT_SLV, .has_msr_atom_pkg_c6_residency = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_CORE, + .plat_rapl_msrs = RAPL_PKG | RAPL_CORE, .rapl_quirk_tdp = 30, }; @@ -1082,7 +1092,7 @@ static const struct platform_features gmt_features = { .cst_limit = CST_LIMIT_GMT, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, }; static const struct platform_features gmtd_features = { @@ -1095,7 +1105,7 @@ static const struct platform_features gmtd_features = { .has_irtl_msrs = 1, .has_msr_core_c1_res = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS, }; static const struct platform_features gmtp_features = { @@ -1107,7 +1117,7 @@ static const struct platform_features gmtp_features = { .cst_limit = CST_LIMIT_GMT, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, + .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO, }; static const struct platform_features tmt_features = { @@ -1118,7 +1128,7 @@ static const struct platform_features tmt_features = { .cst_limit = CST_LIMIT_GMT, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE, - .rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX, .enable_tsc_tweak = 1, }; @@ -1130,7 +1140,7 @@ static const struct platform_features tmtd_features = { .cst_limit = CST_LIMIT_GMT, .has_irtl_msrs = 1, .trl_msrs = TRL_BASE | TRL_CORECOUNT, - .rapl_msrs = RAPL_PKG_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL, }; static const struct platform_features knl_features = { @@ -1142,7 +1152,7 @@ static const struct platform_features knl_features = { .cst_limit = CST_LIMIT_KNL, .has_msr_knl_core_c6_residency = 1, .trl_msrs = TRL_KNL, - .rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, + .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL, .has_fixed_rapl_unit = 1, .need_perf_multiplier = 1, }; @@ -1151,7 +1161,7 @@ static const struct platform_features default_features = { }; static const struct platform_features amd_features_with_rapl = { - .rapl_msrs = RAPL_AMD_F17H, + .plat_rapl_msrs = RAPL_AMD_F17H, .has_per_core_rapl = 1, .rapl_quirk_tdp = 280, /* This is the max stock TDP of HEDT/Server Fam17h+ chips */ }; @@ -1210,6 +1220,9 @@ static const struct platform_data turbostat_pdata[] = { { INTEL_ARROWLAKE, &adl_features }, { INTEL_LUNARLAKE_M, &lnl_features }, { INTEL_PANTHERLAKE_L, &lnl_features }, + { INTEL_NOVALAKE, &lnl_features }, + { INTEL_NOVALAKE_L, &lnl_features }, + { INTEL_WILDCATLAKE_L, &lnl_features }, { INTEL_ATOM_SILVERMONT, &slv_features }, { INTEL_ATOM_SILVERMONT_D, &slvd_features }, { INTEL_ATOM_AIRMONT, &amt_features }, @@ -1294,8 +1307,7 @@ char *progname; #define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */ cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset; -size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, - cpu_subset_size; +size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size; #define MAX_ADDED_THREAD_COUNTERS 24 #define MAX_ADDED_CORE_COUNTERS 8 #define MAX_ADDED_PACKAGE_COUNTERS 16 @@ -1991,6 +2003,10 @@ void pmt_counter_resize(struct pmt_counter *pcounter, unsigned int new_size) pmt_counter_resize_(pcounter, new_size); } +struct llc_stats { + unsigned long long references; + unsigned long long misses; +}; struct thread_data { struct timeval tv_begin; struct timeval tv_end; @@ -2003,6 +2019,7 @@ struct thread_data { unsigned long long irq_count; unsigned long long nmi_count; unsigned int smi_count; + struct llc_stats llc; unsigned int cpu_id; unsigned int apic_id; unsigned int x2apic_id; @@ -2118,7 +2135,7 @@ off_t idx_to_offset(int idx) switch (idx) { case IDX_PKG_ENERGY: - if (platform->rapl_msrs & RAPL_AMD_F17H) + if (valid_rapl_msrs & RAPL_AMD_F17H) offset = MSR_PKG_ENERGY_STAT; else offset = MSR_PKG_ENERGY_STATUS; @@ -2184,19 +2201,19 @@ int idx_valid(int idx) { switch (idx) { case IDX_PKG_ENERGY: - return platform->rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H); + return valid_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H); case IDX_DRAM_ENERGY: - return platform->rapl_msrs & RAPL_DRAM; + return valid_rapl_msrs & RAPL_DRAM; case IDX_PP0_ENERGY: - return platform->rapl_msrs & RAPL_CORE_ENERGY_STATUS; + return valid_rapl_msrs & RAPL_CORE_ENERGY_STATUS; case IDX_PP1_ENERGY: - return platform->rapl_msrs & RAPL_GFX; + return valid_rapl_msrs & RAPL_GFX; case IDX_PKG_PERF: - return platform->rapl_msrs & RAPL_PKG_PERF_STATUS; + return valid_rapl_msrs & RAPL_PKG_PERF_STATUS; case IDX_DRAM_PERF: - return platform->rapl_msrs & RAPL_DRAM_PERF_STATUS; + return valid_rapl_msrs & RAPL_DRAM_PERF_STATUS; case IDX_PSYS_ENERGY: - return platform->rapl_msrs & RAPL_PSYS; + return valid_rapl_msrs & RAPL_PSYS; default: return 0; } @@ -2362,23 +2379,19 @@ int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pk return retval; } -int is_cpu_first_thread_in_core(PER_THREAD_PARAMS) +int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c) { - UNUSED(p); - return ((int)t->cpu_id == c->base_cpu || c->base_cpu < 0); } -int is_cpu_first_core_in_package(PER_THREAD_PARAMS) +int is_cpu_first_core_in_package(struct thread_data *t, struct pkg_data *p) { - UNUSED(c); - return ((int)t->cpu_id == p->base_cpu || p->base_cpu < 0); } -int is_cpu_first_thread_in_package(PER_THREAD_PARAMS) +int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p) { - return is_cpu_first_thread_in_core(t, c, p) && is_cpu_first_core_in_package(t, c, p); + return is_cpu_first_thread_in_core(t, c) && is_cpu_first_core_in_package(t, p); } int cpu_migrate(int cpu) @@ -2400,20 +2413,11 @@ int get_msr_fd(int cpu) if (fd) return fd; -#if defined(ANDROID) - sprintf(pathname, "/dev/msr%d", cpu); -#else - sprintf(pathname, "/dev/cpu/%d/msr", cpu); -#endif + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu); fd = open(pathname, O_RDONLY); if (fd < 0) -#if defined(ANDROID) - err(-1, "%s open failed, try chown or chmod +r /dev/msr*, " - "or run with --no-msr, or run as root", pathname); -#else - err(-1, "%s open failed, try chown or chmod +r /dev/cpu/*/msr, " - "or run with --no-msr, or run as root", pathname); -#endif + err(-1, "%s open failed, try chown or chmod +r %s, " + "or run with --no-msr, or run as root", pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr"); fd_percpu[cpu] = fd; return fd; @@ -2432,6 +2436,13 @@ static void bic_disable_msr_access(void) free_sys_msr_counters(); } +static void bic_disable_perf_access(void) +{ + CLR_BIC(BIC_IPC, &bic_enabled); + CLR_BIC(BIC_LLC_RPS, &bic_enabled); + CLR_BIC(BIC_LLC_HIT, &bic_enabled); +} + static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags) { assert(!no_perf); @@ -2512,7 +2523,7 @@ int add_rapl_msr_counter(int cpu, const struct rapl_counter_arch_info *cai) { int ret; - if (!(platform->rapl_msrs & cai->feature_mask)) + if (!(valid_rapl_msrs & cai->feature_mask)) return -1; ret = add_msr_counter(cpu, cai->msr); @@ -2656,6 +2667,12 @@ void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode) } else if (!strcmp(name_list, "idle")) { CPU_OR(ret_set, ret_set, &bic_group_idle); break; + } else if (!strcmp(name_list, "cache")) { + CPU_OR(ret_set, ret_set, &bic_group_cache); + break; + } else if (!strcmp(name_list, "llc")) { + CPU_OR(ret_set, ret_set, &bic_group_cache); + break; } else if (!strcmp(name_list, "swidle")) { CPU_OR(ret_set, ret_set, &bic_group_sw_idle); break; @@ -2677,8 +2694,7 @@ void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode) if (mode == SHOW_LIST) { deferred_add_names[deferred_add_index++] = name_list; if (deferred_add_index >= MAX_DEFERRED) { - fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", - MAX_DEFERRED, name_list); + fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", MAX_DEFERRED, name_list); help(); exit(1); } @@ -2687,8 +2703,7 @@ void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode) if (debug) fprintf(stderr, "deferred \"%s\"\n", name_list); if (deferred_skip_index >= MAX_DEFERRED) { - fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", - MAX_DEFERRED, name_list); + fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", MAX_DEFERRED, name_list); help(); exit(1); } @@ -2702,6 +2717,47 @@ void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode) } } +/* + * print_name() + * Print column header name for raw 64-bit counter in 16 columns (at least 8-char plus a tab) + * Otherwise, allow the name + tab to fit within 8-coumn tab-stop. + * In both cases, left justififed, just like other turbostat columns, + * to allow the column values to consume the tab. + * + * Yes, 32-bit counters can overflow 8-columns, and + * 64-bit counters can overflow 16-columns, but that is uncommon. + */ +static inline int print_name(int width, int *printed, char *delim, char *name, enum counter_type type, enum counter_format format) +{ + UNUSED(type); + + if (format == FORMAT_RAW && width >= 64) + return (sprintf(outp, "%s%-8s", (*printed++ ? delim : ""), name)); + else + return (sprintf(outp, "%s%s", (*printed++ ? delim : ""), name)); +} + +static inline int print_hex_value(int width, int *printed, char *delim, unsigned long long value) +{ + if (width <= 32) + return (sprintf(outp, "%s%08x", (*printed++ ? delim : ""), (unsigned int)value)); + else + return (sprintf(outp, "%s%016llx", (*printed++ ? delim : ""), value)); +} + +static inline int print_decimal_value(int width, int *printed, char *delim, unsigned long long value) +{ + if (width <= 32) + return (sprintf(outp, "%s%d", (*printed++ ? delim : ""), (unsigned int)value)); + else + return (sprintf(outp, "%s%-8lld", (*printed++ ? delim : ""), value)); +} + +static inline int print_float_value(int *printed, char *delim, double value) +{ + return (sprintf(outp, "%s%0.2f", (*printed++ ? delim : ""), value)); +} + void print_header(char *delim) { struct msr_counter *mp; @@ -2757,50 +2813,28 @@ void print_header(char *delim) if (DO_BIC(BIC_SMI)) outp += sprintf(outp, "%sSMI", (printed++ ? delim : "")); - for (mp = sys.tp; mp; mp = mp->next) { + if (DO_BIC(BIC_LLC_RPS)) + outp += sprintf(outp, "%sLLCkRPS", (printed++ ? delim : "")); - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) { - if (mp->width == 64) - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), mp->name); - else - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), mp->name); - } else { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), mp->name); - else - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), mp->name); - } - } + if (DO_BIC(BIC_LLC_HIT)) + outp += sprintf(outp, "%sLLC%%hit", (printed++ ? delim : "")); - for (pp = sys.perf_tp; pp; pp = pp->next) { + for (mp = sys.tp; mp; mp = mp->next) + outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); - if (pp->format == FORMAT_RAW) { - if (pp->width == 64) - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), pp->name); - } else { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), pp->name); - } - } + for (pp = sys.perf_tp; pp; pp = pp->next) + outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format); ppmt = sys.pmt_tp; while (ppmt) { switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), ppmt->name); - else - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), ppmt->name); - + outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; case PMT_TYPE_XTAL_TIME: case PMT_TYPE_TCORE_CLOCK: - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), ppmt->name); + outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; } @@ -2825,63 +2859,36 @@ void print_header(char *delim) if (DO_BIC(BIC_CORE_THROT_CNT)) outp += sprintf(outp, "%sCoreThr", (printed++ ? delim : "")); - if (platform->rapl_msrs && !rapl_joules) { + if (valid_rapl_msrs && !rapl_joules) { if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl) outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : "")); - } else if (platform->rapl_msrs && rapl_joules) { + } else if (valid_rapl_msrs && rapl_joules) { if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl) outp += sprintf(outp, "%sCor_J", (printed++ ? delim : "")); } - for (mp = sys.cp; mp; mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) { - if (mp->width == 64) - outp += sprintf(outp, "%s%18.18s", delim, mp->name); - else - outp += sprintf(outp, "%s%10.10s", delim, mp->name); - } else { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", delim, mp->name); - else - outp += sprintf(outp, "%s%s", delim, mp->name); - } - } + for (mp = sys.cp; mp; mp = mp->next) + outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); - for (pp = sys.perf_cp; pp; pp = pp->next) { - - if (pp->format == FORMAT_RAW) { - if (pp->width == 64) - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), pp->name); - } else { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), pp->name); - } - } + for (pp = sys.perf_cp; pp; pp = pp->next) + outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format); ppmt = sys.pmt_cp; while (ppmt) { switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), ppmt->name); - else - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), ppmt->name); + outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; case PMT_TYPE_XTAL_TIME: case PMT_TYPE_TCORE_CLOCK: - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), ppmt->name); + outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; } ppmt = ppmt->next; } - if (DO_BIC(BIC_PkgTmp)) outp += sprintf(outp, "%sPkgTmp", (printed++ ? delim : "")); @@ -2963,51 +2970,22 @@ void print_header(char *delim) if (DO_BIC(BIC_UNCORE_MHZ)) outp += sprintf(outp, "%sUncMHz", (printed++ ? delim : "")); - for (mp = sys.pp; mp; mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) { - if (mp->width == 64) - outp += sprintf(outp, "%s%18.18s", delim, mp->name); - else if (mp->width == 32) - outp += sprintf(outp, "%s%10.10s", delim, mp->name); - else - outp += sprintf(outp, "%s%7.7s", delim, mp->name); - } else { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", delim, mp->name); - else - outp += sprintf(outp, "%s%7.7s", delim, mp->name); - } - } - - for (pp = sys.perf_pp; pp; pp = pp->next) { + for (mp = sys.pp; mp; mp = mp->next) + outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format); - if (pp->format == FORMAT_RAW) { - if (pp->width == 64) - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), pp->name); - } else { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8s", (printed++ ? delim : ""), pp->name); - else - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), pp->name); - } - } + for (pp = sys.perf_pp; pp; pp = pp->next) + outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format); ppmt = sys.pmt_pp; while (ppmt) { switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s%10.10s", (printed++ ? delim : ""), ppmt->name); - else - outp += sprintf(outp, "%s%18.18s", (printed++ ? delim : ""), ppmt->name); - + outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; case PMT_TYPE_XTAL_TIME: case PMT_TYPE_TCORE_CLOCK: - outp += sprintf(outp, "%s%s", (printed++ ? delim : ""), ppmt->name); + outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format); break; } @@ -3022,6 +3000,25 @@ void print_header(char *delim) outp += sprintf(outp, "\n"); } +/* + * pct() + * + * If absolute value is < 1.1, return percentage + * otherwise, return nan + * + * return value is appropriate for printing percentages with %f + * while flagging some obvious erroneous values. + */ +double pct(double d) +{ + + double abs = fabs(d); + + if (abs < 1.10) + return (100.0 * d); + return nan(""); +} + int dump_counters(PER_THREAD_PARAMS) { int i; @@ -3047,14 +3044,16 @@ int dump_counters(PER_THREAD_PARAMS) if (DO_BIC(BIC_SMI)) outp += sprintf(outp, "SMI: %d\n", t->smi_count); + outp += sprintf(outp, "LLC refs: %lld", t->llc.references); + outp += sprintf(outp, "LLC miss: %lld", t->llc.misses); + outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses) / t->llc.references)); + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - outp += - sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, - t->counter[i], mp->sp->path); + outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path); } } - if (c && is_cpu_first_thread_in_core(t, c, p)) { + if (c && is_cpu_first_thread_in_core(t, c)) { outp += sprintf(outp, "core: %d\n", c->core_id); outp += sprintf(outp, "c3: %016llX\n", c->c3); outp += sprintf(outp, "c6: %016llX\n", c->c6); @@ -3069,14 +3068,12 @@ int dump_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "Joules: %0llX (scale: %lf)\n", energy_value, energy_scale); for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - outp += - sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, - c->counter[i], mp->sp->path); + outp += sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, c->counter[i], mp->sp->path); } outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us); } - if (p && is_cpu_first_core_in_package(t, c, p)) { + if (p && is_cpu_first_core_in_package(t, p)) { outp += sprintf(outp, "package: %d\n", p->package_id); outp += sprintf(outp, "Weighted cores: %016llX\n", p->pkg_wtd_core_c0); @@ -3106,9 +3103,7 @@ int dump_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "PTM: %dC\n", p->pkg_temp_c); for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - outp += - sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, - p->counter[i], mp->sp->path); + outp += sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, p->counter[i], mp->sp->path); } } @@ -3134,6 +3129,26 @@ double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desir return scaled; } +void get_perf_llc_stats(int cpu, struct llc_stats *llc) +{ + struct read_format { + unsigned long long num_read; + struct llc_stats llc; + } r; + const ssize_t expected_read_size = sizeof(r); + ssize_t actual_read_size; + + actual_read_size = read(fd_llc_percpu[cpu], &r, expected_read_size); + + if (actual_read_size == -1) + err(-1, "%s(cpu%d,) %d,,%ld\n", __func__, cpu, fd_llc_percpu[cpu], expected_read_size); + + llc->references = r.llc.references; + llc->misses = r.llc.misses; + if (actual_read_size != expected_read_size) + warn("%s: failed to read perf_data (req %zu act %zu)", __func__, expected_read_size, actual_read_size); +} + /* * column formatting convention & formats */ @@ -3143,7 +3158,8 @@ int format_counters(PER_THREAD_PARAMS) struct platform_counters *pplat_cnt = NULL; double interval_float, tsc; - char *fmt8; + char *fmt8 = "%s%.2f"; + int i; struct msr_counter *mp; struct perf_counter_info *pp; @@ -3157,11 +3173,11 @@ int format_counters(PER_THREAD_PARAMS) } /* if showing only 1st thread in core and this isn't one, bail out */ - if (show_core_only && !is_cpu_first_thread_in_core(t, c, p)) + if (show_core_only && !is_cpu_first_thread_in_core(t, c)) return 0; /* if showing only 1st thread in pkg and this isn't one, bail out */ - if (show_pkg_only && !is_cpu_first_core_in_package(t, c, p)) + if (show_pkg_only && !is_cpu_first_core_in_package(t, p)) return 0; /*if not summary line and --cpu is used */ @@ -3223,8 +3239,7 @@ int format_counters(PER_THREAD_PARAMS) } if (DO_BIC(BIC_Node)) { if (t) - outp += sprintf(outp, "%s%d", - (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id); + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id); else outp += sprintf(outp, "%s-", (printed++ ? delim : "")); } @@ -3246,15 +3261,13 @@ int format_counters(PER_THREAD_PARAMS) outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float); if (DO_BIC(BIC_Busy)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->mperf / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf / tsc)); if (DO_BIC(BIC_Bzy_MHz)) { if (has_base_hz) - outp += - sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf); + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf); else - outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), - tsc / units * t->aperf / t->mperf / interval_float); + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), tsc / units * t->aperf / t->mperf / interval_float); } if (DO_BIC(BIC_TSC_MHz)) @@ -3283,96 +3296,81 @@ int format_counters(PER_THREAD_PARAMS) if (DO_BIC(BIC_SMI)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count); - /* Added counters */ + /* LLC Stats */ + if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT)) { + if (DO_BIC(BIC_LLC_RPS)) + outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000); + + if (DO_BIC(BIC_LLC_HIT)) + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses) / t->llc.references)); + } + + /* Added Thread Counters */ for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) { - if (mp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)t->counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->counter[i]); - } else if (mp->format == FORMAT_DELTA) { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->counter[i]); - } else if (mp->format == FORMAT_PERCENT) { + if (mp->format == FORMAT_RAW) + outp += print_hex_value(mp->width, &printed, delim, t->counter[i]); + else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(mp->width, &printed, delim, t->counter[i]); + else if (mp->format == FORMAT_PERCENT) { if (mp->type == COUNTER_USEC) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - t->counter[i] / interval_float / 10000); + outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000); else - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->counter[i] / tsc); + outp += print_float_value(&printed, delim, pct(t->counter[i] / tsc)); } } - /* Added perf counters */ + /* Added perf Thread Counters */ for (i = 0, pp = sys.perf_tp; pp; ++i, pp = pp->next) { - if (pp->format == FORMAT_RAW) { - if (pp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)t->perf_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->perf_counter[i]); - } else if (pp->format == FORMAT_DELTA) { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->perf_counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->perf_counter[i]); - } else if (pp->format == FORMAT_PERCENT) { + if (pp->format == FORMAT_RAW) + outp += print_hex_value(pp->width, &printed, delim, t->perf_counter[i]); + else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(pp->width, &printed, delim, t->perf_counter[i]); + else if (pp->format == FORMAT_PERCENT) { if (pp->type == COUNTER_USEC) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - t->perf_counter[i] / interval_float / 10000); + outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000); else - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->perf_counter[i] / tsc); + outp += print_float_value(&printed, delim, pct(t->perf_counter[i] / tsc)); } } + /* Added PMT Thread Counters */ for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) { const unsigned long value_raw = t->pmt_counter[i]; double value_converted; switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)t->pmt_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), t->pmt_counter[i]); - + outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, t->pmt_counter[i]); break; case PMT_TYPE_XTAL_TIME: - value_converted = 100.0 * value_raw / crystal_hz / interval_float; + value_converted = pct(value_raw / crystal_hz / interval_float); outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = 100.0 * value_raw / tcore_clock_freq_hz / interval_float; + value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); } } /* C1 */ if (DO_BIC(BIC_CPU_c1)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * t->c1 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1 / tsc)); /* print per-core data only for 1st thread in core */ - if (!is_cpu_first_thread_in_core(t, c, p)) + if (!is_cpu_first_thread_in_core(t, c)) goto done; if (DO_BIC(BIC_CPU_c3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c3 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3 / tsc)); if (DO_BIC(BIC_CPU_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c6 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6 / tsc)); if (DO_BIC(BIC_CPU_c7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->c7 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7 / tsc)); /* Mod%c6 */ if (DO_BIC(BIC_Mod_c6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->mc6_us / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us / tsc)); if (DO_BIC(BIC_CoreTmp)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c); @@ -3381,77 +3379,53 @@ int format_counters(PER_THREAD_PARAMS) if (DO_BIC(BIC_CORE_THROT_CNT)) outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt); + /* Added Core Counters */ for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) { - if (mp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)c->counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->counter[i]); - } else if (mp->format == FORMAT_DELTA) { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), c->counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->counter[i]); - } else if (mp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->counter[i] / tsc); - } + if (mp->format == FORMAT_RAW) + outp += print_hex_value(mp->width, &printed, delim, c->counter[i]); + else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]); + else if (mp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(c->counter[i] / tsc)); } + /* Added perf Core counters */ for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) { - if (pp->format == FORMAT_RAW) { - if (pp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)c->perf_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->perf_counter[i]); - } else if (pp->format == FORMAT_DELTA) { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), c->perf_counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->perf_counter[i]); - } else if (pp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * c->perf_counter[i] / tsc); - } + if (pp->format == FORMAT_RAW) + outp += print_hex_value(pp->width, &printed, delim, c->perf_counter[i]); + else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]); + else if (pp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(c->perf_counter[i] / tsc)); } + /* Added PMT Core counters */ for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) { const unsigned long value_raw = c->pmt_counter[i]; double value_converted; switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)c->pmt_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), c->pmt_counter[i]); - + outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, c->pmt_counter[i]); break; case PMT_TYPE_XTAL_TIME: - value_converted = 100.0 * value_raw / crystal_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / crystal_hz / interval_float); + outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = 100.0 * value_raw / tcore_clock_freq_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + outp += print_float_value(&printed, delim, value_converted); } } - fmt8 = "%s%.2f"; - if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&c->core_energy, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&c->core_energy, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_JOULES, interval_float)); /* print per-package data only for 1st core in package */ - if (!is_cpu_first_core_in_package(t, c, p)) + if (!is_cpu_first_core_in_package(t, p)) goto done; /* PkgTmp */ @@ -3463,8 +3437,7 @@ int format_counters(PER_THREAD_PARAMS) if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */ outp += sprintf(outp, "%s**.**", (printed++ ? delim : "")); } else { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - p->gfx_rc6_ms / 10.0 / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->gfx_rc6_ms / 10.0 / interval_float); } } @@ -3481,8 +3454,7 @@ int format_counters(PER_THREAD_PARAMS) if (p->sam_mc6_ms == -1) { /* detect GFX counter reset */ outp += sprintf(outp, "%s**.**", (printed++ ? delim : "")); } else { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - p->sam_mc6_ms / 10.0 / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->sam_mc6_ms / 10.0 / interval_float); } } @@ -3496,150 +3468,112 @@ int format_counters(PER_THREAD_PARAMS) /* Totl%C0, Any%C0 GFX%C0 CPUGFX% */ if (DO_BIC(BIC_Totl_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_wtd_core_c0 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */ if (DO_BIC(BIC_Any_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_core_c0 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0 / tsc)); if (DO_BIC(BIC_GFX_c0)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_any_gfxe_c0 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0 / tsc)); if (DO_BIC(BIC_CPUGFX)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pkg_both_core_gfxe_c0 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0 / tsc)); if (DO_BIC(BIC_Pkgpc2)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc2 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2 / tsc)); if (DO_BIC(BIC_Pkgpc3)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc3 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3 / tsc)); if (DO_BIC(BIC_Pkgpc6)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc6 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6 / tsc)); if (DO_BIC(BIC_Pkgpc7)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc7 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7 / tsc)); if (DO_BIC(BIC_Pkgpc8)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc8 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8 / tsc)); if (DO_BIC(BIC_Pkgpc9)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc9 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9 / tsc)); if (DO_BIC(BIC_Pkgpc10)) - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->pc10 / tsc); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10 / tsc)); if (DO_BIC(BIC_Diec6)) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->die_c6 / crystal_hz / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz / interval_float)); if (DO_BIC(BIC_CPU_LPI)) { if (p->cpu_lpi >= 0) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - 100.0 * p->cpu_lpi / 1000000.0 / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0 / interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } if (DO_BIC(BIC_SYS_LPI)) { if (p->sys_lpi >= 0) - outp += - sprintf(outp, "%s%.2f", (printed++ ? delim : ""), - 100.0 * p->sys_lpi / 1000000.0 / interval_float); + outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0 / interval_float)); else outp += sprintf(outp, "%s(neg)", (printed++ ? delim : "")); } if (DO_BIC(BIC_PkgWatt)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_GFXWatt)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_RAMWatt)) - outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_WATTS, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_Pkg_J)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_JOULES, interval_float)); if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_JOULES, interval_float)); if (DO_BIC(BIC_GFX_J)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_JOULES, interval_float)); if (DO_BIC(BIC_RAM_J)) - outp += sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float)); + outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float)); if (DO_BIC(BIC_PKG__)) outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float)); + sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float)); if (DO_BIC(BIC_RAM__)) outp += - sprintf(outp, fmt8, (printed++ ? delim : ""), - rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float)); + sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float)); /* UncMHz */ if (DO_BIC(BIC_UNCORE_MHZ)) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz); + /* Added Package Counters */ for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) { - if (mp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), (unsigned int)p->counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->counter[i]); - } else if (mp->format == FORMAT_DELTA) { - if ((mp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), p->counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), p->counter[i]); - } else if (mp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->counter[i] / tsc); - } else if (mp->type == COUNTER_K2M) + if (mp->format == FORMAT_RAW) + outp += print_hex_value(mp->width, &printed, delim, p->counter[i]); + else if (mp->type == COUNTER_K2M) outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->counter[i] / 1000); + else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]); + else if (mp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(p->counter[i] / tsc)); } + /* Added perf Package Counters */ for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) { - if (pp->format == FORMAT_RAW) { - if (pp->width == 32) - outp += - sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)p->perf_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->perf_counter[i]); - } else if (pp->format == FORMAT_DELTA) { - if ((pp->type == COUNTER_ITEMS) && sums_need_wide_columns) - outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), p->perf_counter[i]); - else - outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), p->perf_counter[i]); - } else if (pp->format == FORMAT_PERCENT) { - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100.0 * p->perf_counter[i] / tsc); - } else if (pp->type == COUNTER_K2M) { - outp += - sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000); - } + if (pp->format == FORMAT_RAW) + outp += print_hex_value(pp->width, &printed, delim, p->perf_counter[i]); + else if (pp->type == COUNTER_K2M) + outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000); + else if (pp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE) + outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]); + else if (pp->format == FORMAT_PERCENT) + outp += print_float_value(&printed, delim, pct(p->perf_counter[i] / tsc)); } + /* Added PMT Package Counters */ for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) { const unsigned long value_raw = p->pmt_counter[i]; double value_converted; switch (ppmt->type) { case PMT_TYPE_RAW: - if (pmt_counter_get_width(ppmt) <= 32) - outp += sprintf(outp, "%s0x%08x", (printed++ ? delim : ""), - (unsigned int)p->pmt_counter[i]); - else - outp += sprintf(outp, "%s0x%016llx", (printed++ ? delim : ""), p->pmt_counter[i]); - + outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, p->pmt_counter[i]); break; case PMT_TYPE_XTAL_TIME: - value_converted = 100.0 * value_raw / crystal_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / crystal_hz / interval_float); + outp += print_float_value(&printed, delim, value_converted); break; case PMT_TYPE_TCORE_CLOCK: - value_converted = 100.0 * value_raw / tcore_clock_freq_hz / interval_float; - outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), value_converted); + value_converted = pct(value_raw / tcore_clock_freq_hz / interval_float); + outp += print_float_value(&printed, delim, value_converted); } } @@ -3754,11 +3688,10 @@ int delta_package(struct pkg_data *new, struct pkg_data *old) old->energy_gfx.raw_value = new->energy_gfx.raw_value - old->energy_gfx.raw_value; old->energy_dram.raw_value = new->energy_dram.raw_value - old->energy_dram.raw_value; old->rapl_pkg_perf_status.raw_value = new->rapl_pkg_perf_status.raw_value - old->rapl_pkg_perf_status.raw_value; - old->rapl_dram_perf_status.raw_value = - new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value; + old->rapl_dram_perf_status.raw_value = new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value; for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) { - if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) + if (mp->format == FORMAT_RAW) old->counter[i] = new->counter[i]; else if (mp->format == FORMAT_AVERAGE) old->counter[i] = new->counter[i]; @@ -3862,8 +3795,7 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d /* check for TSC < 1 Mcycles over interval */ if (old->tsc < (1000 * 1000)) errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n" - "You can disable all c-states by booting with \"idle=poll\"\n" - "or just the deep ones with \"processor.max_cstate=1\""); + "You can disable all c-states by booting with \"idle=poll\"\n" "or just the deep ones with \"processor.max_cstate=1\""); old->c1 = new->c1 - old->c1; @@ -3892,8 +3824,7 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d old->c1 = 0; else { /* normal case, derive c1 */ - old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - - core_delta->c6 - core_delta->c7; + old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - core_delta->c6 - core_delta->c7; } } @@ -3915,6 +3846,12 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d if (DO_BIC(BIC_SMI)) old->smi_count = new->smi_count - old->smi_count; + if (DO_BIC(BIC_LLC_RPS)) + old->llc.references = new->llc.references - old->llc.references; + + if (DO_BIC(BIC_LLC_HIT)) + old->llc.misses = new->llc.misses - old->llc.misses; + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE) old->counter[i] = new->counter[i]; @@ -3939,20 +3876,19 @@ int delta_thread(struct thread_data *new, struct thread_data *old, struct core_d return 0; } -int delta_cpu(struct thread_data *t, struct core_data *c, - struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2) +int delta_cpu(struct thread_data *t, struct core_data *c, struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2) { int retval = 0; /* calculate core delta only for 1st thread in core */ - if (is_cpu_first_thread_in_core(t, c, p)) + if (is_cpu_first_thread_in_core(t, c)) delta_core(c, c2); /* always calculate thread delta */ retval = delta_thread(t, t2, c2); /* c2 is core delta */ /* calculate package delta only for 1st core in package */ - if (is_cpu_first_core_in_package(t, c, p)) + if (is_cpu_first_core_in_package(t, p)) retval |= delta_package(p, p2); return retval; @@ -3993,6 +3929,9 @@ void clear_counters(PER_THREAD_PARAMS) t->nmi_count = 0; t->smi_count = 0; + t->llc.references = 0; + t->llc.misses = 0; + c->c3 = 0; c->c6 = 0; c->c7 = 0; @@ -4001,6 +3940,9 @@ void clear_counters(PER_THREAD_PARAMS) rapl_counter_clear(&c->core_energy); c->core_throt_cnt = 0; + t->llc.references = 0; + t->llc.misses = 0; + p->pkg_wtd_core_c0 = 0; p->pkg_any_core_c0 = 0; p->pkg_any_gfxe_c0 = 0; @@ -4098,6 +4040,9 @@ int sum_counters(PER_THREAD_PARAMS) average.threads.nmi_count += t->nmi_count; average.threads.smi_count += t->smi_count; + average.threads.llc.references += t->llc.references; + average.threads.llc.misses += t->llc.misses; + for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) { if (mp->format == FORMAT_RAW) continue; @@ -4115,7 +4060,7 @@ int sum_counters(PER_THREAD_PARAMS) } /* sum per-core values only for 1st thread in core */ - if (!is_cpu_first_thread_in_core(t, c, p)) + if (!is_cpu_first_thread_in_core(t, c)) return 0; average.cores.c3 += c->c3; @@ -4145,7 +4090,7 @@ int sum_counters(PER_THREAD_PARAMS) } /* sum per-pkg values only for 1st core in pkg */ - if (!is_cpu_first_core_in_package(t, c, p)) + if (!is_cpu_first_core_in_package(t, p)) return 0; if (DO_BIC(BIC_Totl_c0)) @@ -4411,8 +4356,7 @@ unsigned long long get_legacy_uncore_mhz(int package) */ for (die = 0; die <= topo.max_die_id; ++die) { - sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", - package, die); + sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", package, die); if (access(path, R_OK) == 0) return (snapshot_sysfs_counter(path) / 1000); @@ -4523,11 +4467,6 @@ int get_core_throt_cnt(int cpu, unsigned long long *cnt) return 0; } -struct amperf_group_fd { - int aperf; /* Also the group descriptor */ - int mperf; -}; - static int read_perf_counter_info(const char *const path, const char *const parse_format, void *value_ptr) { int fdmt; @@ -4727,8 +4666,7 @@ int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct const ssize_t actual_read_size = read(rci->fd_perf, &perf_data[0], sizeof(perf_data)); if (actual_read_size != expected_read_size) - err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, - actual_read_size); + err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size); } for (unsigned int i = 0, pi = 1; i < NUM_RAPL_COUNTERS; ++i) { @@ -4966,8 +4904,7 @@ int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t) const ssize_t actual_read_size = read(mci->fd_perf, &perf_data[0], sizeof(perf_data)); if (actual_read_size != expected_read_size) - err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, - actual_read_size); + err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size); } for (unsigned int i = 0, pi = 1; i < NUM_MSR_COUNTERS; ++i) { @@ -5121,6 +5058,9 @@ int get_counters(PER_THREAD_PARAMS) get_smi_aperf_mperf(cpu, t); + if (DO_BIC(BIC_LLC_RPS) || DO_BIC(BIC_LLC_HIT)) + get_perf_llc_stats(cpu, &t->llc); + if (DO_BIC(BIC_IPC)) if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long)) return -4; @@ -5144,7 +5084,7 @@ int get_counters(PER_THREAD_PARAMS) t->pmt_counter[i] = pmt_read_counter(pp, t->cpu_id); /* collect core counters only for 1st thread in core */ - if (!is_cpu_first_thread_in_core(t, c, p)) + if (!is_cpu_first_thread_in_core(t, c)) goto done; if (platform->has_per_core_rapl) { @@ -5188,7 +5128,7 @@ int get_counters(PER_THREAD_PARAMS) c->pmt_counter[i] = pmt_read_counter(pp, c->core_id); /* collect package counters only for 1st core in package */ - if (!is_cpu_first_core_in_package(t, c, p)) + if (!is_cpu_first_core_in_package(t, p)) goto done; if (DO_BIC(BIC_Totl_c0)) { @@ -5277,48 +5217,39 @@ char *pkg_cstate_limit_strings[] = { "unknown", "reserved", "pc0", "pc1", "pc2", "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited" }; -int nhm_pkg_cstate_limits[16] = - { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int nhm_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int snb_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int snb_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int hsw_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int hsw_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int slv_pkg_cstate_limits[16] = - { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int slv_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7 }; -int amt_pkg_cstate_limits[16] = - { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int amt_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int phi_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int phi_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int glm_pkg_cstate_limits[16] = - { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int glm_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int skx_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int skx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; -int icx_pkg_cstate_limits[16] = - { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, +int icx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV }; @@ -5393,8 +5324,7 @@ static void dump_power_ctl(void) return; get_msr(base_cpu, MSR_IA32_POWER_CTL, &msr); - fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", - base_cpu, msr, msr & 0x2 ? "EN" : "DIS"); + fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", base_cpu, msr, msr & 0x2 ? "EN" : "DIS"); /* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */ if (platform->has_cst_prewake_bit) @@ -5487,8 +5417,7 @@ static void dump_turbo_ratio_limits(int trl_msr_offset) ratio = (msr >> shift) & 0xFF; group_size = (core_counts >> shift) & 0xFF; if (ratio) - fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", - ratio, bclk, ratio * bclk, group_size); + fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio, bclk, ratio * bclk, group_size); } return; @@ -5586,9 +5515,7 @@ static void dump_knl_turbo_ratio_limits(void) for (i = buckets_no - 1; i >= 0; i--) if (i > 0 ? ratio[i] != ratio[i - 1] : 1) - fprintf(outf, - "%d * %.1f = %.1f MHz max turbo %d active cores\n", - ratio[i], bclk, ratio[i] * bclk, cores[i]); + fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio[i], bclk, ratio[i] * bclk, cores[i]); } static void dump_cst_cfg(void) @@ -5673,43 +5600,37 @@ void print_irtl(void) if (platform->supported_cstates & PC3) { get_msr(base_cpu, MSR_PKGC3_IRTL, &msr); fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC6) { get_msr(base_cpu, MSR_PKGC6_IRTL, &msr); fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC7) { get_msr(base_cpu, MSR_PKGC7_IRTL, &msr); fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC8) { get_msr(base_cpu, MSR_PKGC8_IRTL, &msr); fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC9) { get_msr(base_cpu, MSR_PKGC9_IRTL, &msr); fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } if (platform->supported_cstates & PC10) { get_msr(base_cpu, MSR_PKGC10_IRTL, &msr); fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", base_cpu, msr); - fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", - (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); + fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]); } } @@ -5743,6 +5664,20 @@ void free_fd_instr_count_percpu(void) fd_instr_count_percpu = NULL; } +void free_fd_llc_percpu(void) +{ + if (!fd_llc_percpu) + return; + + for (int i = 0; i < topo.max_cpu_num + 1; ++i) { + if (fd_llc_percpu[i] != 0) + close(fd_llc_percpu[i]); + } + + free(fd_llc_percpu); + fd_llc_percpu = NULL; +} + void free_fd_cstate(void) { if (!ccstate_counter_info) @@ -5867,6 +5802,7 @@ void free_all_buffers(void) free_fd_percpu(); free_fd_instr_count_percpu(); + free_fd_llc_percpu(); free_fd_msr(); free_fd_rapl_percpu(); free_fd_cstate(); @@ -6213,6 +6149,7 @@ void linux_perf_init(void); void msr_perf_init(void); void rapl_perf_init(void); void cstate_perf_init(void); +void perf_llc_init(void); void added_perf_counters_init(void); void pmt_init(void); @@ -6224,10 +6161,10 @@ void re_initialize(void) msr_perf_init(); rapl_perf_init(); cstate_perf_init(); + perf_llc_init(); added_perf_counters_init(); pmt_init(); - fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, - topo.allowed_cpus); + fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus); } void set_max_cpu_num(void) @@ -6673,6 +6610,7 @@ release_timer: timer_delete(timerid); release_msr: free(per_cpu_msr_sum); + per_cpu_msr_sum = NULL; } /* @@ -6797,21 +6735,43 @@ restart: } } -void check_dev_msr() +int probe_dev_msr(void) { struct stat sb; char pathname[32]; - if (no_msr) - return; -#if defined(ANDROID) sprintf(pathname, "/dev/msr%d", base_cpu); -#else + return !stat(pathname, &sb); +} + +int probe_dev_cpu_msr(void) +{ + struct stat sb; + char pathname[32]; + sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); -#endif - if (stat(pathname, &sb)) - if (system("/sbin/modprobe msr > /dev/null 2>&1")) - no_msr = 1; + return !stat(pathname, &sb); +} + +int probe_msr_driver(void) +{ + if (probe_dev_msr()) { + use_android_msr_path = 1; + return 1; + } + return probe_dev_cpu_msr(); +} + +void check_msr_driver(void) +{ + if (probe_msr_driver()) + return; + + if (system("/sbin/modprobe msr > /dev/null 2>&1")) + no_msr = 1; + + if (!probe_msr_driver()) + no_msr = 1; } /* @@ -6866,11 +6826,7 @@ void check_msr_permission(void) failed += check_for_cap_sys_rawio(); /* test file permissions */ -#if defined(ANDROID) - sprintf(pathname, "/dev/msr%d", base_cpu); -#else - sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); -#endif + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", base_cpu); if (euidaccess(pathname, R_OK)) { failed++; } @@ -6999,8 +6955,7 @@ static void probe_intel_uncore_frequency_legacy(void) int k, l; char path_base[128]; - sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, - j); + sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, j); sprintf(path, "%s/current_freq_khz", path_base); if (access(path, R_OK)) @@ -7083,8 +7038,7 @@ static void probe_intel_uncore_frequency_cluster(void) */ if BIC_IS_ENABLED (BIC_UNCORE_MHZ) - add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, - package_id); + add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, package_id); if (quiet) continue; @@ -7093,8 +7047,7 @@ static void probe_intel_uncore_frequency_cluster(void) k = read_sysfs_int(path); sprintf(path, "%s/max_freq_khz", path_base); l = read_sysfs_int(path); - fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", package_id, domain_id, - cluster_id, k / 1000, l / 1000); + fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", package_id, domain_id, cluster_id, k / 1000, l / 1000); sprintf(path, "%s/initial_min_freq_khz", path_base); k = read_sysfs_int(path); @@ -7156,21 +7109,17 @@ static void probe_graphics(void) else goto next; - set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", - gt0_is_gt ? GFX_rc6 : SAM_mc6); + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", gt0_is_gt ? GFX_rc6 : SAM_mc6); set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/cur_freq", gt0_is_gt ? GFX_MHz : SAM_MHz); - set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/act_freq", - gt0_is_gt ? GFX_ACTMHz : SAM_ACTMHz); + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/act_freq", gt0_is_gt ? GFX_ACTMHz : SAM_ACTMHz); - set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/gtidle/idle_residency_ms", - gt0_is_gt ? SAM_mc6 : GFX_rc6); + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/gtidle/idle_residency_ms", gt0_is_gt ? SAM_mc6 : GFX_rc6); set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/cur_freq", gt0_is_gt ? SAM_MHz : GFX_MHz); - set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/act_freq", - gt0_is_gt ? SAM_ACTMHz : GFX_ACTMHz); + set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/act_freq", gt0_is_gt ? SAM_ACTMHz : GFX_ACTMHz); goto end; } @@ -7425,8 +7374,7 @@ int print_hwp(PER_THREAD_PARAMS) "(high %d guar %d eff %d low %d)\n", cpu, msr, (unsigned int)HWP_HIGHEST_PERF(msr), - (unsigned int)HWP_GUARANTEED_PERF(msr), - (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr)); + (unsigned int)HWP_GUARANTEED_PERF(msr), (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr)); if (get_msr(cpu, MSR_HWP_REQUEST, &msr)) return 0; @@ -7437,8 +7385,7 @@ int print_hwp(PER_THREAD_PARAMS) (unsigned int)(((msr) >> 0) & 0xff), (unsigned int)(((msr) >> 8) & 0xff), (unsigned int)(((msr) >> 16) & 0xff), - (unsigned int)(((msr) >> 24) & 0xff), - (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1)); + (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1)); if (has_hwp_pkg) { if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr)) @@ -7449,23 +7396,20 @@ int print_hwp(PER_THREAD_PARAMS) cpu, msr, (unsigned int)(((msr) >> 0) & 0xff), (unsigned int)(((msr) >> 8) & 0xff), - (unsigned int)(((msr) >> 16) & 0xff), - (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3)); + (unsigned int)(((msr) >> 16) & 0xff), (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3)); } if (has_hwp_notify) { if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr)) return 0; fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx " - "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", - cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis"); + "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis"); } if (get_msr(cpu, MSR_HWP_STATUS, &msr)) return 0; fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx " - "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", - cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-"); + "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-"); return 0; } @@ -7510,8 +7454,7 @@ int print_perf_limit(PER_THREAD_PARAMS) (msr & 1 << 6) ? "VR-Therm, " : "", (msr & 1 << 5) ? "Auto-HWP, " : "", (msr & 1 << 4) ? "Graphics, " : "", - (msr & 1 << 2) ? "bit2, " : "", - (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : ""); + (msr & 1 << 2) ? "bit2, " : "", (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : ""); fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n", (msr & 1 << 31) ? "bit31, " : "", (msr & 1 << 30) ? "bit30, " : "", @@ -7524,8 +7467,7 @@ int print_perf_limit(PER_THREAD_PARAMS) (msr & 1 << 22) ? "VR-Therm, " : "", (msr & 1 << 21) ? "Auto-HWP, " : "", (msr & 1 << 20) ? "Graphics, " : "", - (msr & 1 << 18) ? "bit18, " : "", - (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : ""); + (msr & 1 << 18) ? "bit18, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : ""); } if (platform->plr_msrs & PLR_GFX) { @@ -7537,16 +7479,14 @@ int print_perf_limit(PER_THREAD_PARAMS) (msr & 1 << 4) ? "Graphics, " : "", (msr & 1 << 6) ? "VR-Therm, " : "", (msr & 1 << 8) ? "Amps, " : "", - (msr & 1 << 9) ? "GFXPwr, " : "", - (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); + (msr & 1 << 9) ? "GFXPwr, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s)\n", (msr & 1 << 16) ? "PROCHOT, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 20) ? "Graphics, " : "", (msr & 1 << 22) ? "VR-Therm, " : "", (msr & 1 << 24) ? "Amps, " : "", - (msr & 1 << 25) ? "GFXPwr, " : "", - (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); + (msr & 1 << 25) ? "GFXPwr, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); } if (platform->plr_msrs & PLR_RING) { get_msr(cpu, MSR_RING_PERF_LIMIT_REASONS, &msr); @@ -7555,14 +7495,12 @@ int print_perf_limit(PER_THREAD_PARAMS) (msr & 1 << 0) ? "PROCHOT, " : "", (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 6) ? "VR-Therm, " : "", - (msr & 1 << 8) ? "Amps, " : "", - (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); + (msr & 1 << 8) ? "Amps, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : ""); fprintf(outf, " (Logged: %s%s%s%s%s%s)\n", (msr & 1 << 16) ? "PROCHOT, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 22) ? "VR-Therm, " : "", - (msr & 1 << 24) ? "Amps, " : "", - (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); + (msr & 1 << 24) ? "Amps, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : ""); } return 0; } @@ -7582,7 +7520,7 @@ double get_tdp_intel(void) { unsigned long long msr; - if (platform->rapl_msrs & RAPL_PKG_POWER_INFO) + if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) if (!get_msr(base_cpu, MSR_PKG_POWER_INFO, &msr)) return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units; return get_quirk_tdp(); @@ -7613,12 +7551,12 @@ void rapl_probe_intel(void) CLR_BIC(BIC_GFX_J, &bic_enabled); } - if (!platform->rapl_msrs || no_msr) + if (!valid_rapl_msrs || no_msr) return; - if (!(platform->rapl_msrs & RAPL_PKG_PERF_STATUS)) + if (!(valid_rapl_msrs & RAPL_PKG_PERF_STATUS)) CLR_BIC(BIC_PKG__, &bic_enabled); - if (!(platform->rapl_msrs & RAPL_DRAM_PERF_STATUS)) + if (!(valid_rapl_msrs & RAPL_DRAM_PERF_STATUS)) CLR_BIC(BIC_RAM__, &bic_enabled); /* units on package 0, verify later other packages match */ @@ -7667,7 +7605,7 @@ void rapl_probe_amd(void) CLR_BIC(BIC_Cor_J, &bic_enabled); } - if (!platform->rapl_msrs || no_msr) + if (!valid_rapl_msrs || no_msr) return; if (get_msr(base_cpu, MSR_RAPL_PWR_UNIT, &msr)) @@ -7690,8 +7628,7 @@ void print_power_limit_msr(int cpu, unsigned long long msr, char *label) cpu, label, ((msr >> 15) & 1) ? "EN" : "DIS", ((msr >> 0) & 0x7FFF) * rapl_power_units, - (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, - (((msr >> 16) & 1) ? "EN" : "DIS")); + (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, (((msr >> 16) & 1) ? "EN" : "DIS")); return; } @@ -7857,7 +7794,7 @@ int print_rapl(PER_THREAD_PARAMS) UNUSED(c); UNUSED(p); - if (!platform->rapl_msrs) + if (!valid_rapl_msrs) return 0; /* RAPL counters are per package, so print only for 1st thread/package */ @@ -7870,7 +7807,7 @@ int print_rapl(PER_THREAD_PARAMS) return -1; } - if (platform->rapl_msrs & RAPL_AMD_F17H) { + if (valid_rapl_msrs & RAPL_AMD_F17H) { msr_name = "MSR_RAPL_PWR_UNIT"; if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr)) return -1; @@ -7883,7 +7820,7 @@ int print_rapl(PER_THREAD_PARAMS) fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, rapl_power_units, rapl_energy_units, rapl_time_units); - if (platform->rapl_msrs & RAPL_PKG_POWER_INFO) { + if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) { if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr)) return -5; @@ -7892,25 +7829,22 @@ int print_rapl(PER_THREAD_PARAMS) cpu, msr, ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); + ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); } - if (platform->rapl_msrs & RAPL_PKG) { + if (valid_rapl_msrs & RAPL_PKG) { if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr)) return -9; - fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 63) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 63) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "PKG Limit #1"); fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%0.3f Watts, %f* sec, clamp %sabled)\n", cpu, ((msr >> 47) & 1) ? "EN" : "DIS", ((msr >> 32) & 0x7FFF) * rapl_power_units, - (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, - ((msr >> 48) & 1) ? "EN" : "DIS"); + (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, ((msr >> 48) & 1) ? "EN" : "DIS"); if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr)) return -9; @@ -7920,7 +7854,7 @@ int print_rapl(PER_THREAD_PARAMS) cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN"); } - if (platform->rapl_msrs & RAPL_DRAM_POWER_INFO) { + if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) { if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr)) return -6; @@ -7928,31 +7862,28 @@ int print_rapl(PER_THREAD_PARAMS) cpu, msr, ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, - ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); + ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units); } - if (platform->rapl_msrs & RAPL_DRAM) { + if (valid_rapl_msrs & RAPL_DRAM) { if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr)) return -9; - fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 31) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "DRAM Limit"); } - if (platform->rapl_msrs & RAPL_CORE_POLICY) { + if (valid_rapl_msrs & RAPL_CORE_POLICY) { if (get_msr(cpu, MSR_PP0_POLICY, &msr)) return -7; fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF); } - if (platform->rapl_msrs & RAPL_CORE_POWER_LIMIT) { + if (valid_rapl_msrs & RAPL_CORE_POWER_LIMIT) { if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr)) return -9; - fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 31) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "Cores Limit"); } - if (platform->rapl_msrs & RAPL_GFX) { + if (valid_rapl_msrs & RAPL_GFX) { if (get_msr(cpu, MSR_PP1_POLICY, &msr)) return -8; @@ -7960,20 +7891,58 @@ int print_rapl(PER_THREAD_PARAMS) if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr)) return -9; - fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", - cpu, msr, (msr >> 31) & 1 ? "" : "UN"); + fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN"); print_power_limit_msr(cpu, msr, "GFX Limit"); } return 0; } /* + * probe_rapl_msrs + * + * initialize global valid_rapl_msrs to platform->plat_rapl_msrs + * only if PKG_ENERGY counter is enumerated and reads non-zero + */ +void probe_rapl_msrs(void) +{ + int ret; + off_t offset; + unsigned long long msr_value; + + if (no_msr) + return; + + if ((platform->plat_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H)) == 0) + return; + + offset = idx_to_offset(IDX_PKG_ENERGY); + if (offset < 0) + return; + + ret = get_msr(base_cpu, offset, &msr_value); + if (ret) { + if (debug) + fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset); + return; + } + if (msr_value == 0) { + if (debug) + fprintf(outf, "RAPL_PKG_ENERGY MSR(0x%llx) == ZERO: disabling all RAPL MSRs\n", (unsigned long long)offset); + return; + } + + valid_rapl_msrs = platform->plat_rapl_msrs; /* success */ +} + +/* * probe_rapl() * * sets rapl_power_units, rapl_energy_units, rapl_time_units */ void probe_rapl(void) { + probe_rapl_msrs(); + if (genuine_intel) rapl_probe_intel(); if (authentic_amd || hygon_genuine) @@ -7984,7 +7953,7 @@ void probe_rapl(void) print_rapl_sysfs(); - if (!platform->rapl_msrs || no_msr) + if (!valid_rapl_msrs || no_msr) return; for_all_cpus(print_rapl, ODD_COUNTERS); @@ -8088,7 +8057,7 @@ int print_thermal(PER_THREAD_PARAMS) cpu = t->cpu_id; /* DTS is per-core, no need to print for each thread */ - if (!is_cpu_first_thread_in_core(t, c, p)) + if (!is_cpu_first_thread_in_core(t, c)) return 0; if (cpu_migrate(cpu)) { @@ -8096,7 +8065,7 @@ int print_thermal(PER_THREAD_PARAMS) return -1; } - if (do_ptm && is_cpu_first_core_in_package(t, c, p)) { + if (do_ptm && is_cpu_first_core_in_package(t, p)) { if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr)) return 0; @@ -8108,8 +8077,7 @@ int print_thermal(PER_THREAD_PARAMS) dts = (msr >> 16) & 0x7F; dts2 = (msr >> 8) & 0x7F; - fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", - cpu, msr, tj_max - dts, tj_max - dts2); + fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2); } if (do_dts && debug) { @@ -8120,16 +8088,14 @@ int print_thermal(PER_THREAD_PARAMS) dts = (msr >> 16) & 0x7F; resolution = (msr >> 27) & 0xF; - fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", - cpu, msr, tj_max - dts, resolution); + fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", cpu, msr, tj_max - dts, resolution); if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr)) return 0; dts = (msr >> 16) & 0x7F; dts2 = (msr >> 8) & 0x7F; - fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", - cpu, msr, tj_max - dts, tj_max - dts2); + fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2); } return 0; @@ -8203,8 +8169,7 @@ void decode_misc_enable_msr(void) msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-", msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-", msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-", - msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", - msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : ""); + msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : ""); } void decode_misc_feature_control(void) @@ -8243,8 +8208,7 @@ void decode_misc_pwr_mgmt_msr(void) if (!get_msr(base_cpu, MSR_MISC_PWR_MGMT, &msr)) fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n", - base_cpu, msr, - msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); + base_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS"); } /* @@ -8297,30 +8261,26 @@ void print_dev_latency(void) close(fd); } -static int has_instr_count_access(void) +static int has_perf_instr_count_access(void) { int fd; - int has_access; if (no_perf) return 0; fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0); - has_access = fd != -1; - if (fd != -1) close(fd); - if (!has_access) + if (fd == -1) warnx("Failed to access %s. Some of the counters may not be available\n" - "\tRun as root to enable them or use %s to disable the access explicitly", - "instructions retired perf counter", "--no-perf"); + "\tRun as root to enable them or use %s to disable the access explicitly", "perf instructions retired counter", + "'--hide IPC' or '--no-perf'"); - return has_access; + return (fd != -1); } -int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, - double *scale_, enum rapl_unit *unit_) +int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, double *scale_, enum rapl_unit *unit_) { int ret = -1; @@ -8370,11 +8330,16 @@ void linux_perf_init(void) if (access("/proc/sys/kernel/perf_event_paranoid", F_OK)) return; - if (BIC_IS_ENABLED(BIC_IPC) && has_aperf) { + if (BIC_IS_ENABLED(BIC_IPC) && cpuid_has_aperf_mperf) { fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); if (fd_instr_count_percpu == NULL) err(-1, "calloc fd_instr_count_percpu"); } + if (BIC_IS_ENABLED(BIC_LLC_RPS)) { + fd_llc_percpu = calloc(topo.max_cpu_num + 1, sizeof(int)); + if (fd_llc_percpu == NULL) + err(-1, "calloc fd_llc_percpu"); + } } void rapl_perf_init(void) @@ -8485,7 +8450,7 @@ void rapl_perf_init(void) /* Assumes msr_counter_info is populated */ static int has_amperf_access(void) { - return msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present && + return cpuid_has_aperf_mperf && msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present && msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present; } @@ -8708,8 +8673,7 @@ void cstate_perf_init_(bool soft_c1) cci->source[cai->rci_index] = COUNTER_SOURCE_PERF; /* User MSR for this counter */ - } else if (pkg_cstate_limit >= cai->pkg_cstate_limit - && add_msr_counter(cpu, cai->msr) >= 0) { + } else if (pkg_cstate_limit >= cai->pkg_cstate_limit && add_msr_counter(cpu, cai->msr) >= 0) { cci->source[cai->rci_index] = COUNTER_SOURCE_MSR; cci->msr[cai->rci_index] = cai->msr; } @@ -8827,8 +8791,7 @@ void process_cpuid() hygon_genuine = 1; if (!quiet) - fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", - (char *)&ebx, (char *)&edx, (char *)&ecx, max_level); + fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", (char *)&ebx, (char *)&edx, (char *)&ecx, max_level); __cpuid(1, fms, ebx, ecx, edx); family = (fms >> 8) & 0xf; @@ -8857,8 +8820,7 @@ void process_cpuid() __cpuid(0x80000000, max_extended_level, ebx, ecx, edx); if (!quiet) { - fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", - family, model, stepping, family, model, stepping); + fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", family, model, stepping, family, model, stepping); if (ucode_patch_valid) fprintf(outf, " microcode 0x%x", (unsigned int)((ucode_patch >> 32) & 0xFFFFFFFF)); fputc('\n', outf); @@ -8872,8 +8834,7 @@ void process_cpuid() ecx_flags & (1 << 8) ? "TM2" : "-", edx_flags & (1 << 4) ? "TSC" : "-", edx_flags & (1 << 5) ? "MSR" : "-", - edx_flags & (1 << 22) ? "ACPI-TM" : "-", - edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-"); + edx_flags & (1 << 22) ? "ACPI-TM" : "-", edx_flags & (1 << 28) ? "HT" : "-", edx_flags & (1 << 29) ? "TM" : "-"); } probe_platform_features(family, model); @@ -8897,7 +8858,7 @@ void process_cpuid() */ __cpuid(0x6, eax, ebx, ecx, edx); - has_aperf = ecx & (1 << 0); + cpuid_has_aperf_mperf = ecx & (1 << 0); do_dts = eax & (1 << 0); if (do_dts) BIC_PRESENT(BIC_CoreTmp); @@ -8915,14 +8876,13 @@ void process_cpuid() if (!quiet) fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, " "%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n", - has_aperf ? "" : "No-", + cpuid_has_aperf_mperf ? "" : "No-", has_turbo ? "" : "No-", do_dts ? "" : "No-", do_ptm ? "" : "No-", has_hwp ? "" : "No-", has_hwp_notify ? "" : "No-", - has_hwp_activity_window ? "" : "No-", - has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-"); + has_hwp_activity_window ? "" : "No-", has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-"); if (!quiet) decode_misc_enable_msr(); @@ -8956,8 +8916,7 @@ void process_cpuid() if (ebx_tsc != 0) { if (!quiet && (ebx != 0)) - fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", - eax_crystal, ebx_tsc, crystal_hz); + fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", eax_crystal, ebx_tsc, crystal_hz); if (crystal_hz == 0) crystal_hz = platform->crystal_freq; @@ -8989,11 +8948,10 @@ void process_cpuid() tsc_tweak = base_hz / tsc_hz; if (!quiet) - fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", - base_mhz, max_mhz, bus_mhz); + fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", base_mhz, max_mhz, bus_mhz); } - if (has_aperf) + if (cpuid_has_aperf_mperf) aperf_mperf_multiplier = platform->need_perf_multiplier ? 1024 : 1; BIC_PRESENT(BIC_IRQ); @@ -9045,6 +9003,62 @@ void probe_pm_features(void) decode_misc_feature_control(); } +/* perf_llc_probe + * + * return 1 on success, else 0 + */ +int has_perf_llc_access(void) +{ + int fd; + + if (no_perf) + return 0; + + fd = open_perf_counter(base_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP); + if (fd != -1) + close(fd); + + if (fd == -1) + warnx("Failed to access %s. Some of the counters may not be available\n" + "\tRun as root to enable them or use %s to disable the access explicitly", "perf LLC counters", "'--hide LLC' or '--no-perf'"); + + return (fd != -1); +} + +void perf_llc_init(void) +{ + int cpu; + int retval; + + if (no_perf) + return; + if (!(BIC_IS_ENABLED(BIC_LLC_RPS) && BIC_IS_ENABLED(BIC_LLC_HIT))) + return; + + for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) { + + if (cpu_is_not_allowed(cpu)) + continue; + + assert(fd_llc_percpu != 0); + fd_llc_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP); + if (fd_llc_percpu[cpu] == -1) { + warnx("%s: perf REFS: failed to open counter on cpu%d", __func__, cpu); + free_fd_llc_percpu(); + return; + } + assert(fd_llc_percpu != 0); + retval = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, fd_llc_percpu[cpu], PERF_FORMAT_GROUP); + if (retval == -1) { + warnx("%s: perf MISS: failed to open counter on cpu%d", __func__, cpu); + free_fd_llc_percpu(); + return; + } + } + BIC_PRESENT(BIC_LLC_RPS); + BIC_PRESENT(BIC_LLC_HIT); +} + /* * in /dev/cpu/ return success for names that are numbers * ie. filter out ".", "..", "microcode". @@ -9351,6 +9365,7 @@ void init_counter(struct thread_data *thread_base, struct core_data *core_base, t->cpu_id = cpu_id; if (!cpu_is_not_allowed(cpu_id)) { + if (c->base_cpu < 0) c->base_cpu = t->cpu_id; if (pkg_base[pkg_id].base_cpu < 0) @@ -9456,7 +9471,7 @@ bool has_added_counters(void) void check_msr_access(void) { - check_dev_msr(); + check_msr_driver(); check_msr_permission(); if (no_msr) @@ -9465,8 +9480,16 @@ void check_msr_access(void) void check_perf_access(void) { - if (no_perf || !BIC_IS_ENABLED(BIC_IPC) || !has_instr_count_access()) - CLR_BIC(BIC_IPC, &bic_enabled); + if (BIC_IS_ENABLED(BIC_IPC)) + if (!has_perf_instr_count_access()) + no_perf = 1; + + if (BIC_IS_ENABLED(BIC_LLC_RPS) || BIC_IS_ENABLED(BIC_LLC_HIT)) + if (!has_perf_llc_access()) + no_perf = 1; + + if (no_perf) + bic_disable_perf_access(); } bool perf_has_hybrid_devices(void) @@ -9589,8 +9612,7 @@ int added_perf_counters_init_(struct perf_counter_info *pinfo) perf_config = read_perf_config(perf_device, pinfo->event); if (perf_config == (unsigned int)-1) { - warnx("%s: perf/%s/%s: failed to read %s", - __func__, perf_device, pinfo->event, "config"); + warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "config"); continue; } @@ -9601,8 +9623,7 @@ int added_perf_counters_init_(struct perf_counter_info *pinfo) fd_perf = open_perf_counter(cpu, perf_type, perf_config, -1, 0); if (fd_perf == -1) { - warnx("%s: perf/%s/%s: failed to open counter on cpu%d", - __func__, perf_device, pinfo->event, cpu); + warnx("%s: perf/%s/%s: failed to open counter on cpu%d", __func__, perf_device, pinfo->event, cpu); continue; } @@ -9611,8 +9632,7 @@ int added_perf_counters_init_(struct perf_counter_info *pinfo) pinfo->scale = perf_scale; if (debug) - fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n", - perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]); + fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n", perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]); } pinfo = pinfo->next; @@ -9926,8 +9946,7 @@ int pmt_add_counter(unsigned int guid, unsigned int seq, const char *name, enum } if (conflict) { - fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n", - __func__, name); + fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n", __func__, name); exit(1); } @@ -9970,8 +9989,7 @@ void pmt_init(void) * CWF with newer firmware might require a PMT_TYPE_XTAL_TIME intead of PMT_TYPE_TCORE_CLOCK. */ pmt_add_counter(PMT_CWF_MC1E_GUID, seq, "CPU%c1e", PMT_TYPE_TCORE_CLOCK, - PMT_COUNTER_CWF_MC1E_LSB, PMT_COUNTER_CWF_MC1E_MSB, offset, SCOPE_CPU, - FORMAT_DELTA, cpu_num, PMT_OPEN_TRY); + PMT_COUNTER_CWF_MC1E_LSB, PMT_COUNTER_CWF_MC1E_MSB, offset, SCOPE_CPU, FORMAT_DELTA, cpu_num, PMT_OPEN_TRY); /* * Rather complex logic for each time we go to the next loop iteration, @@ -10021,6 +10039,7 @@ void turbostat_init() linux_perf_init(); rapl_perf_init(); cstate_perf_init(); + perf_llc_init(); added_perf_counters_init(); pmt_init(); @@ -10126,7 +10145,7 @@ int get_and_dump_counters(void) void print_version() { - fprintf(outf, "turbostat version 2025.09.09 - Len Brown <lenb@kernel.org>\n"); + fprintf(outf, "turbostat version 2025.12.02 - Len Brown <lenb@kernel.org>\n"); } #define COMMAND_LINE_SIZE 2048 @@ -10166,8 +10185,7 @@ struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name) } int add_counter(unsigned int msr_num, char *path, char *name, - unsigned int width, enum counter_scope scope, - enum counter_type type, enum counter_format format, int flags, int id) + unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int id) { struct msr_counter *msrp; @@ -10276,9 +10294,7 @@ int add_counter(unsigned int msr_num, char *path, char *name, struct perf_counter_info *make_perf_counter_info(const char *perf_device, const char *perf_event, const char *name, - unsigned int width, - enum counter_scope scope, - enum counter_type type, enum counter_format format) + unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format) { struct perf_counter_info *pinfo; @@ -10353,8 +10369,7 @@ int add_perf_counter(const char *perf_device, const char *perf_event, const char // FIXME: we might not have debug here yet if (debug) - fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n", - __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope); + fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n", __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope); return 0; } @@ -10523,8 +10538,7 @@ int pmt_parse_from_path(const char *target_path, unsigned int *out_guid, unsigne pmt_diriter_init(&pmt_iter); - for (dirname = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH); dirname != NULL; - dirname = pmt_diriter_next(&pmt_iter)) { + for (dirname = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH); dirname != NULL; dirname = pmt_diriter_next(&pmt_iter)) { fd_telem_dir = openat(dirfd(pmt_iter.dir), dirname->d_name, O_RDONLY | O_DIRECTORY); if (fd_telem_dir == -1) @@ -10536,8 +10550,7 @@ int pmt_parse_from_path(const char *target_path, unsigned int *out_guid, unsigne } if (fstat(fd_telem_dir, &stat) == -1) { - fprintf(stderr, "%s: Failed to stat %s directory: %s", __func__, - dirname->d_name, strerror(errno)); + fprintf(stderr, "%s: Failed to stat %s directory: %s", __func__, dirname->d_name, strerror(errno)); continue; } @@ -10633,8 +10646,7 @@ void parse_add_command_pmt(char *add_command) } if (!has_scope) { - printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n", - __func__); + printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n", __func__); exit(1); } @@ -10710,8 +10722,7 @@ next: } if (!has_format) { - fprintf(stderr, "%s: Invalid format %s. Expected raw, average or delta\n", - __func__, format_name); + fprintf(stderr, "%s: Invalid format %s. Expected raw, average or delta\n", __func__, format_name); exit(1); } } @@ -10878,7 +10889,7 @@ void probe_cpuidle_residency(void) if (is_deferred_skip(name_buf)) continue; - add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0); + add_counter(0, path, name_buf, 32, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0); if (state > max_state) max_state = state; diff --git a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c index 884a4c746f32..ac37132207a4 100644 --- a/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c +++ b/tools/power/x86/x86_energy_perf_policy/x86_energy_perf_policy.c @@ -95,6 +95,8 @@ unsigned int bdx_highest_ratio; #define PATH_TO_CPU "/sys/devices/system/cpu/" #define SYSFS_PATH_MAX 255 +static int use_android_msr_path; + /* * maintain compatibility with original implementation, but don't document it: */ @@ -370,7 +372,7 @@ void validate_cpu_selected_set(void) for (cpu = 0; cpu <= max_cpu_num; ++cpu) { if (CPU_ISSET_S(cpu, cpu_setsize, cpu_selected_set)) if (!CPU_ISSET_S(cpu, cpu_setsize, cpu_present_set)) - errx(1, "Requested cpu% is not present", cpu); + errx(1, "Requested cpu%d is not present", cpu); } } @@ -518,7 +520,7 @@ void for_packages(unsigned long long pkg_set, int (func)(int)) void print_version(void) { - printf("x86_energy_perf_policy 2025.9.19 Len Brown <lenb@kernel.org>\n"); + printf("x86_energy_perf_policy 2025.11.22 Len Brown <lenb@kernel.org>\n"); } void cmdline(int argc, char **argv) @@ -660,6 +662,11 @@ void err_on_hypervisor(void) } flags = strstr(buffer, "flags"); + if (!flags) { + fclose(cpuinfo); + free(buffer); + err(1, "Failed to find 'flags' in /proc/cpuinfo"); + } rewind(cpuinfo); fseek(cpuinfo, flags - buffer, SEEK_SET); if (!fgets(buffer, 4096, cpuinfo)) { @@ -684,10 +691,12 @@ int get_msr(int cpu, int offset, unsigned long long *msr) char pathname[32]; int fd; - sprintf(pathname, "/dev/cpu/%d/msr", cpu); + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu); fd = open(pathname, O_RDONLY); if (fd < 0) - err(-1, "%s open failed, try chown or chmod +r /dev/cpu/*/msr, or run as root", pathname); + err(-1, "%s open failed, try chown or chmod +r %s, or run as root", + pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr"); + retval = pread(fd, msr, sizeof(*msr), offset); if (retval != sizeof(*msr)) { @@ -708,10 +717,11 @@ int put_msr(int cpu, int offset, unsigned long long new_msr) int retval; int fd; - sprintf(pathname, "/dev/cpu/%d/msr", cpu); + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu); fd = open(pathname, O_RDWR); if (fd < 0) - err(-1, "%s open failed, try chown or chmod +r /dev/cpu/*/msr, or run as root", pathname); + err(-1, "%s open failed, try chown or chmod +r %s, or run as root", + pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr"); retval = pwrite(fd, &new_msr, sizeof(new_msr), offset); if (retval != sizeof(new_msr)) @@ -1421,16 +1431,32 @@ void set_base_cpu(void) err(-ENODEV, "No valid cpus found"); } +static void probe_android_msr_path(void) +{ + struct stat sb; + char test_path[32]; + + sprintf(test_path, "/dev/msr%d", base_cpu); + if (stat(test_path, &sb) == 0) + use_android_msr_path = 1; +} void probe_dev_msr(void) { struct stat sb; char pathname[32]; - sprintf(pathname, "/dev/cpu/%d/msr", base_cpu); - if (stat(pathname, &sb)) - if (system("/sbin/modprobe msr > /dev/null 2>&1")) - err(-5, "no /dev/cpu/0/msr, Try \"# modprobe msr\" "); + probe_android_msr_path(); + + sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", base_cpu); + if (stat(pathname, &sb)) { + if (system("/sbin/modprobe msr > /dev/null 2>&1")) { + if (use_android_msr_path) + err(-5, "no /dev/msr0, Try \"# modprobe msr\" "); + else + err(-5, "no /dev/cpu/0/msr, Try \"# modprobe msr\" "); + } + } } static void get_cpuid_or_exit(unsigned int leaf, @@ -1547,6 +1573,7 @@ void parse_cpuid(void) int main(int argc, char **argv) { set_base_cpu(); + probe_dev_msr(); init_data_structures(); diff --git a/tools/testing/nvdimm/test/nfit.c b/tools/testing/nvdimm/test/nfit.c index cfd4378e2129..f87e9f251d13 100644 --- a/tools/testing/nvdimm/test/nfit.c +++ b/tools/testing/nvdimm/test/nfit.c @@ -670,6 +670,7 @@ static int nfit_test_search_spa(struct nvdimm_bus *bus, .addr = spa->spa, .region = NULL, }; + struct nfit_mem *nfit_mem; u64 dpa; ret = device_for_each_child(&bus->dev, &ctx, @@ -687,8 +688,12 @@ static int nfit_test_search_spa(struct nvdimm_bus *bus, */ nd_mapping = &nd_region->mapping[nd_region->ndr_mappings - 1]; nvdimm = nd_mapping->nvdimm; + nfit_mem = nvdimm_provider_data(nvdimm); + if (!nfit_mem) + return -EINVAL; - spa->devices[0].nfit_device_handle = handle[nvdimm->id]; + spa->devices[0].nfit_device_handle = + __to_nfit_memdev(nfit_mem)->device_handle; spa->num_nvdimms = 1; spa->devices[0].dpa = dpa; diff --git a/tools/testing/selftests/Makefile b/tools/testing/selftests/Makefile index c46ebdb9b8ef..56e44a98d6a5 100644 --- a/tools/testing/selftests/Makefile +++ b/tools/testing/selftests/Makefile @@ -54,6 +54,7 @@ TARGETS += kvm TARGETS += landlock TARGETS += lib TARGETS += livepatch +TARGETS += liveupdate TARGETS += lkdtm TARGETS += lsm TARGETS += membarrier diff --git a/tools/testing/selftests/acct/acct_syscall.c b/tools/testing/selftests/acct/acct_syscall.c index 87c044fb9293..421adbdc299d 100644 --- a/tools/testing/selftests/acct/acct_syscall.c +++ b/tools/testing/selftests/acct/acct_syscall.c @@ -9,7 +9,7 @@ #include <string.h> #include <sys/wait.h> -#include "../kselftest.h" +#include "kselftest.h" int main(void) { diff --git a/tools/testing/selftests/alsa/conf.c b/tools/testing/selftests/alsa/conf.c index dc7f40e68dee..317212078e36 100644 --- a/tools/testing/selftests/alsa/conf.c +++ b/tools/testing/selftests/alsa/conf.c @@ -14,7 +14,7 @@ #include <regex.h> #include <sys/stat.h> -#include "../kselftest.h" +#include "kselftest.h" #include "alsa-local.h" #define SYSFS_ROOT "/sys" diff --git a/tools/testing/selftests/alsa/mixer-test.c b/tools/testing/selftests/alsa/mixer-test.c index e113dafa5c24..d4f845c32804 100644 --- a/tools/testing/selftests/alsa/mixer-test.c +++ b/tools/testing/selftests/alsa/mixer-test.c @@ -25,7 +25,7 @@ #include <poll.h> #include <stdint.h> -#include "../kselftest.h" +#include "kselftest.h" #include "alsa-local.h" #define TESTS_PER_CONTROL 7 diff --git a/tools/testing/selftests/alsa/pcm-test.c b/tools/testing/selftests/alsa/pcm-test.c index ce92548670c8..ee04ccef7d7c 100644 --- a/tools/testing/selftests/alsa/pcm-test.c +++ b/tools/testing/selftests/alsa/pcm-test.c @@ -17,7 +17,7 @@ #include <assert.h> #include <pthread.h> -#include "../kselftest.h" +#include "kselftest.h" #include "alsa-local.h" typedef struct timespec timestamp_t; diff --git a/tools/testing/selftests/alsa/test-pcmtest-driver.c b/tools/testing/selftests/alsa/test-pcmtest-driver.c index ca81afa4ee90..95065ef3b441 100644 --- a/tools/testing/selftests/alsa/test-pcmtest-driver.c +++ b/tools/testing/selftests/alsa/test-pcmtest-driver.c @@ -7,7 +7,7 @@ */ #include <string.h> #include <alsa/asoundlib.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define CH_NUM 4 diff --git a/tools/testing/selftests/alsa/utimer-test.c b/tools/testing/selftests/alsa/utimer-test.c index 37964f311a33..c45cb226bd8f 100644 --- a/tools/testing/selftests/alsa/utimer-test.c +++ b/tools/testing/selftests/alsa/utimer-test.c @@ -6,7 +6,7 @@ * * Author: Ivan Orlov <ivan.orlov0322@gmail.com> */ -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <sound/asound.h> #include <unistd.h> #include <fcntl.h> diff --git a/tools/testing/selftests/arm64/abi/hwcap.c b/tools/testing/selftests/arm64/abi/hwcap.c index 3b96d090c5eb..c41640f18e4e 100644 --- a/tools/testing/selftests/arm64/abi/hwcap.c +++ b/tools/testing/selftests/arm64/abi/hwcap.c @@ -19,7 +19,7 @@ #include <linux/auxvec.h> -#include "../../kselftest.h" +#include "kselftest.h" #define TESTS_PER_HWCAP 3 diff --git a/tools/testing/selftests/arm64/abi/ptrace.c b/tools/testing/selftests/arm64/abi/ptrace.c index b51d21f78cf9..0e46ac21c81d 100644 --- a/tools/testing/selftests/arm64/abi/ptrace.c +++ b/tools/testing/selftests/arm64/abi/ptrace.c @@ -18,7 +18,7 @@ #include <asm/sigcontext.h> #include <asm/ptrace.h> -#include "../../kselftest.h" +#include "kselftest.h" #define EXPECTED_TESTS 11 diff --git a/tools/testing/selftests/arm64/abi/syscall-abi.c b/tools/testing/selftests/arm64/abi/syscall-abi.c index 5ec9a18ec802..b67e3e26fa6d 100644 --- a/tools/testing/selftests/arm64/abi/syscall-abi.c +++ b/tools/testing/selftests/arm64/abi/syscall-abi.c @@ -16,7 +16,7 @@ #include <asm/sigcontext.h> #include <asm/unistd.h> -#include "../../kselftest.h" +#include "kselftest.h" #include "syscall-abi.h" diff --git a/tools/testing/selftests/arm64/fp/fp-ptrace.c b/tools/testing/selftests/arm64/fp/fp-ptrace.c index 0114108ab25f..22c584b78be5 100644 --- a/tools/testing/selftests/arm64/fp/fp-ptrace.c +++ b/tools/testing/selftests/arm64/fp/fp-ptrace.c @@ -27,7 +27,7 @@ #include <asm/sve_context.h> #include <asm/ptrace.h> -#include "../../kselftest.h" +#include "kselftest.h" #include "fp-ptrace.h" diff --git a/tools/testing/selftests/arm64/fp/fp-stress.c b/tools/testing/selftests/arm64/fp/fp-stress.c index 9349aa630c84..65e01aba96ff 100644 --- a/tools/testing/selftests/arm64/fp/fp-stress.c +++ b/tools/testing/selftests/arm64/fp/fp-stress.c @@ -24,7 +24,7 @@ #include <sys/wait.h> #include <asm/hwcap.h> -#include "../../kselftest.h" +#include "kselftest.h" #define MAX_VLS 16 diff --git a/tools/testing/selftests/arm64/fp/sve-probe-vls.c b/tools/testing/selftests/arm64/fp/sve-probe-vls.c index a24eca7a4ecb..df0c1b6eb114 100644 --- a/tools/testing/selftests/arm64/fp/sve-probe-vls.c +++ b/tools/testing/selftests/arm64/fp/sve-probe-vls.c @@ -12,7 +12,7 @@ #include <sys/prctl.h> #include <asm/sigcontext.h> -#include "../../kselftest.h" +#include "kselftest.h" #include "rdvl.h" int main(int argc, char **argv) diff --git a/tools/testing/selftests/arm64/fp/sve-ptrace.c b/tools/testing/selftests/arm64/fp/sve-ptrace.c index f44d44618575..28f6b996c5e2 100644 --- a/tools/testing/selftests/arm64/fp/sve-ptrace.c +++ b/tools/testing/selftests/arm64/fp/sve-ptrace.c @@ -19,7 +19,7 @@ #include <asm/sigcontext.h> #include <asm/ptrace.h> -#include "../../kselftest.h" +#include "kselftest.h" /* <linux/elf.h> and <sys/auxv.h> don't like each other, so: */ #ifndef NT_ARM_SVE diff --git a/tools/testing/selftests/arm64/fp/vec-syscfg.c b/tools/testing/selftests/arm64/fp/vec-syscfg.c index 2d75d342eeb9..8dd932fdcdc4 100644 --- a/tools/testing/selftests/arm64/fp/vec-syscfg.c +++ b/tools/testing/selftests/arm64/fp/vec-syscfg.c @@ -19,7 +19,7 @@ #include <asm/sigcontext.h> #include <asm/hwcap.h> -#include "../../kselftest.h" +#include "kselftest.h" #include "rdvl.h" #define ARCH_MIN_VL SVE_VL_MIN diff --git a/tools/testing/selftests/arm64/fp/za-ptrace.c b/tools/testing/selftests/arm64/fp/za-ptrace.c index 08c777f87ea2..787eed22d059 100644 --- a/tools/testing/selftests/arm64/fp/za-ptrace.c +++ b/tools/testing/selftests/arm64/fp/za-ptrace.c @@ -18,7 +18,7 @@ #include <asm/sigcontext.h> #include <asm/ptrace.h> -#include "../../kselftest.h" +#include "kselftest.h" /* <linux/elf.h> and <sys/auxv.h> don't like each other, so: */ #ifndef NT_ARM_ZA diff --git a/tools/testing/selftests/arm64/fp/zt-ptrace.c b/tools/testing/selftests/arm64/fp/zt-ptrace.c index a7f34040fbf1..f3fa49fd0fbd 100644 --- a/tools/testing/selftests/arm64/fp/zt-ptrace.c +++ b/tools/testing/selftests/arm64/fp/zt-ptrace.c @@ -18,7 +18,7 @@ #include <asm/sigcontext.h> #include <asm/ptrace.h> -#include "../../kselftest.h" +#include "kselftest.h" /* <linux/elf.h> and <sys/auxv.h> don't like each other, so: */ #ifndef NT_ARM_ZA diff --git a/tools/testing/selftests/arm64/gcs/gcs-stress.c b/tools/testing/selftests/arm64/gcs/gcs-stress.c index cf316d78ea97..86d8cd42aee7 100644 --- a/tools/testing/selftests/arm64/gcs/gcs-stress.c +++ b/tools/testing/selftests/arm64/gcs/gcs-stress.c @@ -24,7 +24,7 @@ #include <sys/wait.h> #include <asm/hwcap.h> -#include "../../kselftest.h" +#include "kselftest.h" struct child_data { char *name, *output; diff --git a/tools/testing/selftests/arm64/pauth/pac.c b/tools/testing/selftests/arm64/pauth/pac.c index 6d21b2fc758d..67d138057707 100644 --- a/tools/testing/selftests/arm64/pauth/pac.c +++ b/tools/testing/selftests/arm64/pauth/pac.c @@ -10,7 +10,7 @@ #include <setjmp.h> #include <sched.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #include "helper.h" #define PAC_COLLISION_ATTEMPTS 1000 diff --git a/tools/testing/selftests/arm64/tags/tags_test.c b/tools/testing/selftests/arm64/tags/tags_test.c index 8ae26e496c89..375ab47f0edb 100644 --- a/tools/testing/selftests/arm64/tags/tags_test.c +++ b/tools/testing/selftests/arm64/tags/tags_test.c @@ -6,7 +6,7 @@ #include <stdint.h> #include <sys/prctl.h> #include <sys/utsname.h> -#include "../../kselftest.h" +#include "kselftest.h" #define SHIFT_TAG(tag) ((uint64_t)(tag) << 56) #define SET_TAG(ptr, tag) (((uint64_t)(ptr) & ~SHIFT_TAG(0xff)) | \ diff --git a/tools/testing/selftests/bpf/xskxceiver.c b/tools/testing/selftests/bpf/xskxceiver.c index 9234a58b0a97..05b3cebc5ca9 100644 --- a/tools/testing/selftests/bpf/xskxceiver.c +++ b/tools/testing/selftests/bpf/xskxceiver.c @@ -96,7 +96,7 @@ #include "xskxceiver.h" #include <bpf/bpf.h> #include <linux/filter.h> -#include "../kselftest.h" +#include "kselftest.h" #include "xsk_xdp_common.h" #include <network_helpers.h> diff --git a/tools/testing/selftests/breakpoints/breakpoint_test.c b/tools/testing/selftests/breakpoints/breakpoint_test.c index d46962a24724..1159d81890c2 100644 --- a/tools/testing/selftests/breakpoints/breakpoint_test.c +++ b/tools/testing/selftests/breakpoints/breakpoint_test.c @@ -18,7 +18,7 @@ #include <errno.h> #include <string.h> -#include "../kselftest.h" +#include "kselftest.h" #define COUNT_ISN_BPS 4 #define COUNT_WPS 4 diff --git a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c index e7041816085a..5fc0f37f3fd4 100644 --- a/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c +++ b/tools/testing/selftests/breakpoints/breakpoint_test_arm64.c @@ -26,7 +26,7 @@ #include <errno.h> #include <signal.h> -#include "../kselftest.h" +#include "kselftest.h" static volatile uint8_t var[96] __attribute__((__aligned__(32))); diff --git a/tools/testing/selftests/breakpoints/step_after_suspend_test.c b/tools/testing/selftests/breakpoints/step_after_suspend_test.c index 8d233ac95696..ca2aaab9e4ca 100644 --- a/tools/testing/selftests/breakpoints/step_after_suspend_test.c +++ b/tools/testing/selftests/breakpoints/step_after_suspend_test.c @@ -19,7 +19,7 @@ #include <sys/types.h> #include <sys/wait.h> -#include "../kselftest.h" +#include "kselftest.h" void child(int cpu) { diff --git a/tools/testing/selftests/cachestat/test_cachestat.c b/tools/testing/selftests/cachestat/test_cachestat.c index ab838bcb9ec5..542cd09cb443 100644 --- a/tools/testing/selftests/cachestat/test_cachestat.c +++ b/tools/testing/selftests/cachestat/test_cachestat.c @@ -16,7 +16,7 @@ #include <fcntl.h> #include <errno.h> -#include "../kselftest.h" +#include "kselftest.h" #define NR_TESTS 9 diff --git a/tools/testing/selftests/capabilities/test_execve.c b/tools/testing/selftests/capabilities/test_execve.c index 47bad7ddc5bc..46fc8d46b6e6 100644 --- a/tools/testing/selftests/capabilities/test_execve.c +++ b/tools/testing/selftests/capabilities/test_execve.c @@ -18,7 +18,7 @@ #include <sys/prctl.h> #include <sys/stat.h> -#include "../kselftest.h" +#include "kselftest.h" static int nerrs; static pid_t mpid; /* main() pid is used to avoid duplicate test counts */ diff --git a/tools/testing/selftests/capabilities/validate_cap.c b/tools/testing/selftests/capabilities/validate_cap.c index 65f2a1c89239..cef1d9937b9f 100644 --- a/tools/testing/selftests/capabilities/validate_cap.c +++ b/tools/testing/selftests/capabilities/validate_cap.c @@ -7,7 +7,7 @@ #include <sys/prctl.h> #include <sys/auxv.h> -#include "../kselftest.h" +#include "kselftest.h" #if __GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 19) # define HAVE_GETAUXVAL diff --git a/tools/testing/selftests/cgroup/test_core.c b/tools/testing/selftests/cgroup/test_core.c index 1d778c8b7764..102262555a59 100644 --- a/tools/testing/selftests/cgroup/test_core.c +++ b/tools/testing/selftests/cgroup/test_core.c @@ -17,7 +17,7 @@ #include <string.h> #include <pthread.h> -#include "../kselftest.h" +#include "kselftest.h" #include "cgroup_util.h" static bool nsdelegate; diff --git a/tools/testing/selftests/cgroup/test_cpu.c b/tools/testing/selftests/cgroup/test_cpu.c index b1b30e82dd7c..c83f05438d7c 100644 --- a/tools/testing/selftests/cgroup/test_cpu.c +++ b/tools/testing/selftests/cgroup/test_cpu.c @@ -11,7 +11,7 @@ #include <time.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #include "cgroup_util.h" enum hog_clock_type { diff --git a/tools/testing/selftests/cgroup/test_cpuset.c b/tools/testing/selftests/cgroup/test_cpuset.c index 8086d2ea394f..c5cf8b56ceb8 100644 --- a/tools/testing/selftests/cgroup/test_cpuset.c +++ b/tools/testing/selftests/cgroup/test_cpuset.c @@ -3,7 +3,7 @@ #include <linux/limits.h> #include <signal.h> -#include "../kselftest.h" +#include "kselftest.h" #include "cgroup_util.h" static int idle_process_fn(const char *cgroup, void *arg) diff --git a/tools/testing/selftests/cgroup/test_freezer.c b/tools/testing/selftests/cgroup/test_freezer.c index 465cdad2bfca..97fae92c8387 100644 --- a/tools/testing/selftests/cgroup/test_freezer.c +++ b/tools/testing/selftests/cgroup/test_freezer.c @@ -11,7 +11,7 @@ #include <string.h> #include <sys/wait.h> -#include "../kselftest.h" +#include "kselftest.h" #include "cgroup_util.h" #define DEBUG diff --git a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c index 856f9508ea56..f451aa449be6 100644 --- a/tools/testing/selftests/cgroup/test_hugetlb_memcg.c +++ b/tools/testing/selftests/cgroup/test_hugetlb_memcg.c @@ -7,7 +7,7 @@ #include <stdlib.h> #include <string.h> #include <fcntl.h> -#include "../kselftest.h" +#include "kselftest.h" #include "cgroup_util.h" #define ADDR ((void *)(0x0UL)) diff --git a/tools/testing/selftests/cgroup/test_kill.c b/tools/testing/selftests/cgroup/test_kill.c index ed590b150a17..c8c9d306925b 100644 --- a/tools/testing/selftests/cgroup/test_kill.c +++ b/tools/testing/selftests/cgroup/test_kill.c @@ -9,7 +9,7 @@ #include <sys/types.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #include "../pidfd/pidfd.h" #include "cgroup_util.h" diff --git a/tools/testing/selftests/cgroup/test_kmem.c b/tools/testing/selftests/cgroup/test_kmem.c index d4c4a514ee43..ca38525484e3 100644 --- a/tools/testing/selftests/cgroup/test_kmem.c +++ b/tools/testing/selftests/cgroup/test_kmem.c @@ -14,7 +14,7 @@ #include <sys/sysinfo.h> #include <pthread.h> -#include "../kselftest.h" +#include "kselftest.h" #include "cgroup_util.h" diff --git a/tools/testing/selftests/cgroup/test_memcontrol.c b/tools/testing/selftests/cgroup/test_memcontrol.c index b117325c0439..4e1647568c5b 100644 --- a/tools/testing/selftests/cgroup/test_memcontrol.c +++ b/tools/testing/selftests/cgroup/test_memcontrol.c @@ -18,7 +18,7 @@ #include <errno.h> #include <sys/mman.h> -#include "../kselftest.h" +#include "kselftest.h" #include "cgroup_util.h" static bool has_localevents; diff --git a/tools/testing/selftests/cgroup/test_pids.c b/tools/testing/selftests/cgroup/test_pids.c index d8a1d1cd5007..9a387c815d2c 100644 --- a/tools/testing/selftests/cgroup/test_pids.c +++ b/tools/testing/selftests/cgroup/test_pids.c @@ -9,7 +9,7 @@ #include <sys/types.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #include "cgroup_util.h" static int run_success(const char *cgroup, void *arg) diff --git a/tools/testing/selftests/cgroup/test_zswap.c b/tools/testing/selftests/cgroup/test_zswap.c index 86a8930b47e3..64ebc3f3f203 100644 --- a/tools/testing/selftests/cgroup/test_zswap.c +++ b/tools/testing/selftests/cgroup/test_zswap.c @@ -10,7 +10,7 @@ #include <sys/wait.h> #include <sys/mman.h> -#include "../kselftest.h" +#include "kselftest.h" #include "cgroup_util.h" static int read_int(const char *path, size_t *value) diff --git a/tools/testing/selftests/clone3/clone3.c b/tools/testing/selftests/clone3/clone3.c index e61f07973ce5..289e0c7c1f09 100644 --- a/tools/testing/selftests/clone3/clone3.c +++ b/tools/testing/selftests/clone3/clone3.c @@ -18,7 +18,7 @@ #include <unistd.h> #include <sched.h> -#include "../kselftest.h" +#include "kselftest.h" #include "clone3_selftests.h" enum test_mode { diff --git a/tools/testing/selftests/clone3/clone3_cap_checkpoint_restore.c b/tools/testing/selftests/clone3/clone3_cap_checkpoint_restore.c index 3c196fa86c99..e82281efa273 100644 --- a/tools/testing/selftests/clone3/clone3_cap_checkpoint_restore.c +++ b/tools/testing/selftests/clone3/clone3_cap_checkpoint_restore.c @@ -24,7 +24,7 @@ #include <unistd.h> #include <sched.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "clone3_selftests.h" static void child_exit(int ret) diff --git a/tools/testing/selftests/clone3/clone3_clear_sighand.c b/tools/testing/selftests/clone3/clone3_clear_sighand.c index ce0426786828..de0c9d62015d 100644 --- a/tools/testing/selftests/clone3/clone3_clear_sighand.c +++ b/tools/testing/selftests/clone3/clone3_clear_sighand.c @@ -13,7 +13,7 @@ #include <sys/syscall.h> #include <sys/wait.h> -#include "../kselftest.h" +#include "kselftest.h" #include "clone3_selftests.h" static void nop_handler(int signo) diff --git a/tools/testing/selftests/clone3/clone3_selftests.h b/tools/testing/selftests/clone3/clone3_selftests.h index eeca8005723f..a0593e8950f0 100644 --- a/tools/testing/selftests/clone3/clone3_selftests.h +++ b/tools/testing/selftests/clone3/clone3_selftests.h @@ -11,7 +11,7 @@ #include <syscall.h> #include <sys/wait.h> -#include "../kselftest.h" +#include "kselftest.h" #define ptr_to_u64(ptr) ((__u64)((uintptr_t)(ptr))) diff --git a/tools/testing/selftests/clone3/clone3_set_tid.c b/tools/testing/selftests/clone3/clone3_set_tid.c index bfb0da2b4fdd..5c944aee6b41 100644 --- a/tools/testing/selftests/clone3/clone3_set_tid.c +++ b/tools/testing/selftests/clone3/clone3_set_tid.c @@ -20,7 +20,7 @@ #include <unistd.h> #include <sched.h> -#include "../kselftest.h" +#include "kselftest.h" #include "clone3_selftests.h" #define MAX_PID_NS_LEVEL 32 diff --git a/tools/testing/selftests/connector/proc_filter.c b/tools/testing/selftests/connector/proc_filter.c index 4a825b997666..36c11467a8f1 100644 --- a/tools/testing/selftests/connector/proc_filter.c +++ b/tools/testing/selftests/connector/proc_filter.c @@ -16,7 +16,7 @@ #include <signal.h> #include <string.h> -#include "../kselftest.h" +#include "kselftest.h" #define NL_MESSAGE_SIZE (sizeof(struct nlmsghdr) + sizeof(struct cn_msg) + \ sizeof(struct proc_input)) diff --git a/tools/testing/selftests/core/close_range_test.c b/tools/testing/selftests/core/close_range_test.c index e0d9851fe1c9..f14eca63f20c 100644 --- a/tools/testing/selftests/core/close_range_test.c +++ b/tools/testing/selftests/core/close_range_test.c @@ -14,7 +14,7 @@ #include <sys/resource.h> #include <linux/close_range.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "../clone3/clone3_selftests.h" diff --git a/tools/testing/selftests/core/unshare_test.c b/tools/testing/selftests/core/unshare_test.c index 7fec9dfb1b0e..ffce75a6c228 100644 --- a/tools/testing/selftests/core/unshare_test.c +++ b/tools/testing/selftests/core/unshare_test.c @@ -14,7 +14,7 @@ #include <sys/resource.h> #include <linux/close_range.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "../clone3/clone3_selftests.h" TEST(unshare_EMFILE) diff --git a/tools/testing/selftests/coredump/stackdump_test.c b/tools/testing/selftests/coredump/stackdump_test.c index c2e895bcc160..1ec88937a1c2 100644 --- a/tools/testing/selftests/coredump/stackdump_test.c +++ b/tools/testing/selftests/coredump/stackdump_test.c @@ -19,7 +19,7 @@ #include <sys/un.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "../filesystems/wrappers.h" #include "../pidfd/pidfd.h" diff --git a/tools/testing/selftests/dma/Makefile b/tools/testing/selftests/dma/Makefile deleted file mode 100644 index cd8c5ece1cba..000000000000 --- a/tools/testing/selftests/dma/Makefile +++ /dev/null @@ -1,7 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0 -CFLAGS += -I../../../../usr/include/ -CFLAGS += -I../../../../include/ - -TEST_GEN_PROGS := dma_map_benchmark - -include ../lib.mk diff --git a/tools/testing/selftests/dmabuf-heaps/dmabuf-heap.c b/tools/testing/selftests/dmabuf-heaps/dmabuf-heap.c index 5d0a809dc2df..fc9694fc4e89 100644 --- a/tools/testing/selftests/dmabuf-heaps/dmabuf-heap.c +++ b/tools/testing/selftests/dmabuf-heaps/dmabuf-heap.c @@ -15,7 +15,7 @@ #include <linux/dma-buf.h> #include <linux/dma-heap.h> #include <drm/drm.h> -#include "../kselftest.h" +#include "kselftest.h" #define DEVPATH "/dev/dma_heap" diff --git a/tools/testing/selftests/drivers/dma-buf/udmabuf.c b/tools/testing/selftests/drivers/dma-buf/udmabuf.c index 77aa2897e79f..d78aec662586 100644 --- a/tools/testing/selftests/drivers/dma-buf/udmabuf.c +++ b/tools/testing/selftests/drivers/dma-buf/udmabuf.c @@ -16,7 +16,7 @@ #include <sys/mman.h> #include <linux/memfd.h> #include <linux/udmabuf.h> -#include "../../kselftest.h" +#include "kselftest.h" #define TEST_PREFIX "drivers/dma-buf/udmabuf" #define NUM_PAGES 4 diff --git a/tools/testing/selftests/drivers/net/gro.c b/tools/testing/selftests/drivers/net/gro.c index 995b492f5bcb..e894037d2e3e 100644 --- a/tools/testing/selftests/drivers/net/gro.c +++ b/tools/testing/selftests/drivers/net/gro.c @@ -57,7 +57,7 @@ #include <string.h> #include <unistd.h> -#include "../../kselftest.h" +#include "kselftest.h" #include "../../net/lib/ksft.h" #define DPORT 8000 diff --git a/tools/testing/selftests/drivers/net/hw/toeplitz.c b/tools/testing/selftests/drivers/net/hw/toeplitz.c index a4d04438c313..d23b3b0c20a3 100644 --- a/tools/testing/selftests/drivers/net/hw/toeplitz.c +++ b/tools/testing/selftests/drivers/net/hw/toeplitz.c @@ -55,7 +55,7 @@ #include <ynl.h> #include "ethtool-user.h" -#include "../../../kselftest.h" +#include "kselftest.h" #include "../../../net/lib/ksft.h" #define TOEPLITZ_KEY_MIN_LEN 40 diff --git a/tools/testing/selftests/drivers/ntsync/ntsync.c b/tools/testing/selftests/drivers/ntsync/ntsync.c index 3aad311574c4..e6a37214aa46 100644 --- a/tools/testing/selftests/drivers/ntsync/ntsync.c +++ b/tools/testing/selftests/drivers/ntsync/ntsync.c @@ -12,7 +12,7 @@ #include <time.h> #include <pthread.h> #include <linux/ntsync.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" static int read_sem_state(int sem, __u32 *count, __u32 *max) { diff --git a/tools/testing/selftests/drivers/s390x/uvdevice/test_uvdevice.c b/tools/testing/selftests/drivers/s390x/uvdevice/test_uvdevice.c index 7ee7492138c6..14df9aa07308 100644 --- a/tools/testing/selftests/drivers/s390x/uvdevice/test_uvdevice.c +++ b/tools/testing/selftests/drivers/s390x/uvdevice/test_uvdevice.c @@ -14,7 +14,7 @@ #include <asm/uvdevice.h> -#include "../../../kselftest_harness.h" +#include "kselftest_harness.h" #define UV_PATH "/dev/uv" #define BUFFER_SIZE 0x200 diff --git a/tools/testing/selftests/exec/check-exec.c b/tools/testing/selftests/exec/check-exec.c index 55bce47e56b7..f2397e75aa7c 100644 --- a/tools/testing/selftests/exec/check-exec.c +++ b/tools/testing/selftests/exec/check-exec.c @@ -30,7 +30,7 @@ #define _ASM_GENERIC_FCNTL_H #include <linux/fcntl.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static int sys_execveat(int dirfd, const char *pathname, char *const argv[], char *const envp[], int flags) diff --git a/tools/testing/selftests/exec/execveat.c b/tools/testing/selftests/exec/execveat.c index 8fb7395fd35b..d37c068ed5fe 100644 --- a/tools/testing/selftests/exec/execveat.c +++ b/tools/testing/selftests/exec/execveat.c @@ -21,7 +21,7 @@ #include <string.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #define TESTS_EXPECTED 54 #define TEST_NAME_LEN (PATH_MAX * 4) diff --git a/tools/testing/selftests/exec/load_address.c b/tools/testing/selftests/exec/load_address.c index 8257fddba8c8..55fd3732f029 100644 --- a/tools/testing/selftests/exec/load_address.c +++ b/tools/testing/selftests/exec/load_address.c @@ -6,7 +6,7 @@ #include <stdio.h> #include <stdlib.h> #include <stdbool.h> -#include "../kselftest.h" +#include "kselftest.h" struct Statistics { unsigned long long load_address; diff --git a/tools/testing/selftests/exec/non-regular.c b/tools/testing/selftests/exec/non-regular.c index cd3a34aca93e..14ac36487df5 100644 --- a/tools/testing/selftests/exec/non-regular.c +++ b/tools/testing/selftests/exec/non-regular.c @@ -9,7 +9,7 @@ #include <sys/sysmacros.h> #include <sys/types.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" /* Remove a file, ignoring the result if it didn't exist. */ void rm(struct __test_metadata *_metadata, const char *pathname, diff --git a/tools/testing/selftests/exec/null-argv.c b/tools/testing/selftests/exec/null-argv.c index c19726e710d1..4940aee5bb38 100644 --- a/tools/testing/selftests/exec/null-argv.c +++ b/tools/testing/selftests/exec/null-argv.c @@ -5,7 +5,7 @@ #include <sys/types.h> #include <sys/wait.h> -#include "../kselftest.h" +#include "kselftest.h" #define FORK(exec) \ do { \ diff --git a/tools/testing/selftests/exec/recursion-depth.c b/tools/testing/selftests/exec/recursion-depth.c index 438c8ff2fd26..7b5c4f6d1928 100644 --- a/tools/testing/selftests/exec/recursion-depth.c +++ b/tools/testing/selftests/exec/recursion-depth.c @@ -23,7 +23,7 @@ #include <fcntl.h> #include <sys/mount.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" int main(void) { diff --git a/tools/testing/selftests/fchmodat2/fchmodat2_test.c b/tools/testing/selftests/fchmodat2/fchmodat2_test.c index e0319417124d..e397339495f6 100644 --- a/tools/testing/selftests/fchmodat2/fchmodat2_test.c +++ b/tools/testing/selftests/fchmodat2/fchmodat2_test.c @@ -7,7 +7,7 @@ #include <syscall.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" int sys_fchmodat2(int dfd, const char *filename, mode_t mode, int flags) { diff --git a/tools/testing/selftests/filelock/ofdlocks.c b/tools/testing/selftests/filelock/ofdlocks.c index a55b79810ab2..ff8d47fc373a 100644 --- a/tools/testing/selftests/filelock/ofdlocks.c +++ b/tools/testing/selftests/filelock/ofdlocks.c @@ -6,7 +6,7 @@ #include <stdio.h> #include <unistd.h> #include <string.h> -#include "../kselftest.h" +#include "kselftest.h" static int lock_set(int fd, struct flock *fl) { diff --git a/tools/testing/selftests/filesystems/anon_inode_test.c b/tools/testing/selftests/filesystems/anon_inode_test.c index 73e0a4d4fb2f..94c6c81c2301 100644 --- a/tools/testing/selftests/filesystems/anon_inode_test.c +++ b/tools/testing/selftests/filesystems/anon_inode_test.c @@ -6,7 +6,7 @@ #include <stdio.h> #include <sys/stat.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "wrappers.h" TEST(anon_inode_no_chown) diff --git a/tools/testing/selftests/filesystems/binderfs/binderfs_test.c b/tools/testing/selftests/filesystems/binderfs/binderfs_test.c index 39a68078a79b..a1a79a6fef17 100644 --- a/tools/testing/selftests/filesystems/binderfs/binderfs_test.c +++ b/tools/testing/selftests/filesystems/binderfs/binderfs_test.c @@ -21,7 +21,7 @@ #include <linux/android/binder.h> #include <linux/android/binderfs.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define DEFAULT_THREADS 4 diff --git a/tools/testing/selftests/filesystems/devpts_pts.c b/tools/testing/selftests/filesystems/devpts_pts.c index b1fc9b916ace..54fea349204e 100644 --- a/tools/testing/selftests/filesystems/devpts_pts.c +++ b/tools/testing/selftests/filesystems/devpts_pts.c @@ -11,7 +11,7 @@ #include <asm/ioctls.h> #include <sys/mount.h> #include <sys/wait.h> -#include "../kselftest.h" +#include "kselftest.h" static bool terminal_dup2(int duplicate, int original) { diff --git a/tools/testing/selftests/filesystems/epoll/epoll_wakeup_test.c b/tools/testing/selftests/filesystems/epoll/epoll_wakeup_test.c index 65ede506305c..8bc57a2ef966 100644 --- a/tools/testing/selftests/filesystems/epoll/epoll_wakeup_test.c +++ b/tools/testing/selftests/filesystems/epoll/epoll_wakeup_test.c @@ -11,7 +11,7 @@ #include <sys/epoll.h> #include <sys/socket.h> #include <sys/eventfd.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" struct epoll_mtcontext { diff --git a/tools/testing/selftests/filesystems/eventfd/eventfd_test.c b/tools/testing/selftests/filesystems/eventfd/eventfd_test.c index 72d51ad0ee0e..1b48f267157d 100644 --- a/tools/testing/selftests/filesystems/eventfd/eventfd_test.c +++ b/tools/testing/selftests/filesystems/eventfd/eventfd_test.c @@ -11,7 +11,7 @@ #include <pthread.h> #include <sys/epoll.h> #include <sys/eventfd.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define EVENTFD_TEST_ITERATIONS 100000UL diff --git a/tools/testing/selftests/filesystems/fclog.c b/tools/testing/selftests/filesystems/fclog.c index 912a8b755c3b..551c4a0f395a 100644 --- a/tools/testing/selftests/filesystems/fclog.c +++ b/tools/testing/selftests/filesystems/fclog.c @@ -13,7 +13,7 @@ #include <unistd.h> #include <sys/mount.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define ASSERT_ERRNO(expected, _t, seen) \ __EXPECT(expected, #expected, \ diff --git a/tools/testing/selftests/filesystems/file_stressor.c b/tools/testing/selftests/filesystems/file_stressor.c index 01dd89f8e52f..141badd671a9 100644 --- a/tools/testing/selftests/filesystems/file_stressor.c +++ b/tools/testing/selftests/filesystems/file_stressor.c @@ -12,7 +12,7 @@ #include <sys/mount.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <linux/types.h> #include <linux/mount.h> diff --git a/tools/testing/selftests/filesystems/fuse/fusectl_test.c b/tools/testing/selftests/filesystems/fuse/fusectl_test.c index 8d124d1cacb2..0d1d012c35ed 100644 --- a/tools/testing/selftests/filesystems/fuse/fusectl_test.c +++ b/tools/testing/selftests/filesystems/fuse/fusectl_test.c @@ -17,7 +17,7 @@ #include <sched.h> #include <linux/limits.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define FUSECTL_MOUNTPOINT "/sys/fs/fuse/connections" #define FUSE_MOUNTPOINT "/tmp/fuse_mnt_XXXXXX" diff --git a/tools/testing/selftests/filesystems/kernfs_test.c b/tools/testing/selftests/filesystems/kernfs_test.c index 16538b3b318e..84c2b910a60d 100644 --- a/tools/testing/selftests/filesystems/kernfs_test.c +++ b/tools/testing/selftests/filesystems/kernfs_test.c @@ -7,7 +7,7 @@ #include <sys/stat.h> #include <sys/xattr.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "wrappers.h" TEST(kernfs_listxattr) diff --git a/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c b/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c index e4b7c2b457ee..6381af6a40e3 100644 --- a/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c +++ b/tools/testing/selftests/filesystems/mount-notify/mount-notify_test.c @@ -19,7 +19,7 @@ typedef struct { #include <sys/syscall.h> #include <sys/fanotify.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #include "../statmount/statmount.h" #include "../utils.h" diff --git a/tools/testing/selftests/filesystems/mount-notify/mount-notify_test_ns.c b/tools/testing/selftests/filesystems/mount-notify/mount-notify_test_ns.c index 9f57ca46e3af..320ee25dc8a5 100644 --- a/tools/testing/selftests/filesystems/mount-notify/mount-notify_test_ns.c +++ b/tools/testing/selftests/filesystems/mount-notify/mount-notify_test_ns.c @@ -19,7 +19,7 @@ typedef struct { #include <sys/syscall.h> #include <sys/fanotify.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #include "../statmount/statmount.h" #include "../utils.h" diff --git a/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c b/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c index a3d8015897e9..61e55dfbf121 100644 --- a/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c +++ b/tools/testing/selftests/filesystems/nsfs/iterate_mntns.c @@ -12,7 +12,7 @@ #include <sys/mount.h> #include <unistd.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define MNT_NS_COUNT 11 #define MNT_NS_LAST_INDEX 10 diff --git a/tools/testing/selftests/filesystems/overlayfs/dev_in_maps.c b/tools/testing/selftests/filesystems/overlayfs/dev_in_maps.c index 31db54b00e64..8924cea6aa4b 100644 --- a/tools/testing/selftests/filesystems/overlayfs/dev_in_maps.c +++ b/tools/testing/selftests/filesystems/overlayfs/dev_in_maps.c @@ -15,7 +15,7 @@ #include <sched.h> #include <fcntl.h> -#include "../../kselftest.h" +#include "kselftest.h" #include "log.h" #include "../wrappers.h" diff --git a/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c b/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c index dc0449fa628f..3c0b93183348 100644 --- a/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c +++ b/tools/testing/selftests/filesystems/overlayfs/set_layers_via_fds.c @@ -12,7 +12,7 @@ #include <sys/mount.h> #include <unistd.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #include "../../pidfd/pidfd.h" #include "log.h" #include "../utils.h" diff --git a/tools/testing/selftests/filesystems/statmount/listmount_test.c b/tools/testing/selftests/filesystems/statmount/listmount_test.c index 15f0834f7557..8bc82f38c42f 100644 --- a/tools/testing/selftests/filesystems/statmount/listmount_test.c +++ b/tools/testing/selftests/filesystems/statmount/listmount_test.c @@ -11,7 +11,7 @@ #include <unistd.h> #include "statmount.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #ifndef LISTMOUNT_REVERSE #define LISTMOUNT_REVERSE (1 << 0) /* List later mounts first */ diff --git a/tools/testing/selftests/filesystems/statmount/statmount_test.c b/tools/testing/selftests/filesystems/statmount/statmount_test.c index f048042e53e9..6e53430423d2 100644 --- a/tools/testing/selftests/filesystems/statmount/statmount_test.c +++ b/tools/testing/selftests/filesystems/statmount/statmount_test.c @@ -13,7 +13,7 @@ #include <linux/stat.h> #include "statmount.h" -#include "../../kselftest.h" +#include "kselftest.h" static const char *const known_fs[] = { "9p", "adfs", "affs", "afs", "aio", "anon_inodefs", "apparmorfs", diff --git a/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c b/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c index 605a3fa16bf7..d56d4103182f 100644 --- a/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c +++ b/tools/testing/selftests/filesystems/statmount/statmount_test_ns.c @@ -15,7 +15,7 @@ #include "statmount.h" #include "../utils.h" -#include "../../kselftest.h" +#include "kselftest.h" #define NSID_PASS 0 #define NSID_FAIL 1 diff --git a/tools/testing/selftests/filesystems/utils.c b/tools/testing/selftests/filesystems/utils.c index a0c64f415a7f..c9dd5412b37b 100644 --- a/tools/testing/selftests/filesystems/utils.c +++ b/tools/testing/selftests/filesystems/utils.c @@ -20,7 +20,7 @@ #include <sys/xattr.h> #include <sys/mount.h> -#include "../kselftest.h" +#include "kselftest.h" #include "wrappers.h" #include "utils.h" diff --git a/tools/testing/selftests/futex/functional/futex_numa_mpol.c b/tools/testing/selftests/futex/functional/futex_numa_mpol.c index d037a3f10ee8..ab8555752137 100644 --- a/tools/testing/selftests/futex/functional/futex_numa_mpol.c +++ b/tools/testing/selftests/futex/functional/futex_numa_mpol.c @@ -18,7 +18,7 @@ #include "futextest.h" #include "futex2test.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define MAX_THREADS 64 diff --git a/tools/testing/selftests/futex/functional/futex_priv_hash.c b/tools/testing/selftests/futex/functional/futex_priv_hash.c index 3b7b5851f290..e8079d7c65e8 100644 --- a/tools/testing/selftests/futex/functional/futex_priv_hash.c +++ b/tools/testing/selftests/futex/functional/futex_priv_hash.c @@ -14,7 +14,7 @@ #include <linux/prctl.h> #include <sys/prctl.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define MAX_THREADS 64 diff --git a/tools/testing/selftests/futex/functional/futex_requeue.c b/tools/testing/selftests/futex/functional/futex_requeue.c index 69e2555b6039..35d4be23db5d 100644 --- a/tools/testing/selftests/futex/functional/futex_requeue.c +++ b/tools/testing/selftests/futex/functional/futex_requeue.c @@ -9,7 +9,7 @@ #include <limits.h> #include "futextest.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define timeout_ns 30000000 #define WAKE_WAIT_US 10000 diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi.c b/tools/testing/selftests/futex/functional/futex_requeue_pi.c index f299d75848cd..46d2858e15a8 100644 --- a/tools/testing/selftests/futex/functional/futex_requeue_pi.c +++ b/tools/testing/selftests/futex/functional/futex_requeue_pi.c @@ -29,7 +29,7 @@ #include "atomic.h" #include "futextest.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define MAX_WAKE_ITERS 1000 #define THREAD_MAX 10 diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c b/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c index 77135a22a583..f686e605359c 100644 --- a/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c +++ b/tools/testing/selftests/futex/functional/futex_requeue_pi_mismatched_ops.c @@ -25,7 +25,7 @@ #include <time.h> #include "futextest.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" futex_t f1 = FUTEX_INITIALIZER; futex_t f2 = FUTEX_INITIALIZER; diff --git a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c index e34ee0f9ebcc..a18ccae73eb1 100644 --- a/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c +++ b/tools/testing/selftests/futex/functional/futex_requeue_pi_signal_restart.c @@ -27,7 +27,7 @@ #include "atomic.h" #include "futextest.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define DELAY_US 100 diff --git a/tools/testing/selftests/futex/functional/futex_wait.c b/tools/testing/selftests/futex/functional/futex_wait.c index 152ca4612886..0e69c53524c1 100644 --- a/tools/testing/selftests/futex/functional/futex_wait.c +++ b/tools/testing/selftests/futex/functional/futex_wait.c @@ -11,7 +11,7 @@ #include <fcntl.h> #include "futextest.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define timeout_ns 30000000 #define WAKE_WAIT_US 10000 diff --git a/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c b/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c index 8952ebda14ab..2a749f9b14eb 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c +++ b/tools/testing/selftests/futex/functional/futex_wait_private_mapped_file.c @@ -28,7 +28,7 @@ #include <signal.h> #include "futextest.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define PAGE_SZ 4096 diff --git a/tools/testing/selftests/futex/functional/futex_wait_timeout.c b/tools/testing/selftests/futex/functional/futex_wait_timeout.c index 0c8766aced2e..674dd13af421 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_timeout.c +++ b/tools/testing/selftests/futex/functional/futex_wait_timeout.c @@ -19,7 +19,7 @@ #include "futextest.h" #include "futex2test.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" static long timeout_ns = 100000; /* 100us default timeout */ static futex_t futex_pi; diff --git a/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c b/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c index ce2301500d83..b07d68a67f31 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c +++ b/tools/testing/selftests/futex/functional/futex_wait_uninitialized_heap.c @@ -30,7 +30,7 @@ #include <libgen.h> #include "futextest.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define WAIT_US 5000000 diff --git a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c index 36b7a54a4085..9ff936ecf164 100644 --- a/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c +++ b/tools/testing/selftests/futex/functional/futex_wait_wouldblock.c @@ -24,7 +24,7 @@ #include "futextest.h" #include "futex2test.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define timeout_ns 100000 diff --git a/tools/testing/selftests/futex/functional/futex_waitv.c b/tools/testing/selftests/futex/functional/futex_waitv.c index c684b10eb76e..d60876164d4b 100644 --- a/tools/testing/selftests/futex/functional/futex_waitv.c +++ b/tools/testing/selftests/futex/functional/futex_waitv.c @@ -18,7 +18,7 @@ #include "futextest.h" #include "futex2test.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define WAKE_WAIT_US 10000 #define NR_FUTEXES 30 diff --git a/tools/testing/selftests/hid/hid_common.h b/tools/testing/selftests/hid/hid_common.h index 8085519c47cb..e3b267446fa0 100644 --- a/tools/testing/selftests/hid/hid_common.h +++ b/tools/testing/selftests/hid/hid_common.h @@ -1,7 +1,7 @@ /* SPDX-License-Identifier: GPL-2.0 */ /* Copyright (c) 2022-2024 Red Hat */ -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <fcntl.h> #include <fnmatch.h> diff --git a/tools/testing/selftests/intel_pstate/aperf.c b/tools/testing/selftests/intel_pstate/aperf.c index a8acf3996973..953b63e5aa6a 100644 --- a/tools/testing/selftests/intel_pstate/aperf.c +++ b/tools/testing/selftests/intel_pstate/aperf.c @@ -11,7 +11,7 @@ #include <errno.h> #include <string.h> #include <time.h> -#include "../kselftest.h" +#include "kselftest.h" #define MSEC_PER_SEC 1000L #define NSEC_PER_MSEC 1000000L diff --git a/tools/testing/selftests/iommu/iommufd_utils.h b/tools/testing/selftests/iommu/iommufd_utils.h index 0a0ff6f7926d..5502751d500c 100644 --- a/tools/testing/selftests/iommu/iommufd_utils.h +++ b/tools/testing/selftests/iommu/iommufd_utils.h @@ -11,7 +11,7 @@ #include <assert.h> #include <poll.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "../../../../drivers/iommu/iommufd/iommufd_test.h" /* Hack to make assertions more readable */ diff --git a/tools/testing/selftests/ipc/msgque.c b/tools/testing/selftests/ipc/msgque.c index 5e36aeeb9901..e107379d185c 100644 --- a/tools/testing/selftests/ipc/msgque.c +++ b/tools/testing/selftests/ipc/msgque.c @@ -7,7 +7,7 @@ #include <sys/msg.h> #include <fcntl.h> -#include "../kselftest.h" +#include "kselftest.h" #define MAX_MSG_SIZE 32 diff --git a/tools/testing/selftests/ir/ir_loopback.c b/tools/testing/selftests/ir/ir_loopback.c index f4a15cbdd5ea..adfcf50b1264 100644 --- a/tools/testing/selftests/ir/ir_loopback.c +++ b/tools/testing/selftests/ir/ir_loopback.c @@ -23,7 +23,7 @@ #include <dirent.h> #include <sys/stat.h> #include <fcntl.h> -#include "../kselftest.h" +#include "kselftest.h" #define TEST_SCANCODES 10 #define SYSFS_PATH_MAX 256 diff --git a/tools/testing/selftests/kcmp/kcmp_test.c b/tools/testing/selftests/kcmp/kcmp_test.c index d7a8e321bb16..79aa438b7479 100644 --- a/tools/testing/selftests/kcmp/kcmp_test.c +++ b/tools/testing/selftests/kcmp/kcmp_test.c @@ -18,7 +18,7 @@ #include <sys/wait.h> #include <sys/epoll.h> -#include "../kselftest.h" +#include "kselftest.h" static long sys_kcmp(int pid1, int pid2, int type, unsigned long fd1, unsigned long fd2) { diff --git a/tools/testing/selftests/kho/vmtest.sh b/tools/testing/selftests/kho/vmtest.sh index 3f6c17166846..49fdac8e8b15 100755 --- a/tools/testing/selftests/kho/vmtest.sh +++ b/tools/testing/selftests/kho/vmtest.sh @@ -59,6 +59,7 @@ function build_kernel() { tee "$kconfig" > "$kho_config" <<EOF CONFIG_BLK_DEV_INITRD=y CONFIG_KEXEC_HANDOVER=y +CONFIG_KEXEC_HANDOVER_DEBUGFS=y CONFIG_TEST_KEXEC_HANDOVER=y CONFIG_DEBUG_KERNEL=y CONFIG_DEBUG_VM=y diff --git a/tools/testing/selftests/kselftest_harness.h b/tools/testing/selftests/kselftest_harness.h index 3f66e862e83e..baae6b7ded41 100644 --- a/tools/testing/selftests/kselftest_harness.h +++ b/tools/testing/selftests/kselftest_harness.h @@ -14,7 +14,7 @@ * * .. code-block:: c * - * #include "../kselftest_harness.h" + * #include "kselftest_harness.h" * * TEST(standalone_test) { * do_some_stuff; diff --git a/tools/testing/selftests/kselftest_harness/harness-selftest.c b/tools/testing/selftests/kselftest_harness/harness-selftest.c index b555493bdb4d..7820bb5d0e6d 100644 --- a/tools/testing/selftests/kselftest_harness/harness-selftest.c +++ b/tools/testing/selftests/kselftest_harness/harness-selftest.c @@ -8,7 +8,7 @@ /* Avoid any inconsistencies */ #define TH_LOG_STREAM stdout -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static void test_helper(struct __test_metadata *_metadata) { diff --git a/tools/testing/selftests/landlock/Makefile b/tools/testing/selftests/landlock/Makefile index a3f449914bf9..044b83bde16e 100644 --- a/tools/testing/selftests/landlock/Makefile +++ b/tools/testing/selftests/landlock/Makefile @@ -4,7 +4,7 @@ CFLAGS += -Wall -O2 $(KHDR_INCLUDES) -LOCAL_HDRS += common.h +LOCAL_HDRS += $(wildcard *.h) src_test := $(wildcard *_test.c) diff --git a/tools/testing/selftests/landlock/audit.h b/tools/testing/selftests/landlock/audit.h index 02fd1393947a..44eb433e9666 100644 --- a/tools/testing/selftests/landlock/audit.h +++ b/tools/testing/selftests/landlock/audit.h @@ -20,7 +20,7 @@ #include <sys/time.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #ifndef ARRAY_SIZE #define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0])) diff --git a/tools/testing/selftests/landlock/common.h b/tools/testing/selftests/landlock/common.h index 9acecae36f51..230b75f6015b 100644 --- a/tools/testing/selftests/landlock/common.h +++ b/tools/testing/selftests/landlock/common.h @@ -17,7 +17,7 @@ #include <sys/wait.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "wrappers.h" #define TMP_DIR "tmp" diff --git a/tools/testing/selftests/landlock/fs_test.c b/tools/testing/selftests/landlock/fs_test.c index fa0f18ec62c4..eee814e09dd7 100644 --- a/tools/testing/selftests/landlock/fs_test.c +++ b/tools/testing/selftests/landlock/fs_test.c @@ -2267,6 +2267,22 @@ static int test_exchange(const char *const oldpath, const char *const newpath) return 0; } +static int test_renameat(int olddirfd, const char *oldpath, int newdirfd, + const char *newpath) +{ + if (renameat2(olddirfd, oldpath, newdirfd, newpath, 0)) + return errno; + return 0; +} + +static int test_exchangeat(int olddirfd, const char *oldpath, int newdirfd, + const char *newpath) +{ + if (renameat2(olddirfd, oldpath, newdirfd, newpath, RENAME_EXCHANGE)) + return errno; + return 0; +} + TEST_F_FORK(layout1, rename_file) { const struct rule rules[] = { @@ -4561,6 +4577,18 @@ TEST_F_FORK(ioctl, handle_file_access_file) FIXTURE(layout1_bind) {}; /* clang-format on */ +static const char bind_dir_s1d3[] = TMP_DIR "/s2d1/s2d2/s1d3"; +static const char bind_file1_s1d3[] = TMP_DIR "/s2d1/s2d2/s1d3/f1"; + +/* Move targets for disconnected path tests. */ +static const char dir_s4d1[] = TMP_DIR "/s4d1"; +static const char file1_s4d1[] = TMP_DIR "/s4d1/f1"; +static const char file2_s4d1[] = TMP_DIR "/s4d1/f2"; +static const char dir_s4d2[] = TMP_DIR "/s4d1/s4d2"; +static const char file1_s4d2[] = TMP_DIR "/s4d1/s4d2/f1"; +static const char file1_name[] = "f1"; +static const char file2_name[] = "f2"; + FIXTURE_SETUP(layout1_bind) { prepare_layout(_metadata); @@ -4576,14 +4604,14 @@ FIXTURE_TEARDOWN_PARENT(layout1_bind) { /* umount(dir_s2d2)) is handled by namespace lifetime. */ + remove_path(file1_s4d1); + remove_path(file2_s4d1); + remove_layout1(_metadata); cleanup_layout(_metadata); } -static const char bind_dir_s1d3[] = TMP_DIR "/s2d1/s2d2/s1d3"; -static const char bind_file1_s1d3[] = TMP_DIR "/s2d1/s2d2/s1d3/f1"; - /* * layout1_bind hierarchy: * @@ -4594,20 +4622,25 @@ static const char bind_file1_s1d3[] = TMP_DIR "/s2d1/s2d2/s1d3/f1"; * │  └── s1d2 * │  ├── f1 * │  ├── f2 - * │  └── s1d3 + * │  └── s1d3 [disconnected by path_disconnected] * │  ├── f1 * │  └── f2 * ├── s2d1 * │  ├── f1 - * │  └── s2d2 + * │  └── s2d2 [bind mount from s1d2] * │  ├── f1 * │  ├── f2 * │  └── s1d3 * │  ├── f1 * │  └── f2 - * └── s3d1 - * └── s3d2 - * └── s3d3 + * ├── s3d1 + * │  └── s3d2 + * │  └── s3d3 + * └── s4d1 [renamed from s1d3 by path_disconnected] + *   ├── f1 + *   ├── f2 + * └── s4d2 + * └── f1 */ TEST_F_FORK(layout1_bind, no_restriction) @@ -4806,6 +4839,1431 @@ TEST_F_FORK(layout1_bind, reparent_cross_mount) ASSERT_EQ(0, rename(bind_file1_s1d3, file1_s2d2)); } +/* + * Make sure access to file through a disconnected path works as expected. + * This test moves s1d3 to s4d1. + */ +TEST_F_FORK(layout1_bind, path_disconnected) +{ + const struct rule layer1_allow_all[] = { + { + .path = TMP_DIR, + .access = ACCESS_ALL, + }, + {}, + }; + const struct rule layer2_allow_just_f1[] = { + { + .path = file1_s1d3, + .access = LANDLOCK_ACCESS_FS_READ_FILE, + }, + {}, + }; + const struct rule layer3_only_s1d2[] = { + { + .path = dir_s1d2, + .access = LANDLOCK_ACCESS_FS_READ_FILE, + }, + {}, + }; + + /* Landlock should not deny access just because it is disconnected. */ + int ruleset_fd_l1 = + create_ruleset(_metadata, ACCESS_ALL, layer1_allow_all); + + /* Creates the new ruleset now before we move the dir containing the file. */ + int ruleset_fd_l2 = + create_ruleset(_metadata, ACCESS_RW, layer2_allow_just_f1); + int ruleset_fd_l3 = + create_ruleset(_metadata, ACCESS_RW, layer3_only_s1d2); + int bind_s1d3_fd; + + ASSERT_LE(0, ruleset_fd_l1); + ASSERT_LE(0, ruleset_fd_l2); + ASSERT_LE(0, ruleset_fd_l3); + + enforce_ruleset(_metadata, ruleset_fd_l1); + EXPECT_EQ(0, close(ruleset_fd_l1)); + + bind_s1d3_fd = open(bind_dir_s1d3, O_PATH | O_CLOEXEC); + ASSERT_LE(0, bind_s1d3_fd); + + /* Tests access is possible before we move. */ + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)); + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file2_name, O_RDONLY)); + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, "..", O_RDONLY | O_DIRECTORY)); + + /* Makes it disconnected. */ + ASSERT_EQ(0, rename(dir_s1d3, dir_s4d1)) + { + TH_LOG("Failed to rename %s to %s: %s", dir_s1d3, dir_s4d1, + strerror(errno)); + } + + /* Tests that access is still possible. */ + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)); + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file2_name, O_RDONLY)); + + /* + * Tests that ".." is not possible (not because of Landlock, but just + * because it's disconnected). + */ + EXPECT_EQ(ENOENT, + test_open_rel(bind_s1d3_fd, "..", O_RDONLY | O_DIRECTORY)); + + /* This should still work with a narrower rule. */ + enforce_ruleset(_metadata, ruleset_fd_l2); + EXPECT_EQ(0, close(ruleset_fd_l2)); + + EXPECT_EQ(0, test_open(file1_s4d1, O_RDONLY)); + /* + * Accessing a file through a disconnected file descriptor can still be + * allowed by a rule tied to this file, even if it is no longer visible in + * its mount point. + */ + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)); + EXPECT_EQ(EACCES, test_open_rel(bind_s1d3_fd, file2_name, O_RDONLY)); + + enforce_ruleset(_metadata, ruleset_fd_l3); + EXPECT_EQ(0, close(ruleset_fd_l3)); + + EXPECT_EQ(EACCES, test_open(file1_s4d1, O_RDONLY)); + /* + * Accessing a file through a disconnected file descriptor can still be + * allowed by a rule tied to the original mount point, even if it is no + * longer visible in its mount point. + */ + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)); + EXPECT_EQ(EACCES, test_open_rel(bind_s1d3_fd, file2_name, O_RDONLY)); +} + +/* + * Test that renameat with disconnected paths works under Landlock. This test + * moves s1d3 to s4d2, so that we can have a rule allowing refers on the move + * target's immediate parent. + */ +TEST_F_FORK(layout1_bind, path_disconnected_rename) +{ + const struct rule layer1[] = { + { + .path = dir_s1d2, + .access = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_MAKE_DIR | + LANDLOCK_ACCESS_FS_REMOVE_DIR | + LANDLOCK_ACCESS_FS_MAKE_REG | + LANDLOCK_ACCESS_FS_REMOVE_FILE | + LANDLOCK_ACCESS_FS_READ_FILE, + }, + { + .path = dir_s4d1, + .access = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_MAKE_DIR | + LANDLOCK_ACCESS_FS_REMOVE_DIR | + LANDLOCK_ACCESS_FS_MAKE_REG | + LANDLOCK_ACCESS_FS_REMOVE_FILE | + LANDLOCK_ACCESS_FS_READ_FILE, + }, + {} + }; + + /* This layer only handles LANDLOCK_ACCESS_FS_READ_FILE. */ + const struct rule layer2_only_s1d2[] = { + { + .path = dir_s1d2, + .access = LANDLOCK_ACCESS_FS_READ_FILE, + }, + {}, + }; + int ruleset_fd_l1, ruleset_fd_l2; + pid_t child_pid; + int bind_s1d3_fd, status; + + ASSERT_EQ(0, mkdir(dir_s4d1, 0755)) + { + TH_LOG("Failed to create %s: %s", dir_s4d1, strerror(errno)); + } + ruleset_fd_l1 = create_ruleset(_metadata, ACCESS_ALL, layer1); + ruleset_fd_l2 = create_ruleset(_metadata, LANDLOCK_ACCESS_FS_READ_FILE, + layer2_only_s1d2); + ASSERT_LE(0, ruleset_fd_l1); + ASSERT_LE(0, ruleset_fd_l2); + + enforce_ruleset(_metadata, ruleset_fd_l1); + EXPECT_EQ(0, close(ruleset_fd_l1)); + + bind_s1d3_fd = open(bind_dir_s1d3, O_PATH | O_CLOEXEC); + ASSERT_LE(0, bind_s1d3_fd); + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)); + + /* Tests ENOENT priority over EACCES for disconnected directory. */ + EXPECT_EQ(EACCES, test_open_rel(bind_s1d3_fd, "..", O_DIRECTORY)); + ASSERT_EQ(0, rename(dir_s1d3, dir_s4d2)) + { + TH_LOG("Failed to rename %s to %s: %s", dir_s1d3, dir_s4d2, + strerror(errno)); + } + EXPECT_EQ(ENOENT, test_open_rel(bind_s1d3_fd, "..", O_DIRECTORY)); + + /* + * The file is no longer under s1d2 but we should still be able to access it + * with layer 2 because its mount point is evaluated as the first valid + * directory because it was initially a parent. Do a fork to test this so + * we don't prevent ourselves from renaming it back later. + */ + child_pid = fork(); + ASSERT_LE(0, child_pid); + if (child_pid == 0) { + enforce_ruleset(_metadata, ruleset_fd_l2); + EXPECT_EQ(0, close(ruleset_fd_l2)); + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)); + EXPECT_EQ(EACCES, test_open(file1_s4d2, O_RDONLY)); + + /* + * Tests that access widening checks indeed prevents us from renaming it + * back. + */ + EXPECT_EQ(-1, rename(dir_s4d2, dir_s1d3)); + EXPECT_EQ(EXDEV, errno); + + /* + * Including through the now disconnected fd (but it should return + * EXDEV). + */ + EXPECT_EQ(-1, renameat(bind_s1d3_fd, file1_name, AT_FDCWD, + file1_s2d2)); + EXPECT_EQ(EXDEV, errno); + _exit(_metadata->exit_code); + return; + } + + EXPECT_EQ(child_pid, waitpid(child_pid, &status, 0)); + EXPECT_EQ(1, WIFEXITED(status)); + EXPECT_EQ(EXIT_SUCCESS, WEXITSTATUS(status)); + + ASSERT_EQ(0, rename(dir_s4d2, dir_s1d3)) + { + TH_LOG("Failed to rename %s back to %s: %s", dir_s4d1, dir_s1d3, + strerror(errno)); + } + + /* Now checks that we can access it under l2. */ + child_pid = fork(); + ASSERT_LE(0, child_pid); + if (child_pid == 0) { + enforce_ruleset(_metadata, ruleset_fd_l2); + EXPECT_EQ(0, close(ruleset_fd_l2)); + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)); + EXPECT_EQ(0, test_open(file1_s1d3, O_RDONLY)); + _exit(_metadata->exit_code); + return; + } + + EXPECT_EQ(child_pid, waitpid(child_pid, &status, 0)); + EXPECT_EQ(1, WIFEXITED(status)); + EXPECT_EQ(EXIT_SUCCESS, WEXITSTATUS(status)); + + /* + * Also test that we can rename via a disconnected path. We move the + * dir back to the disconnected place first, then we rename file1 to + * file2 through our dir fd. + */ + ASSERT_EQ(0, rename(dir_s1d3, dir_s4d2)) + { + TH_LOG("Failed to rename %s to %s: %s", dir_s1d3, dir_s4d2, + strerror(errno)); + } + ASSERT_EQ(0, + renameat(bind_s1d3_fd, file1_name, bind_s1d3_fd, file2_name)) + { + TH_LOG("Failed to rename %s to %s within disconnected %s: %s", + file1_name, file2_name, bind_dir_s1d3, strerror(errno)); + } + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file2_name, O_RDONLY)); + ASSERT_EQ(0, renameat(bind_s1d3_fd, file2_name, AT_FDCWD, file1_s2d2)) + { + TH_LOG("Failed to rename %s to %s through disconnected %s: %s", + file2_name, file1_s2d2, bind_dir_s1d3, strerror(errno)); + } + EXPECT_EQ(0, test_open(file1_s2d2, O_RDONLY)); + EXPECT_EQ(0, test_open(file1_s1d2, O_RDONLY)); + + /* Move it back using the disconnected path as the target. */ + ASSERT_EQ(0, renameat(AT_FDCWD, file1_s2d2, bind_s1d3_fd, file1_name)) + { + TH_LOG("Failed to rename %s to %s through disconnected %s: %s", + file1_s1d2, file1_name, bind_dir_s1d3, strerror(errno)); + } + + /* Now make it connected again. */ + ASSERT_EQ(0, rename(dir_s4d2, dir_s1d3)) + { + TH_LOG("Failed to rename %s back to %s: %s", dir_s4d2, dir_s1d3, + strerror(errno)); + } + + /* Checks again that we can access it under l2. */ + enforce_ruleset(_metadata, ruleset_fd_l2); + EXPECT_EQ(0, close(ruleset_fd_l2)); + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)); + EXPECT_EQ(0, test_open(file1_s1d3, O_RDONLY)); +} + +/* + * Test that linkat(2) with disconnected paths works under Landlock. This + * test moves s1d3 to s4d1. + */ +TEST_F_FORK(layout1_bind, path_disconnected_link) +{ + /* Ruleset to be applied after renaming s1d3 to s4d1. */ + const struct rule layer1[] = { + { + .path = dir_s4d1, + .access = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG | + LANDLOCK_ACCESS_FS_REMOVE_FILE, + }, + { + .path = dir_s2d2, + .access = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG | + LANDLOCK_ACCESS_FS_REMOVE_FILE, + }, + {} + }; + int ruleset_fd, bind_s1d3_fd; + + /* Removes unneeded files created by layout1, otherwise it will EEXIST. */ + ASSERT_EQ(0, unlink(file1_s1d2)); + ASSERT_EQ(0, unlink(file2_s1d3)); + + bind_s1d3_fd = open(bind_dir_s1d3, O_PATH | O_CLOEXEC); + ASSERT_LE(0, bind_s1d3_fd); + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)); + + /* Disconnects bind_s1d3_fd. */ + ASSERT_EQ(0, rename(dir_s1d3, dir_s4d1)) + { + TH_LOG("Failed to rename %s to %s: %s", dir_s1d3, dir_s4d1, + strerror(errno)); + } + + /* Need this later to test different parent link. */ + ASSERT_EQ(0, mkdir(dir_s4d2, 0755)) + { + TH_LOG("Failed to create %s: %s", dir_s4d2, strerror(errno)); + } + + ruleset_fd = create_ruleset(_metadata, ACCESS_ALL, layer1); + ASSERT_LE(0, ruleset_fd); + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + + /* From disconnected to connected. */ + ASSERT_EQ(0, linkat(bind_s1d3_fd, file1_name, AT_FDCWD, file1_s2d2, 0)) + { + TH_LOG("Failed to link %s to %s via disconnected %s: %s", + file1_name, file1_s2d2, bind_dir_s1d3, strerror(errno)); + } + + /* Tests that we can access via the new link... */ + EXPECT_EQ(0, test_open(file1_s2d2, O_RDONLY)) + { + TH_LOG("Failed to open newly linked %s: %s", file1_s2d2, + strerror(errno)); + } + + /* ...as well as the old one. */ + EXPECT_EQ(0, test_open(file1_s4d1, O_RDONLY)) + { + TH_LOG("Failed to open original %s: %s", file1_s4d1, + strerror(errno)); + } + + /* From connected to disconnected. */ + ASSERT_EQ(0, unlink(file1_s4d1)); + ASSERT_EQ(0, linkat(AT_FDCWD, file1_s2d2, bind_s1d3_fd, file2_name, 0)) + { + TH_LOG("Failed to link %s to %s via disconnected %s: %s", + file1_s2d2, file2_name, bind_dir_s1d3, strerror(errno)); + } + EXPECT_EQ(0, test_open(file2_s4d1, O_RDONLY)); + ASSERT_EQ(0, unlink(file1_s2d2)); + + /* From disconnected to disconnected (same parent). */ + ASSERT_EQ(0, + linkat(bind_s1d3_fd, file2_name, bind_s1d3_fd, file1_name, 0)) + { + TH_LOG("Failed to link %s to %s within disconnected %s: %s", + file2_name, file1_name, bind_dir_s1d3, strerror(errno)); + } + EXPECT_EQ(0, test_open(file1_s4d1, O_RDONLY)) + { + TH_LOG("Failed to open newly linked %s: %s", file1_s4d1, + strerror(errno)); + } + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, file1_name, O_RDONLY)) + { + TH_LOG("Failed to open %s through newly created link under disconnected path: %s", + file1_name, strerror(errno)); + } + ASSERT_EQ(0, unlink(file2_s4d1)); + + /* From disconnected to disconnected (different parent). */ + ASSERT_EQ(0, + linkat(bind_s1d3_fd, file1_name, bind_s1d3_fd, "s4d2/f1", 0)) + { + TH_LOG("Failed to link %s to %s within disconnected %s: %s", + file1_name, "s4d2/f1", bind_dir_s1d3, strerror(errno)); + } + EXPECT_EQ(0, test_open(file1_s4d2, O_RDONLY)) + { + TH_LOG("Failed to open %s after link: %s", file1_s4d2, + strerror(errno)); + } + EXPECT_EQ(0, test_open_rel(bind_s1d3_fd, "s4d2/f1", O_RDONLY)) + { + TH_LOG("Failed to open %s through disconnected path after link: %s", + "s4d2/f1", strerror(errno)); + } +} + +/* + * layout4_disconnected_leafs with bind mount and renames: + * + * tmp + * ├── s1d1 + * │  └── s1d2 [source of the bind mount] + * │ ├── s1d31 + * │  │ └── s1d41 [now renamed beneath s3d1] + * │ │ ├── f1 + * │ │ └── f2 + * │  └── s1d32 + * │ └── s1d42 [now renamed beneath s4d1] + * │ ├── f3 + * │ └── f4 + * ├── s2d1 + * │  └── s2d2 [bind mount of s1d2] + * │ ├── s1d31 + * │  │ └── s1d41 [opened FD, now renamed beneath s3d1] + * │ │ ├── f1 + * │ │ └── f2 + * │  └── s1d32 + * │ └── s1d42 [opened FD, now renamed beneath s4d1] + * │ ├── f3 + * │ └── f4 + * ├── s3d1 + * │ └── s1d41 [renamed here] + * │ ├── f1 + * │ └── f2 + * └── s4d1 + * └── s1d42 [renamed here] + * ├── f3 + * └── f4 + */ +/* clang-format off */ +FIXTURE(layout4_disconnected_leafs) { + int s2d2_fd; +}; +/* clang-format on */ + +FIXTURE_SETUP(layout4_disconnected_leafs) +{ + prepare_layout(_metadata); + + create_file(_metadata, TMP_DIR "/s1d1/s1d2/s1d31/s1d41/f1"); + create_file(_metadata, TMP_DIR "/s1d1/s1d2/s1d31/s1d41/f2"); + create_file(_metadata, TMP_DIR "/s1d1/s1d2/s1d32/s1d42/f3"); + create_file(_metadata, TMP_DIR "/s1d1/s1d2/s1d32/s1d42/f4"); + create_directory(_metadata, TMP_DIR "/s2d1/s2d2"); + create_directory(_metadata, TMP_DIR "/s3d1"); + create_directory(_metadata, TMP_DIR "/s4d1"); + + self->s2d2_fd = + open(TMP_DIR "/s2d1/s2d2", O_DIRECTORY | O_PATH | O_CLOEXEC); + ASSERT_LE(0, self->s2d2_fd); + + set_cap(_metadata, CAP_SYS_ADMIN); + ASSERT_EQ(0, mount(TMP_DIR "/s1d1/s1d2", TMP_DIR "/s2d1/s2d2", NULL, + MS_BIND, NULL)); + clear_cap(_metadata, CAP_SYS_ADMIN); +} + +FIXTURE_TEARDOWN_PARENT(layout4_disconnected_leafs) +{ + /* umount(TMP_DIR "/s2d1") is handled by namespace lifetime. */ + + /* Removes files after renames. */ + remove_path(TMP_DIR "/s3d1/s1d41/f1"); + remove_path(TMP_DIR "/s3d1/s1d41/f2"); + remove_path(TMP_DIR "/s4d1/s1d42/f1"); + remove_path(TMP_DIR "/s4d1/s1d42/f3"); + remove_path(TMP_DIR "/s4d1/s1d42/f4"); + remove_path(TMP_DIR "/s4d1/s1d42/f5"); + + cleanup_layout(_metadata); +} + +FIXTURE_VARIANT(layout4_disconnected_leafs) +{ + /* + * Parent of the bind mount source. It should always be ignored when + * testing against files under the s1d41 or s1d42 disconnected directories. + */ + const __u64 allowed_s1d1; + /* + * Source of bind mount (to s2d2). It should always be enforced when + * testing against files under the s1d41 or s1d42 disconnected directories. + */ + const __u64 allowed_s1d2; + /* + * Original parent of s1d41. It should always be ignored when testing + * against files under the s1d41 disconnected directory. + */ + const __u64 allowed_s1d31; + /* + * Original parent of s1d42. It should always be ignored when testing + * against files under the s1d42 disconnected directory. + */ + const __u64 allowed_s1d32; + /* + * Opened and disconnected source directory. It should always be enforced + * when testing against files under the s1d41 disconnected directory. + */ + const __u64 allowed_s1d41; + /* + * Opened and disconnected source directory. It should always be enforced + * when testing against files under the s1d42 disconnected directory. + */ + const __u64 allowed_s1d42; + /* + * File in the s1d41 disconnected directory. It should always be enforced + * when testing against itself under the s1d41 disconnected directory. + */ + const __u64 allowed_f1; + /* + * File in the s1d41 disconnected directory. It should always be enforced + * when testing against itself under the s1d41 disconnected directory. + */ + const __u64 allowed_f2; + /* + * File in the s1d42 disconnected directory. It should always be enforced + * when testing against itself under the s1d42 disconnected directory. + */ + const __u64 allowed_f3; + /* + * Parent of the bind mount destination. It should always be enforced when + * testing against files under the s1d41 or s1d42 disconnected directories. + */ + const __u64 allowed_s2d1; + /* + * Directory covered by the bind mount. It should always be ignored when + * testing against files under the s1d41 or s1d42 disconnected directories. + */ + const __u64 allowed_s2d2; + /* + * New parent of the renamed s1d41. It should always be ignored when + * testing against files under the s1d41 disconnected directory. + */ + const __u64 allowed_s3d1; + /* + * New parent of the renamed s1d42. It should always be ignored when + * testing against files under the s1d42 disconnected directory. + */ + const __u64 allowed_s4d1; + + /* Expected result of the call to open([fd:s1d41]/f1, O_RDONLY). */ + const int expected_read_result; + /* Expected result of the call to renameat([fd:s1d41]/f1, [fd:s1d42]/f1). */ + const int expected_rename_result; + /* + * Expected result of the call to renameat([fd:s1d41]/f2, [fd:s1d42]/f3, + * RENAME_EXCHANGE). + */ + const int expected_exchange_result; + /* Expected result of the call to renameat([fd:s1d42]/f4, [fd:s1d42]/f5). */ + const int expected_same_dir_rename_result; +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d1_mount_src_parent) { + /* clang-format on */ + .allowed_s1d1 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d2_mount_src_refer) { + /* clang-format on */ + .allowed_s1d2 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d2_mount_src_create) { + /* clang-format on */ + .allowed_s1d2 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d2_mount_src_rename) { + /* clang-format on */ + .allowed_s1d2 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d31_s1d32_old_parent) { + /* clang-format on */ + .allowed_s1d31 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .allowed_s1d32 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d41_s1d42_disconnected_refer) { + /* clang-format on */ + .allowed_s1d41 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE, + .allowed_s1d42 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d41_s1d42_disconnected_create) { + /* clang-format on */ + .allowed_s1d41 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .allowed_s1d42 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d41_s1d42_disconnected_rename_even) { + /* clang-format on */ + .allowed_s1d41 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .allowed_s1d42 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* The destination directory has more access right. */ +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d41_s1d42_disconnected_rename_more) { + /* clang-format on */ + .allowed_s1d41 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .allowed_s1d42 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_MAKE_REG | + LANDLOCK_ACCESS_FS_EXECUTE, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + /* Access denied. */ + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* The destination directory has less access right. */ +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s1d41_s1d42_disconnected_rename_less) { + /* clang-format on */ + .allowed_s1d41 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_MAKE_REG | + LANDLOCK_ACCESS_FS_EXECUTE, + .allowed_s1d42 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + /* Access allowed. */ + .expected_rename_result = 0, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s2d1_mount_dst_parent_create) { + /* clang-format on */ + .allowed_s2d1 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s2d1_mount_dst_parent_refer) { + /* clang-format on */ + .allowed_s2d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s2d1_mount_dst_parent_mini) { + /* clang-format on */ + .allowed_s2d1 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s2d2_covered_by_mount) { + /* clang-format on */ + .allowed_s2d2 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* Tests collect_domain_accesses(). */ +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s3d1_s4d1_new_parent_refer) { + /* clang-format on */ + .allowed_s3d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .allowed_s4d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s3d1_s4d1_new_parent_create) { + /* clang-format on */ + .allowed_s3d1 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .allowed_s4d1 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, + s3d1_s4d1_disconnected_rename_even){ + /* clang-format on */ + .allowed_s3d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .allowed_s4d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* The destination directory has more access right. */ +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s3d1_s4d1_disconnected_rename_more) { + /* clang-format on */ + .allowed_s3d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .allowed_s4d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG | + LANDLOCK_ACCESS_FS_EXECUTE, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + /* Access denied. */ + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* The destination directory has less access right. */ +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, s3d1_s4d1_disconnected_rename_less) { + /* clang-format on */ + .allowed_s3d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG | + LANDLOCK_ACCESS_FS_EXECUTE, + .allowed_s4d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + /* Access allowed. */ + .expected_rename_result = 0, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout4_disconnected_leafs, f1_f2_f3) { + /* clang-format on */ + .allowed_f1 = LANDLOCK_ACCESS_FS_READ_FILE, + .allowed_f2 = LANDLOCK_ACCESS_FS_READ_FILE, + .allowed_f3 = LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +TEST_F_FORK(layout4_disconnected_leafs, read_rename_exchange) +{ + const __u64 handled_access = + LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | LANDLOCK_ACCESS_FS_MAKE_REG; + const struct rule rules[] = { + { + .path = TMP_DIR "/s1d1", + .access = variant->allowed_s1d1, + }, + { + .path = TMP_DIR "/s1d1/s1d2", + .access = variant->allowed_s1d2, + }, + { + .path = TMP_DIR "/s1d1/s1d2/s1d31", + .access = variant->allowed_s1d31, + }, + { + .path = TMP_DIR "/s1d1/s1d2/s1d32", + .access = variant->allowed_s1d32, + }, + { + .path = TMP_DIR "/s1d1/s1d2/s1d31/s1d41", + .access = variant->allowed_s1d41, + }, + { + .path = TMP_DIR "/s1d1/s1d2/s1d32/s1d42", + .access = variant->allowed_s1d42, + }, + { + .path = TMP_DIR "/s1d1/s1d2/s1d31/s1d41/f1", + .access = variant->allowed_f1, + }, + { + .path = TMP_DIR "/s1d1/s1d2/s1d31/s1d41/f2", + .access = variant->allowed_f2, + }, + { + .path = TMP_DIR "/s1d1/s1d2/s1d32/s1d42/f3", + .access = variant->allowed_f3, + }, + { + .path = TMP_DIR "/s2d1", + .access = variant->allowed_s2d1, + }, + /* s2d2_fd */ + { + .path = TMP_DIR "/s3d1", + .access = variant->allowed_s3d1, + }, + { + .path = TMP_DIR "/s4d1", + .access = variant->allowed_s4d1, + }, + {}, + }; + int ruleset_fd, s1d41_bind_fd, s1d42_bind_fd; + + ruleset_fd = create_ruleset(_metadata, handled_access, rules); + ASSERT_LE(0, ruleset_fd); + + /* Adds rule for the covered directory. */ + if (variant->allowed_s2d2) { + ASSERT_EQ(0, landlock_add_rule( + ruleset_fd, LANDLOCK_RULE_PATH_BENEATH, + &(struct landlock_path_beneath_attr){ + .parent_fd = self->s2d2_fd, + .allowed_access = + variant->allowed_s2d2, + }, + 0)); + } + EXPECT_EQ(0, close(self->s2d2_fd)); + + s1d41_bind_fd = open(TMP_DIR "/s2d1/s2d2/s1d31/s1d41", + O_DIRECTORY | O_PATH | O_CLOEXEC); + ASSERT_LE(0, s1d41_bind_fd); + s1d42_bind_fd = open(TMP_DIR "/s2d1/s2d2/s1d32/s1d42", + O_DIRECTORY | O_PATH | O_CLOEXEC); + ASSERT_LE(0, s1d42_bind_fd); + + /* Disconnects and checks source and destination directories. */ + EXPECT_EQ(0, test_open_rel(s1d41_bind_fd, "..", O_DIRECTORY)); + EXPECT_EQ(0, test_open_rel(s1d42_bind_fd, "..", O_DIRECTORY)); + /* Renames to make it accessible through s3d1/s1d41 */ + ASSERT_EQ(0, test_renameat(AT_FDCWD, TMP_DIR "/s1d1/s1d2/s1d31/s1d41", + AT_FDCWD, TMP_DIR "/s3d1/s1d41")); + /* Renames to make it accessible through s4d1/s1d42 */ + ASSERT_EQ(0, test_renameat(AT_FDCWD, TMP_DIR "/s1d1/s1d2/s1d32/s1d42", + AT_FDCWD, TMP_DIR "/s4d1/s1d42")); + EXPECT_EQ(ENOENT, test_open_rel(s1d41_bind_fd, "..", O_DIRECTORY)); + EXPECT_EQ(ENOENT, test_open_rel(s1d42_bind_fd, "..", O_DIRECTORY)); + + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + + EXPECT_EQ(variant->expected_read_result, + test_open_rel(s1d41_bind_fd, "f1", O_RDONLY)); + + EXPECT_EQ(variant->expected_rename_result, + test_renameat(s1d41_bind_fd, "f1", s1d42_bind_fd, "f1")); + EXPECT_EQ(variant->expected_exchange_result, + test_exchangeat(s1d41_bind_fd, "f2", s1d42_bind_fd, "f3")); + + EXPECT_EQ(variant->expected_same_dir_rename_result, + test_renameat(s1d42_bind_fd, "f4", s1d42_bind_fd, "f5")); +} + +/* + * layout5_disconnected_branch before rename: + * + * tmp + * ├── s1d1 + * │  └── s1d2 [source of the first bind mount] + * │  └── s1d3 + * │  ├── s1d41 + * │  │  ├── f1 + * │  │  └── f2 + * │  └── s1d42 + * │  ├── f3 + * │  └── f4 + * ├── s2d1 + * │  └── s2d2 [source of the second bind mount] + * │  └── s2d3 + * │  └── s2d4 [first s1d2 bind mount] + * │  └── s1d3 + * │  ├── s1d41 + * │  │  ├── f1 + * │  │  └── f2 + * │  └── s1d42 + * │  ├── f3 + * │  └── f4 + * ├── s3d1 + * │  └── s3d2 [second s2d2 bind mount] + * │  └── s2d3 + * │  └── s2d4 [first s1d2 bind mount] + * │  └── s1d3 + * │  ├── s1d41 + * │  │  ├── f1 + * │  │  └── f2 + * │  └── s1d42 + * │  ├── f3 + * │  └── f4 + * └── s4d1 + * + * After rename: + * + * tmp + * ├── s1d1 + * │  └── s1d2 [source of the first bind mount] + * │  └── s1d3 + * │  ├── s1d41 + * │  │  ├── f1 + * │  │  └── f2 + * │  └── s1d42 + * │  ├── f3 + * │  └── f4 + * ├── s2d1 + * │  └── s2d2 [source of the second bind mount] + * ├── s3d1 + * │  └── s3d2 [second s2d2 bind mount] + * └── s4d1 + * └── s2d3 [renamed here] + * └── s2d4 [first s1d2 bind mount] + * └── s1d3 + * ├── s1d41 + * │  ├── f1 + * │  └── f2 + * └── s1d42 + * ├── f3 + * └── f4 + * + * Decision path for access from the s3d1/s3d2/s2d3/s2d4/s1d3 file descriptor: + * 1. first bind mount: s1d3 -> s1d2 + * 2. second bind mount: s2d3 + * 3. tmp mount: s4d1 -> tmp [disconnected branch] + * 4. second bind mount: s2d2 + * 5. tmp mount: s3d1 -> tmp + * 6. parent mounts: [...] -> / + * + * The s4d1 directory is evaluated even if it is not in the s2d2 mount. + */ + +/* clang-format off */ +FIXTURE(layout5_disconnected_branch) { + int s2d4_fd, s3d2_fd; +}; +/* clang-format on */ + +FIXTURE_SETUP(layout5_disconnected_branch) +{ + prepare_layout(_metadata); + + create_file(_metadata, TMP_DIR "/s1d1/s1d2/s1d3/s1d41/f1"); + create_file(_metadata, TMP_DIR "/s1d1/s1d2/s1d3/s1d41/f2"); + create_file(_metadata, TMP_DIR "/s1d1/s1d2/s1d3/s1d42/f3"); + create_file(_metadata, TMP_DIR "/s1d1/s1d2/s1d3/s1d42/f4"); + create_directory(_metadata, TMP_DIR "/s2d1/s2d2/s2d3/s2d4"); + create_directory(_metadata, TMP_DIR "/s3d1/s3d2"); + create_directory(_metadata, TMP_DIR "/s4d1"); + + self->s2d4_fd = open(TMP_DIR "/s2d1/s2d2/s2d3/s2d4", + O_DIRECTORY | O_PATH | O_CLOEXEC); + ASSERT_LE(0, self->s2d4_fd); + + self->s3d2_fd = + open(TMP_DIR "/s3d1/s3d2", O_DIRECTORY | O_PATH | O_CLOEXEC); + ASSERT_LE(0, self->s3d2_fd); + + set_cap(_metadata, CAP_SYS_ADMIN); + ASSERT_EQ(0, mount(TMP_DIR "/s1d1/s1d2", TMP_DIR "/s2d1/s2d2/s2d3/s2d4", + NULL, MS_BIND, NULL)); + ASSERT_EQ(0, mount(TMP_DIR "/s2d1/s2d2", TMP_DIR "/s3d1/s3d2", NULL, + MS_BIND | MS_REC, NULL)); + clear_cap(_metadata, CAP_SYS_ADMIN); +} + +FIXTURE_TEARDOWN_PARENT(layout5_disconnected_branch) +{ + /* Bind mounts are handled by namespace lifetime. */ + + /* Removes files after renames. */ + remove_path(TMP_DIR "/s1d1/s1d2/s1d3/s1d41/f1"); + remove_path(TMP_DIR "/s1d1/s1d2/s1d3/s1d41/f2"); + remove_path(TMP_DIR "/s1d1/s1d2/s1d3/s1d42/f1"); + remove_path(TMP_DIR "/s1d1/s1d2/s1d3/s1d42/f3"); + remove_path(TMP_DIR "/s1d1/s1d2/s1d3/s1d42/f4"); + remove_path(TMP_DIR "/s1d1/s1d2/s1d3/s1d42/f5"); + + cleanup_layout(_metadata); +} + +FIXTURE_VARIANT(layout5_disconnected_branch) +{ + /* + * Parent of all files. It should always be enforced when testing against + * files under the s1d41 or s1d42 disconnected directories. + */ + const __u64 allowed_base; + /* + * Parent of the first bind mount source. It should always be ignored when + * testing against files under the s1d41 or s1d42 disconnected directories. + */ + const __u64 allowed_s1d1; + const __u64 allowed_s1d2; + const __u64 allowed_s1d3; + const __u64 allowed_s2d1; + const __u64 allowed_s2d2; + const __u64 allowed_s2d3; + const __u64 allowed_s2d4; + const __u64 allowed_s3d1; + const __u64 allowed_s3d2; + const __u64 allowed_s4d1; + + /* Expected result of the call to open([fd:s1d3]/s1d41/f1, O_RDONLY). */ + const int expected_read_result; + /* + * Expected result of the call to renameat([fd:s1d3]/s1d41/f1, + * [fd:s1d3]/s1d42/f1). + */ + const int expected_rename_result; + /* + * Expected result of the call to renameat([fd:s1d3]/s1d41/f2, + * [fd:s1d3]/s1d42/f3, RENAME_EXCHANGE). + */ + const int expected_exchange_result; + /* + * Expected result of the call to renameat([fd:s1d3]/s1d42/f4, + * [fd:s1d3]/s1d42/f5). + */ + const int expected_same_dir_rename_result; +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s1d1_mount1_src_parent) { + /* clang-format on */ + .allowed_s1d1 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s1d2_mount1_src_refer) { + /* clang-format on */ + .allowed_s1d2 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s1d2_mount1_src_create) { + /* clang-format on */ + .allowed_s1d2 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s1d2_mount1_src_rename) { + /* clang-format on */ + .allowed_s1d2 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s1d3_fd_refer) { + /* clang-format on */ + .allowed_s1d3 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s1d3_fd_create) { + /* clang-format on */ + .allowed_s1d3 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s1d3_fd_rename) { + /* clang-format on */ + .allowed_s1d3 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s1d3_fd_full) { + /* clang-format on */ + .allowed_s1d3 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s2d1_mount2_src_parent) { + /* clang-format on */ + .allowed_s2d1 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s2d2_mount2_src_refer) { + /* clang-format on */ + .allowed_s2d2 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s2d2_mount2_src_create) { + /* clang-format on */ + .allowed_s2d2 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s2d2_mount2_src_rename) { + /* clang-format on */ + .allowed_s2d2 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s2d3_mount1_dst_parent_refer) { + /* clang-format on */ + .allowed_s2d3 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s2d3_mount1_dst_parent_create) { + /* clang-format on */ + .allowed_s2d3 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s2d3_mount1_dst_parent_rename) { + /* clang-format on */ + .allowed_s2d3 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s2d4_mount1_dst) { + /* clang-format on */ + .allowed_s2d4 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s3d1_mount2_dst_parent_refer) { + /* clang-format on */ + .allowed_s3d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s3d1_mount2_dst_parent_create) { + /* clang-format on */ + .allowed_s3d1 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s3d1_mount2_dst_parent_rename) { + /* clang-format on */ + .allowed_s3d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s3d2_mount1_dst) { + /* clang-format on */ + .allowed_s3d2 = LANDLOCK_ACCESS_FS_REFER | + LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s4d1_rename_parent_refer) { + /* clang-format on */ + .allowed_s4d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE, + .expected_read_result = 0, + .expected_same_dir_rename_result = EACCES, + .expected_rename_result = EACCES, + .expected_exchange_result = EACCES, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s4d1_rename_parent_create) { + /* clang-format on */ + .allowed_s4d1 = LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = 0, + .expected_same_dir_rename_result = 0, + .expected_rename_result = EXDEV, + .expected_exchange_result = EXDEV, +}; + +/* clang-format off */ +FIXTURE_VARIANT_ADD(layout5_disconnected_branch, s4d1_rename_parent_rename) { + /* clang-format on */ + .allowed_s4d1 = LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_MAKE_REG, + .expected_read_result = EACCES, + .expected_same_dir_rename_result = 0, + .expected_rename_result = 0, + .expected_exchange_result = 0, +}; + +TEST_F_FORK(layout5_disconnected_branch, read_rename_exchange) +{ + const __u64 handled_access = + LANDLOCK_ACCESS_FS_REFER | LANDLOCK_ACCESS_FS_READ_FILE | + LANDLOCK_ACCESS_FS_EXECUTE | LANDLOCK_ACCESS_FS_MAKE_REG; + const struct rule rules[] = { + { + .path = TMP_DIR "/s1d1", + .access = variant->allowed_s1d1, + }, + { + .path = TMP_DIR "/s1d1/s1d2", + .access = variant->allowed_s1d2, + }, + { + .path = TMP_DIR "/s1d1/s1d2/s1d3", + .access = variant->allowed_s1d3, + }, + { + .path = TMP_DIR "/s2d1", + .access = variant->allowed_s2d1, + }, + { + .path = TMP_DIR "/s2d1/s2d2", + .access = variant->allowed_s2d2, + }, + { + .path = TMP_DIR "/s2d1/s2d2/s2d3", + .access = variant->allowed_s2d3, + }, + /* s2d4_fd */ + { + .path = TMP_DIR "/s3d1", + .access = variant->allowed_s3d1, + }, + /* s3d2_fd */ + { + .path = TMP_DIR "/s4d1", + .access = variant->allowed_s4d1, + }, + {}, + }; + int ruleset_fd, s1d3_bind_fd; + + ruleset_fd = create_ruleset(_metadata, handled_access, rules); + ASSERT_LE(0, ruleset_fd); + + /* Adds rules for the covered directories. */ + if (variant->allowed_s2d4) { + ASSERT_EQ(0, landlock_add_rule( + ruleset_fd, LANDLOCK_RULE_PATH_BENEATH, + &(struct landlock_path_beneath_attr){ + .parent_fd = self->s2d4_fd, + .allowed_access = + variant->allowed_s2d4, + }, + 0)); + } + EXPECT_EQ(0, close(self->s2d4_fd)); + + if (variant->allowed_s3d2) { + ASSERT_EQ(0, landlock_add_rule( + ruleset_fd, LANDLOCK_RULE_PATH_BENEATH, + &(struct landlock_path_beneath_attr){ + .parent_fd = self->s3d2_fd, + .allowed_access = + variant->allowed_s3d2, + }, + 0)); + } + EXPECT_EQ(0, close(self->s3d2_fd)); + + s1d3_bind_fd = open(TMP_DIR "/s3d1/s3d2/s2d3/s2d4/s1d3", + O_DIRECTORY | O_PATH | O_CLOEXEC); + ASSERT_LE(0, s1d3_bind_fd); + + /* Disconnects and checks source and destination directories. */ + EXPECT_EQ(0, test_open_rel(s1d3_bind_fd, "..", O_DIRECTORY)); + EXPECT_EQ(0, test_open_rel(s1d3_bind_fd, "../..", O_DIRECTORY)); + /* Renames to make it accessible through s3d1/s1d41 */ + ASSERT_EQ(0, test_renameat(AT_FDCWD, TMP_DIR "/s2d1/s2d2/s2d3", + AT_FDCWD, TMP_DIR "/s4d1/s2d3")); + EXPECT_EQ(0, test_open_rel(s1d3_bind_fd, "..", O_DIRECTORY)); + EXPECT_EQ(ENOENT, test_open_rel(s1d3_bind_fd, "../..", O_DIRECTORY)); + + enforce_ruleset(_metadata, ruleset_fd); + EXPECT_EQ(0, close(ruleset_fd)); + + EXPECT_EQ(variant->expected_read_result, + test_open_rel(s1d3_bind_fd, "s1d41/f1", O_RDONLY)); + + EXPECT_EQ(variant->expected_rename_result, + test_renameat(s1d3_bind_fd, "s1d41/f1", s1d3_bind_fd, + "s1d42/f1")); + EXPECT_EQ(variant->expected_exchange_result, + test_exchangeat(s1d3_bind_fd, "s1d41/f2", s1d3_bind_fd, + "s1d42/f3")); + + EXPECT_EQ(variant->expected_same_dir_rename_result, + test_renameat(s1d3_bind_fd, "s1d42/f4", s1d3_bind_fd, + "s1d42/f5")); +} + #define LOWER_BASE TMP_DIR "/lower" #define LOWER_DATA LOWER_BASE "/data" static const char lower_fl1[] = LOWER_DATA "/fl1"; diff --git a/tools/testing/selftests/lib.mk b/tools/testing/selftests/lib.mk index a448fae57831..f02cc8a2e4ae 100644 --- a/tools/testing/selftests/lib.mk +++ b/tools/testing/selftests/lib.mk @@ -199,6 +199,9 @@ clean: $(if $(TEST_GEN_MODS_DIR),clean_mods_dir) # Build with _GNU_SOURCE by default CFLAGS += -D_GNU_SOURCE= +# Additional include paths needed by kselftest.h and local headers +CFLAGS += -I${top_srcdir}/tools/testing/selftests + # Enables to extend CFLAGS and LDFLAGS from command line, e.g. # make USERCFLAGS=-Werror USERLDFLAGS=-static CFLAGS += $(USERCFLAGS) diff --git a/tools/testing/selftests/liveupdate/.gitignore b/tools/testing/selftests/liveupdate/.gitignore new file mode 100644 index 000000000000..661827083ab6 --- /dev/null +++ b/tools/testing/selftests/liveupdate/.gitignore @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +* +!/**/ +!*.c +!*.h +!*.sh +!.gitignore +!config +!Makefile diff --git a/tools/testing/selftests/liveupdate/Makefile b/tools/testing/selftests/liveupdate/Makefile new file mode 100644 index 000000000000..080754787ede --- /dev/null +++ b/tools/testing/selftests/liveupdate/Makefile @@ -0,0 +1,34 @@ +# SPDX-License-Identifier: GPL-2.0-only + +LIB_C += luo_test_utils.c + +TEST_GEN_PROGS += liveupdate + +TEST_GEN_PROGS_EXTENDED += luo_kexec_simple +TEST_GEN_PROGS_EXTENDED += luo_multi_session + +TEST_FILES += do_kexec.sh + +include ../lib.mk + +CFLAGS += $(KHDR_INCLUDES) +CFLAGS += -Wall -O2 -Wno-unused-function +CFLAGS += -MD + +LIB_O := $(patsubst %.c, $(OUTPUT)/%.o, $(LIB_C)) +TEST_O := $(patsubst %, %.o, $(TEST_GEN_PROGS)) +TEST_O += $(patsubst %, %.o, $(TEST_GEN_PROGS_EXTENDED)) + +TEST_DEP_FILES := $(patsubst %.o, %.d, $(LIB_O)) +TEST_DEP_FILES += $(patsubst %.o, %.d, $(TEST_O)) +-include $(TEST_DEP_FILES) + +$(LIB_O): $(OUTPUT)/%.o: %.c + $(CC) $(CFLAGS) $(CPPFLAGS) $(TARGET_ARCH) -c $< -o $@ + +$(TEST_GEN_PROGS) $(TEST_GEN_PROGS_EXTENDED): $(OUTPUT)/%: %.o $(LIB_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $(TARGET_ARCH) $< $(LIB_O) $(LDLIBS) -o $@ + +EXTRA_CLEAN += $(LIB_O) +EXTRA_CLEAN += $(TEST_O) +EXTRA_CLEAN += $(TEST_DEP_FILES) diff --git a/tools/testing/selftests/liveupdate/config b/tools/testing/selftests/liveupdate/config new file mode 100644 index 000000000000..91d03f9a6a39 --- /dev/null +++ b/tools/testing/selftests/liveupdate/config @@ -0,0 +1,11 @@ +CONFIG_BLK_DEV_INITRD=y +CONFIG_KEXEC_FILE=y +CONFIG_KEXEC_HANDOVER=y +CONFIG_KEXEC_HANDOVER_ENABLE_DEFAULT=y +CONFIG_KEXEC_HANDOVER_DEBUGFS=y +CONFIG_KEXEC_HANDOVER_DEBUG=y +CONFIG_LIVEUPDATE=y +CONFIG_LIVEUPDATE_TEST=y +CONFIG_MEMFD_CREATE=y +CONFIG_TMPFS=y +CONFIG_SHMEM=y diff --git a/tools/testing/selftests/liveupdate/do_kexec.sh b/tools/testing/selftests/liveupdate/do_kexec.sh new file mode 100755 index 000000000000..3c7c6cafbef8 --- /dev/null +++ b/tools/testing/selftests/liveupdate/do_kexec.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 +set -e + +# Use $KERNEL and $INITRAMFS to pass custom Kernel and optional initramfs + +KERNEL="${KERNEL:-/boot/bzImage}" +set -- -l -s --reuse-cmdline "$KERNEL" + +INITRAMFS="${INITRAMFS:-/boot/initramfs}" +if [ -f "$INITRAMFS" ]; then + set -- "$@" --initrd="$INITRAMFS" +fi + +kexec "$@" +kexec -e diff --git a/tools/testing/selftests/liveupdate/liveupdate.c b/tools/testing/selftests/liveupdate/liveupdate.c new file mode 100644 index 000000000000..c2878e3d5ef9 --- /dev/null +++ b/tools/testing/selftests/liveupdate/liveupdate.c @@ -0,0 +1,348 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +/* + * Selftests for the Live Update Orchestrator. + * This test suite verifies the functionality and behavior of the + * /dev/liveupdate character device and its session management capabilities. + * + * Tests include: + * - Device access: basic open/close, and enforcement of exclusive access. + * - Session management: creation of unique sessions, and duplicate name detection. + * - Resource preservation: successfully preserving individual and multiple memfds, + * verifying contents remain accessible. + * - Complex multi-session scenarios involving mixed empty and populated files. + */ + +#include <errno.h> +#include <fcntl.h> +#include <string.h> +#include <sys/ioctl.h> +#include <unistd.h> + +#include <linux/liveupdate.h> + +#include "../kselftest.h" +#include "../kselftest_harness.h" + +#define LIVEUPDATE_DEV "/dev/liveupdate" + +FIXTURE(liveupdate_device) { + int fd1; + int fd2; +}; + +FIXTURE_SETUP(liveupdate_device) +{ + self->fd1 = -1; + self->fd2 = -1; +} + +FIXTURE_TEARDOWN(liveupdate_device) +{ + if (self->fd1 >= 0) + close(self->fd1); + if (self->fd2 >= 0) + close(self->fd2); +} + +/* + * Test Case: Basic Open and Close + * + * Verifies that the /dev/liveupdate device can be opened and subsequently + * closed without errors. Skips if the device does not exist. + */ +TEST_F(liveupdate_device, basic_open_close) +{ + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist.", LIVEUPDATE_DEV); + + ASSERT_GE(self->fd1, 0); + ASSERT_EQ(close(self->fd1), 0); + self->fd1 = -1; +} + +/* + * Test Case: Exclusive Open Enforcement + * + * Verifies that the /dev/liveupdate device can only be opened by one process + * at a time. It checks that a second attempt to open the device fails with + * the EBUSY error code. + */ +TEST_F(liveupdate_device, exclusive_open) +{ + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist.", LIVEUPDATE_DEV); + + ASSERT_GE(self->fd1, 0); + self->fd2 = open(LIVEUPDATE_DEV, O_RDWR); + EXPECT_LT(self->fd2, 0); + EXPECT_EQ(errno, EBUSY); +} + +/* Helper function to create a LUO session via ioctl. */ +static int create_session(int lu_fd, const char *name) +{ + struct liveupdate_ioctl_create_session args = {}; + + args.size = sizeof(args); + strncpy((char *)args.name, name, sizeof(args.name) - 1); + + if (ioctl(lu_fd, LIVEUPDATE_IOCTL_CREATE_SESSION, &args)) + return -errno; + + return args.fd; +} + +/* + * Test Case: Create Duplicate Session + * + * Verifies that attempting to create two sessions with the same name fails + * on the second attempt with EEXIST. + */ +TEST_F(liveupdate_device, create_duplicate_session) +{ + int session_fd1, session_fd2; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + + ASSERT_GE(self->fd1, 0); + + session_fd1 = create_session(self->fd1, "duplicate-session-test"); + ASSERT_GE(session_fd1, 0); + + session_fd2 = create_session(self->fd1, "duplicate-session-test"); + EXPECT_LT(session_fd2, 0); + EXPECT_EQ(-session_fd2, EEXIST); + + ASSERT_EQ(close(session_fd1), 0); +} + +/* + * Test Case: Create Distinct Sessions + * + * Verifies that creating two sessions with different names succeeds. + */ +TEST_F(liveupdate_device, create_distinct_sessions) +{ + int session_fd1, session_fd2; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + + ASSERT_GE(self->fd1, 0); + + session_fd1 = create_session(self->fd1, "distinct-session-1"); + ASSERT_GE(session_fd1, 0); + + session_fd2 = create_session(self->fd1, "distinct-session-2"); + ASSERT_GE(session_fd2, 0); + + ASSERT_EQ(close(session_fd1), 0); + ASSERT_EQ(close(session_fd2), 0); +} + +static int preserve_fd(int session_fd, int fd_to_preserve, __u64 token) +{ + struct liveupdate_session_preserve_fd args = {}; + + args.size = sizeof(args); + args.fd = fd_to_preserve; + args.token = token; + + if (ioctl(session_fd, LIVEUPDATE_SESSION_PRESERVE_FD, &args)) + return -errno; + + return 0; +} + +/* + * Test Case: Preserve MemFD + * + * Verifies that a valid memfd can be successfully preserved in a session and + * that its contents remain intact after the preservation call. + */ +TEST_F(liveupdate_device, preserve_memfd) +{ + const char *test_str = "hello liveupdate"; + char read_buf[64] = {}; + int session_fd, mem_fd; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, "preserve-memfd-test"); + ASSERT_GE(session_fd, 0); + + mem_fd = memfd_create("test-memfd", 0); + ASSERT_GE(mem_fd, 0); + + ASSERT_EQ(write(mem_fd, test_str, strlen(test_str)), strlen(test_str)); + ASSERT_EQ(preserve_fd(session_fd, mem_fd, 0x1234), 0); + ASSERT_EQ(close(session_fd), 0); + + ASSERT_EQ(lseek(mem_fd, 0, SEEK_SET), 0); + ASSERT_EQ(read(mem_fd, read_buf, sizeof(read_buf)), strlen(test_str)); + ASSERT_STREQ(read_buf, test_str); + ASSERT_EQ(close(mem_fd), 0); +} + +/* + * Test Case: Preserve Multiple MemFDs + * + * Verifies that multiple memfds can be preserved in a single session, + * each with a unique token, and that their contents remain distinct and + * correct after preservation. + */ +TEST_F(liveupdate_device, preserve_multiple_memfds) +{ + const char *test_str1 = "data for memfd one"; + const char *test_str2 = "data for memfd two"; + char read_buf[64] = {}; + int session_fd, mem_fd1, mem_fd2; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, "preserve-multi-memfd-test"); + ASSERT_GE(session_fd, 0); + + mem_fd1 = memfd_create("test-memfd-1", 0); + ASSERT_GE(mem_fd1, 0); + mem_fd2 = memfd_create("test-memfd-2", 0); + ASSERT_GE(mem_fd2, 0); + + ASSERT_EQ(write(mem_fd1, test_str1, strlen(test_str1)), strlen(test_str1)); + ASSERT_EQ(write(mem_fd2, test_str2, strlen(test_str2)), strlen(test_str2)); + + ASSERT_EQ(preserve_fd(session_fd, mem_fd1, 0xAAAA), 0); + ASSERT_EQ(preserve_fd(session_fd, mem_fd2, 0xBBBB), 0); + + memset(read_buf, 0, sizeof(read_buf)); + ASSERT_EQ(lseek(mem_fd1, 0, SEEK_SET), 0); + ASSERT_EQ(read(mem_fd1, read_buf, sizeof(read_buf)), strlen(test_str1)); + ASSERT_STREQ(read_buf, test_str1); + + memset(read_buf, 0, sizeof(read_buf)); + ASSERT_EQ(lseek(mem_fd2, 0, SEEK_SET), 0); + ASSERT_EQ(read(mem_fd2, read_buf, sizeof(read_buf)), strlen(test_str2)); + ASSERT_STREQ(read_buf, test_str2); + + ASSERT_EQ(close(mem_fd1), 0); + ASSERT_EQ(close(mem_fd2), 0); + ASSERT_EQ(close(session_fd), 0); +} + +/* + * Test Case: Preserve Complex Scenario + * + * Verifies a more complex scenario with multiple sessions and a mix of empty + * and non-empty memfds distributed across them. + */ +TEST_F(liveupdate_device, preserve_complex_scenario) +{ + const char *data1 = "data for session 1"; + const char *data2 = "data for session 2"; + char read_buf[64] = {}; + int session_fd1, session_fd2; + int mem_fd_data1, mem_fd_empty1, mem_fd_data2, mem_fd_empty2; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd1 = create_session(self->fd1, "complex-session-1"); + ASSERT_GE(session_fd1, 0); + session_fd2 = create_session(self->fd1, "complex-session-2"); + ASSERT_GE(session_fd2, 0); + + mem_fd_data1 = memfd_create("data1", 0); + ASSERT_GE(mem_fd_data1, 0); + ASSERT_EQ(write(mem_fd_data1, data1, strlen(data1)), strlen(data1)); + + mem_fd_empty1 = memfd_create("empty1", 0); + ASSERT_GE(mem_fd_empty1, 0); + + mem_fd_data2 = memfd_create("data2", 0); + ASSERT_GE(mem_fd_data2, 0); + ASSERT_EQ(write(mem_fd_data2, data2, strlen(data2)), strlen(data2)); + + mem_fd_empty2 = memfd_create("empty2", 0); + ASSERT_GE(mem_fd_empty2, 0); + + ASSERT_EQ(preserve_fd(session_fd1, mem_fd_data1, 0x1111), 0); + ASSERT_EQ(preserve_fd(session_fd1, mem_fd_empty1, 0x2222), 0); + ASSERT_EQ(preserve_fd(session_fd2, mem_fd_data2, 0x3333), 0); + ASSERT_EQ(preserve_fd(session_fd2, mem_fd_empty2, 0x4444), 0); + + ASSERT_EQ(lseek(mem_fd_data1, 0, SEEK_SET), 0); + ASSERT_EQ(read(mem_fd_data1, read_buf, sizeof(read_buf)), strlen(data1)); + ASSERT_STREQ(read_buf, data1); + + memset(read_buf, 0, sizeof(read_buf)); + ASSERT_EQ(lseek(mem_fd_data2, 0, SEEK_SET), 0); + ASSERT_EQ(read(mem_fd_data2, read_buf, sizeof(read_buf)), strlen(data2)); + ASSERT_STREQ(read_buf, data2); + + ASSERT_EQ(lseek(mem_fd_empty1, 0, SEEK_SET), 0); + ASSERT_EQ(read(mem_fd_empty1, read_buf, sizeof(read_buf)), 0); + + ASSERT_EQ(lseek(mem_fd_empty2, 0, SEEK_SET), 0); + ASSERT_EQ(read(mem_fd_empty2, read_buf, sizeof(read_buf)), 0); + + ASSERT_EQ(close(mem_fd_data1), 0); + ASSERT_EQ(close(mem_fd_empty1), 0); + ASSERT_EQ(close(mem_fd_data2), 0); + ASSERT_EQ(close(mem_fd_empty2), 0); + ASSERT_EQ(close(session_fd1), 0); + ASSERT_EQ(close(session_fd2), 0); +} + +/* + * Test Case: Preserve Unsupported File Descriptor + * + * Verifies that attempting to preserve a file descriptor that does not have + * a registered Live Update handler fails gracefully. + * Uses /dev/null as a representative of a file type (character device) + * that is not supported by the orchestrator. + */ +TEST_F(liveupdate_device, preserve_unsupported_fd) +{ + int session_fd, unsupported_fd; + int ret; + + self->fd1 = open(LIVEUPDATE_DEV, O_RDWR); + if (self->fd1 < 0 && errno == ENOENT) + SKIP(return, "%s does not exist", LIVEUPDATE_DEV); + ASSERT_GE(self->fd1, 0); + + session_fd = create_session(self->fd1, "unsupported-fd-test"); + ASSERT_GE(session_fd, 0); + + unsupported_fd = open("/dev/null", O_RDWR); + ASSERT_GE(unsupported_fd, 0); + + ret = preserve_fd(session_fd, unsupported_fd, 0xDEAD); + EXPECT_EQ(ret, -ENOENT); + + ASSERT_EQ(close(unsupported_fd), 0); + ASSERT_EQ(close(session_fd), 0); +} + +TEST_HARNESS_MAIN diff --git a/tools/testing/selftests/liveupdate/luo_kexec_simple.c b/tools/testing/selftests/liveupdate/luo_kexec_simple.c new file mode 100644 index 000000000000..d7ac1f3dc4cb --- /dev/null +++ b/tools/testing/selftests/liveupdate/luo_kexec_simple.c @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + * + * A simple selftest to validate the end-to-end lifecycle of a LUO session + * across a single kexec reboot. + */ + +#include "luo_test_utils.h" + +#define TEST_SESSION_NAME "test-session" +#define TEST_MEMFD_TOKEN 0x1A +#define TEST_MEMFD_DATA "hello kexec world" + +/* Constants for the state-tracking mechanism, specific to this test file. */ +#define STATE_SESSION_NAME "kexec_simple_state" +#define STATE_MEMFD_TOKEN 999 + +/* Stage 1: Executed before the kexec reboot. */ +static void run_stage_1(int luo_fd) +{ + int session_fd; + + ksft_print_msg("[STAGE 1] Starting pre-kexec setup...\n"); + + ksft_print_msg("[STAGE 1] Creating state file for next stage (2)...\n"); + create_state_file(luo_fd, STATE_SESSION_NAME, STATE_MEMFD_TOKEN, 2); + + ksft_print_msg("[STAGE 1] Creating session '%s' and preserving memfd...\n", + TEST_SESSION_NAME); + session_fd = luo_create_session(luo_fd, TEST_SESSION_NAME); + if (session_fd < 0) + fail_exit("luo_create_session for '%s'", TEST_SESSION_NAME); + + if (create_and_preserve_memfd(session_fd, TEST_MEMFD_TOKEN, + TEST_MEMFD_DATA) < 0) { + fail_exit("create_and_preserve_memfd for token %#x", + TEST_MEMFD_TOKEN); + } + + close(luo_fd); + daemonize_and_wait(); +} + +/* Stage 2: Executed after the kexec reboot. */ +static void run_stage_2(int luo_fd, int state_session_fd) +{ + int session_fd, mfd, stage; + + ksft_print_msg("[STAGE 2] Starting post-kexec verification...\n"); + + restore_and_read_stage(state_session_fd, STATE_MEMFD_TOKEN, &stage); + if (stage != 2) + fail_exit("Expected stage 2, but state file contains %d", stage); + + ksft_print_msg("[STAGE 2] Retrieving session '%s'...\n", TEST_SESSION_NAME); + session_fd = luo_retrieve_session(luo_fd, TEST_SESSION_NAME); + if (session_fd < 0) + fail_exit("luo_retrieve_session for '%s'", TEST_SESSION_NAME); + + ksft_print_msg("[STAGE 2] Restoring and verifying memfd (token %#x)...\n", + TEST_MEMFD_TOKEN); + mfd = restore_and_verify_memfd(session_fd, TEST_MEMFD_TOKEN, + TEST_MEMFD_DATA); + if (mfd < 0) + fail_exit("restore_and_verify_memfd for token %#x", TEST_MEMFD_TOKEN); + close(mfd); + + ksft_print_msg("[STAGE 2] Test data verified successfully.\n"); + ksft_print_msg("[STAGE 2] Finalizing test session...\n"); + if (luo_session_finish(session_fd) < 0) + fail_exit("luo_session_finish for test session"); + close(session_fd); + + ksft_print_msg("[STAGE 2] Finalizing state session...\n"); + if (luo_session_finish(state_session_fd) < 0) + fail_exit("luo_session_finish for state session"); + close(state_session_fd); + + ksft_print_msg("\n--- SIMPLE KEXEC TEST PASSED ---\n"); +} + +int main(int argc, char *argv[]) +{ + return luo_test(argc, argv, STATE_SESSION_NAME, + run_stage_1, run_stage_2); +} diff --git a/tools/testing/selftests/liveupdate/luo_multi_session.c b/tools/testing/selftests/liveupdate/luo_multi_session.c new file mode 100644 index 000000000000..0ee2d795beef --- /dev/null +++ b/tools/testing/selftests/liveupdate/luo_multi_session.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + * + * A selftest to validate the end-to-end lifecycle of multiple LUO sessions + * across a kexec reboot, including empty sessions and sessions with multiple + * files. + */ + +#include "luo_test_utils.h" + +#define SESSION_EMPTY_1 "multi-test-empty-1" +#define SESSION_EMPTY_2 "multi-test-empty-2" +#define SESSION_FILES_1 "multi-test-files-1" +#define SESSION_FILES_2 "multi-test-files-2" + +#define MFD1_TOKEN 0x1001 +#define MFD2_TOKEN 0x2002 +#define MFD3_TOKEN 0x3003 + +#define MFD1_DATA "Data for session files 1" +#define MFD2_DATA "First file for session files 2" +#define MFD3_DATA "Second file for session files 2" + +#define STATE_SESSION_NAME "kexec_multi_state" +#define STATE_MEMFD_TOKEN 998 + +/* Stage 1: Executed before the kexec reboot. */ +static void run_stage_1(int luo_fd) +{ + int s_empty1_fd, s_empty2_fd, s_files1_fd, s_files2_fd; + + ksft_print_msg("[STAGE 1] Starting pre-kexec setup for multi-session test...\n"); + + ksft_print_msg("[STAGE 1] Creating state file for next stage (2)...\n"); + create_state_file(luo_fd, STATE_SESSION_NAME, STATE_MEMFD_TOKEN, 2); + + ksft_print_msg("[STAGE 1] Creating empty sessions '%s' and '%s'...\n", + SESSION_EMPTY_1, SESSION_EMPTY_2); + s_empty1_fd = luo_create_session(luo_fd, SESSION_EMPTY_1); + if (s_empty1_fd < 0) + fail_exit("luo_create_session for '%s'", SESSION_EMPTY_1); + + s_empty2_fd = luo_create_session(luo_fd, SESSION_EMPTY_2); + if (s_empty2_fd < 0) + fail_exit("luo_create_session for '%s'", SESSION_EMPTY_2); + + ksft_print_msg("[STAGE 1] Creating session '%s' with one memfd...\n", + SESSION_FILES_1); + + s_files1_fd = luo_create_session(luo_fd, SESSION_FILES_1); + if (s_files1_fd < 0) + fail_exit("luo_create_session for '%s'", SESSION_FILES_1); + if (create_and_preserve_memfd(s_files1_fd, MFD1_TOKEN, MFD1_DATA) < 0) { + fail_exit("create_and_preserve_memfd for token %#x", + MFD1_TOKEN); + } + + ksft_print_msg("[STAGE 1] Creating session '%s' with two memfds...\n", + SESSION_FILES_2); + + s_files2_fd = luo_create_session(luo_fd, SESSION_FILES_2); + if (s_files2_fd < 0) + fail_exit("luo_create_session for '%s'", SESSION_FILES_2); + if (create_and_preserve_memfd(s_files2_fd, MFD2_TOKEN, MFD2_DATA) < 0) { + fail_exit("create_and_preserve_memfd for token %#x", + MFD2_TOKEN); + } + if (create_and_preserve_memfd(s_files2_fd, MFD3_TOKEN, MFD3_DATA) < 0) { + fail_exit("create_and_preserve_memfd for token %#x", + MFD3_TOKEN); + } + + close(luo_fd); + daemonize_and_wait(); +} + +/* Stage 2: Executed after the kexec reboot. */ +static void run_stage_2(int luo_fd, int state_session_fd) +{ + int s_empty1_fd, s_empty2_fd, s_files1_fd, s_files2_fd; + int mfd1, mfd2, mfd3, stage; + + ksft_print_msg("[STAGE 2] Starting post-kexec verification...\n"); + + restore_and_read_stage(state_session_fd, STATE_MEMFD_TOKEN, &stage); + if (stage != 2) { + fail_exit("Expected stage 2, but state file contains %d", + stage); + } + + ksft_print_msg("[STAGE 2] Retrieving all sessions...\n"); + s_empty1_fd = luo_retrieve_session(luo_fd, SESSION_EMPTY_1); + if (s_empty1_fd < 0) + fail_exit("luo_retrieve_session for '%s'", SESSION_EMPTY_1); + + s_empty2_fd = luo_retrieve_session(luo_fd, SESSION_EMPTY_2); + if (s_empty2_fd < 0) + fail_exit("luo_retrieve_session for '%s'", SESSION_EMPTY_2); + + s_files1_fd = luo_retrieve_session(luo_fd, SESSION_FILES_1); + if (s_files1_fd < 0) + fail_exit("luo_retrieve_session for '%s'", SESSION_FILES_1); + + s_files2_fd = luo_retrieve_session(luo_fd, SESSION_FILES_2); + if (s_files2_fd < 0) + fail_exit("luo_retrieve_session for '%s'", SESSION_FILES_2); + + ksft_print_msg("[STAGE 2] Verifying contents of session '%s'...\n", + SESSION_FILES_1); + mfd1 = restore_and_verify_memfd(s_files1_fd, MFD1_TOKEN, MFD1_DATA); + if (mfd1 < 0) + fail_exit("restore_and_verify_memfd for token %#x", MFD1_TOKEN); + close(mfd1); + + ksft_print_msg("[STAGE 2] Verifying contents of session '%s'...\n", + SESSION_FILES_2); + + mfd2 = restore_and_verify_memfd(s_files2_fd, MFD2_TOKEN, MFD2_DATA); + if (mfd2 < 0) + fail_exit("restore_and_verify_memfd for token %#x", MFD2_TOKEN); + close(mfd2); + + mfd3 = restore_and_verify_memfd(s_files2_fd, MFD3_TOKEN, MFD3_DATA); + if (mfd3 < 0) + fail_exit("restore_and_verify_memfd for token %#x", MFD3_TOKEN); + close(mfd3); + + ksft_print_msg("[STAGE 2] Test data verified successfully.\n"); + + ksft_print_msg("[STAGE 2] Finalizing all test sessions...\n"); + if (luo_session_finish(s_empty1_fd) < 0) + fail_exit("luo_session_finish for '%s'", SESSION_EMPTY_1); + close(s_empty1_fd); + + if (luo_session_finish(s_empty2_fd) < 0) + fail_exit("luo_session_finish for '%s'", SESSION_EMPTY_2); + close(s_empty2_fd); + + if (luo_session_finish(s_files1_fd) < 0) + fail_exit("luo_session_finish for '%s'", SESSION_FILES_1); + close(s_files1_fd); + + if (luo_session_finish(s_files2_fd) < 0) + fail_exit("luo_session_finish for '%s'", SESSION_FILES_2); + close(s_files2_fd); + + ksft_print_msg("[STAGE 2] Finalizing state session...\n"); + if (luo_session_finish(state_session_fd) < 0) + fail_exit("luo_session_finish for state session"); + close(state_session_fd); + + ksft_print_msg("\n--- MULTI-SESSION KEXEC TEST PASSED ---\n"); +} + +int main(int argc, char *argv[]) +{ + return luo_test(argc, argv, STATE_SESSION_NAME, + run_stage_1, run_stage_2); +} diff --git a/tools/testing/selftests/liveupdate/luo_test_utils.c b/tools/testing/selftests/liveupdate/luo_test_utils.c new file mode 100644 index 000000000000..3c8721c505df --- /dev/null +++ b/tools/testing/selftests/liveupdate/luo_test_utils.c @@ -0,0 +1,266 @@ +// SPDX-License-Identifier: GPL-2.0-only + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + */ + +#define _GNU_SOURCE + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <getopt.h> +#include <fcntl.h> +#include <unistd.h> +#include <sys/ioctl.h> +#include <sys/syscall.h> +#include <sys/mman.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <errno.h> +#include <stdarg.h> + +#include "luo_test_utils.h" + +int luo_open_device(void) +{ + return open(LUO_DEVICE, O_RDWR); +} + +int luo_create_session(int luo_fd, const char *name) +{ + struct liveupdate_ioctl_create_session arg = { .size = sizeof(arg) }; + + snprintf((char *)arg.name, LIVEUPDATE_SESSION_NAME_LENGTH, "%.*s", + LIVEUPDATE_SESSION_NAME_LENGTH - 1, name); + + if (ioctl(luo_fd, LIVEUPDATE_IOCTL_CREATE_SESSION, &arg) < 0) + return -errno; + + return arg.fd; +} + +int luo_retrieve_session(int luo_fd, const char *name) +{ + struct liveupdate_ioctl_retrieve_session arg = { .size = sizeof(arg) }; + + snprintf((char *)arg.name, LIVEUPDATE_SESSION_NAME_LENGTH, "%.*s", + LIVEUPDATE_SESSION_NAME_LENGTH - 1, name); + + if (ioctl(luo_fd, LIVEUPDATE_IOCTL_RETRIEVE_SESSION, &arg) < 0) + return -errno; + + return arg.fd; +} + +int create_and_preserve_memfd(int session_fd, int token, const char *data) +{ + struct liveupdate_session_preserve_fd arg = { .size = sizeof(arg) }; + long page_size = sysconf(_SC_PAGE_SIZE); + void *map = MAP_FAILED; + int mfd = -1, ret = -1; + + mfd = memfd_create("test_mfd", 0); + if (mfd < 0) + return -errno; + + if (ftruncate(mfd, page_size) != 0) + goto out; + + map = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, mfd, 0); + if (map == MAP_FAILED) + goto out; + + snprintf(map, page_size, "%s", data); + munmap(map, page_size); + + arg.fd = mfd; + arg.token = token; + if (ioctl(session_fd, LIVEUPDATE_SESSION_PRESERVE_FD, &arg) < 0) + goto out; + + ret = 0; +out: + if (ret != 0 && errno != 0) + ret = -errno; + if (mfd >= 0) + close(mfd); + return ret; +} + +int restore_and_verify_memfd(int session_fd, int token, + const char *expected_data) +{ + struct liveupdate_session_retrieve_fd arg = { .size = sizeof(arg) }; + long page_size = sysconf(_SC_PAGE_SIZE); + void *map = MAP_FAILED; + int mfd = -1, ret = -1; + + arg.token = token; + if (ioctl(session_fd, LIVEUPDATE_SESSION_RETRIEVE_FD, &arg) < 0) + return -errno; + mfd = arg.fd; + + map = mmap(NULL, page_size, PROT_READ, MAP_SHARED, mfd, 0); + if (map == MAP_FAILED) + goto out; + + if (expected_data && strcmp(expected_data, map) != 0) { + ksft_print_msg("Data mismatch! Expected '%s', Got '%s'\n", + expected_data, (char *)map); + ret = -EINVAL; + goto out_munmap; + } + + ret = mfd; +out_munmap: + munmap(map, page_size); +out: + if (ret < 0 && errno != 0) + ret = -errno; + if (ret < 0 && mfd >= 0) + close(mfd); + return ret; +} + +int luo_session_finish(int session_fd) +{ + struct liveupdate_session_finish arg = { .size = sizeof(arg) }; + + if (ioctl(session_fd, LIVEUPDATE_SESSION_FINISH, &arg) < 0) + return -errno; + + return 0; +} + +void create_state_file(int luo_fd, const char *session_name, int token, + int next_stage) +{ + char buf[32]; + int state_session_fd; + + state_session_fd = luo_create_session(luo_fd, session_name); + if (state_session_fd < 0) + fail_exit("luo_create_session for state tracking"); + + snprintf(buf, sizeof(buf), "%d", next_stage); + if (create_and_preserve_memfd(state_session_fd, token, buf) < 0) + fail_exit("create_and_preserve_memfd for state tracking"); + + /* + * DO NOT close session FD, otherwise it is going to be unpreserved + */ +} + +void restore_and_read_stage(int state_session_fd, int token, int *stage) +{ + char buf[32] = {0}; + int mfd; + + mfd = restore_and_verify_memfd(state_session_fd, token, NULL); + if (mfd < 0) + fail_exit("failed to restore state memfd"); + + if (read(mfd, buf, sizeof(buf) - 1) < 0) + fail_exit("failed to read state mfd"); + + *stage = atoi(buf); + + close(mfd); +} + +void daemonize_and_wait(void) +{ + pid_t pid; + + ksft_print_msg("[STAGE 1] Forking persistent child to hold sessions...\n"); + + pid = fork(); + if (pid < 0) + fail_exit("fork failed"); + + if (pid > 0) { + ksft_print_msg("[STAGE 1] Child PID: %d. Resources are pinned.\n", pid); + ksft_print_msg("[STAGE 1] You may now perform kexec reboot.\n"); + exit(EXIT_SUCCESS); + } + + /* Detach from terminal so closing the window doesn't kill us */ + if (setsid() < 0) + fail_exit("setsid failed"); + + close(STDIN_FILENO); + close(STDOUT_FILENO); + close(STDERR_FILENO); + + /* Change dir to root to avoid locking filesystems */ + if (chdir("/") < 0) + exit(EXIT_FAILURE); + + while (1) + sleep(60); +} + +static int parse_stage_args(int argc, char *argv[]) +{ + static struct option long_options[] = { + {"stage", required_argument, 0, 's'}, + {0, 0, 0, 0} + }; + int option_index = 0; + int stage = 1; + int opt; + + optind = 1; + while ((opt = getopt_long(argc, argv, "s:", long_options, &option_index)) != -1) { + switch (opt) { + case 's': + stage = atoi(optarg); + if (stage != 1 && stage != 2) + fail_exit("Invalid stage argument"); + break; + default: + fail_exit("Unknown argument"); + } + } + return stage; +} + +int luo_test(int argc, char *argv[], + const char *state_session_name, + luo_test_stage1_fn stage1, + luo_test_stage2_fn stage2) +{ + int target_stage = parse_stage_args(argc, argv); + int luo_fd = luo_open_device(); + int state_session_fd; + int detected_stage; + + if (luo_fd < 0) { + ksft_exit_skip("Failed to open %s. Is the luo module loaded?\n", + LUO_DEVICE); + } + + state_session_fd = luo_retrieve_session(luo_fd, state_session_name); + if (state_session_fd == -ENOENT) + detected_stage = 1; + else if (state_session_fd >= 0) + detected_stage = 2; + else + fail_exit("Failed to check for state session"); + + if (target_stage != detected_stage) { + ksft_exit_fail_msg("Stage mismatch Requested --stage %d, but system is in stage %d.\n" + "(State session %s: %s)\n", + target_stage, detected_stage, state_session_name, + (detected_stage == 2) ? "EXISTS" : "MISSING"); + } + + if (target_stage == 1) + stage1(luo_fd); + else + stage2(luo_fd, state_session_fd); + + return 0; +} diff --git a/tools/testing/selftests/liveupdate/luo_test_utils.h b/tools/testing/selftests/liveupdate/luo_test_utils.h new file mode 100644 index 000000000000..90099bf49577 --- /dev/null +++ b/tools/testing/selftests/liveupdate/luo_test_utils.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +/* + * Copyright (c) 2025, Google LLC. + * Pasha Tatashin <pasha.tatashin@soleen.com> + * + * Utility functions for LUO kselftests. + */ + +#ifndef LUO_TEST_UTILS_H +#define LUO_TEST_UTILS_H + +#include <errno.h> +#include <string.h> +#include <linux/liveupdate.h> +#include "../kselftest.h" + +#define LUO_DEVICE "/dev/liveupdate" + +#define fail_exit(fmt, ...) \ + ksft_exit_fail_msg("[%s:%d] " fmt " (errno: %s)\n", \ + __func__, __LINE__, ##__VA_ARGS__, strerror(errno)) + +int luo_open_device(void); +int luo_create_session(int luo_fd, const char *name); +int luo_retrieve_session(int luo_fd, const char *name); +int luo_session_finish(int session_fd); + +int create_and_preserve_memfd(int session_fd, int token, const char *data); +int restore_and_verify_memfd(int session_fd, int token, const char *expected_data); + +void create_state_file(int luo_fd, const char *session_name, int token, + int next_stage); +void restore_and_read_stage(int state_session_fd, int token, int *stage); + +void daemonize_and_wait(void); + +typedef void (*luo_test_stage1_fn)(int luo_fd); +typedef void (*luo_test_stage2_fn)(int luo_fd, int state_session_fd); + +int luo_test(int argc, char *argv[], const char *state_session_name, + luo_test_stage1_fn stage1, luo_test_stage2_fn stage2); + +#endif /* LUO_TEST_UTILS_H */ diff --git a/tools/testing/selftests/lsm/lsm_get_self_attr_test.c b/tools/testing/selftests/lsm/lsm_get_self_attr_test.c index df215e4aa63f..60caf8528f81 100644 --- a/tools/testing/selftests/lsm/lsm_get_self_attr_test.c +++ b/tools/testing/selftests/lsm/lsm_get_self_attr_test.c @@ -13,7 +13,7 @@ #include <stdio.h> #include <unistd.h> #include <sys/types.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "common.h" static struct lsm_ctx *next_ctx(struct lsm_ctx *ctxp) diff --git a/tools/testing/selftests/lsm/lsm_list_modules_test.c b/tools/testing/selftests/lsm/lsm_list_modules_test.c index 1cc8a977c711..54d59044ace1 100644 --- a/tools/testing/selftests/lsm/lsm_list_modules_test.c +++ b/tools/testing/selftests/lsm/lsm_list_modules_test.c @@ -12,7 +12,7 @@ #include <stdio.h> #include <unistd.h> #include <sys/types.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "common.h" TEST(size_null_lsm_list_modules) diff --git a/tools/testing/selftests/lsm/lsm_set_self_attr_test.c b/tools/testing/selftests/lsm/lsm_set_self_attr_test.c index 732e89fe99c0..dcb6f8aa772e 100644 --- a/tools/testing/selftests/lsm/lsm_set_self_attr_test.c +++ b/tools/testing/selftests/lsm/lsm_set_self_attr_test.c @@ -12,7 +12,7 @@ #include <stdio.h> #include <unistd.h> #include <sys/types.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "common.h" TEST(ctx_null_lsm_set_self_attr) diff --git a/tools/testing/selftests/media_tests/media_device_open.c b/tools/testing/selftests/media_tests/media_device_open.c index 93183a37b133..4396bf2273a4 100644 --- a/tools/testing/selftests/media_tests/media_device_open.c +++ b/tools/testing/selftests/media_tests/media_device_open.c @@ -34,7 +34,7 @@ #include <sys/stat.h> #include <linux/media.h> -#include "../kselftest.h" +#include "kselftest.h" int main(int argc, char **argv) { diff --git a/tools/testing/selftests/media_tests/media_device_test.c b/tools/testing/selftests/media_tests/media_device_test.c index 4b9953359e40..6e4a8090a0eb 100644 --- a/tools/testing/selftests/media_tests/media_device_test.c +++ b/tools/testing/selftests/media_tests/media_device_test.c @@ -39,7 +39,7 @@ #include <time.h> #include <linux/media.h> -#include "../kselftest.h" +#include "kselftest.h" int main(int argc, char **argv) { diff --git a/tools/testing/selftests/membarrier/membarrier_test_impl.h b/tools/testing/selftests/membarrier/membarrier_test_impl.h index af89855adb7b..f6d7c44b2288 100644 --- a/tools/testing/selftests/membarrier/membarrier_test_impl.h +++ b/tools/testing/selftests/membarrier/membarrier_test_impl.h @@ -7,7 +7,7 @@ #include <string.h> #include <pthread.h> -#include "../kselftest.h" +#include "kselftest.h" static int registrations; diff --git a/tools/testing/selftests/mincore/mincore_selftest.c b/tools/testing/selftests/mincore/mincore_selftest.c index 17ed3e9917ca..cdd022c1c497 100644 --- a/tools/testing/selftests/mincore/mincore_selftest.c +++ b/tools/testing/selftests/mincore/mincore_selftest.c @@ -15,8 +15,8 @@ #include <string.h> #include <fcntl.h> -#include "../kselftest.h" -#include "../kselftest_harness.h" +#include "kselftest.h" +#include "kselftest_harness.h" /* Default test file size: 4MB */ #define MB (1UL << 20) diff --git a/tools/testing/selftests/mm/compaction_test.c b/tools/testing/selftests/mm/compaction_test.c index 9bc4591c7b16..30209c40b697 100644 --- a/tools/testing/selftests/mm/compaction_test.c +++ b/tools/testing/selftests/mm/compaction_test.c @@ -16,7 +16,7 @@ #include <unistd.h> #include <string.h> -#include "../kselftest.h" +#include "kselftest.h" #define MAP_SIZE_MB 100 #define MAP_SIZE (MAP_SIZE_MB * 1024 * 1024) diff --git a/tools/testing/selftests/mm/cow.c b/tools/testing/selftests/mm/cow.c index 6560c26f47d1..accfd198dbda 100644 --- a/tools/testing/selftests/mm/cow.c +++ b/tools/testing/selftests/mm/cow.c @@ -27,7 +27,7 @@ #endif /* LOCAL_CONFIG_HAVE_LIBURING */ #include "../../../../mm/gup_test.h" -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" #include "thp_settings.h" diff --git a/tools/testing/selftests/mm/droppable.c b/tools/testing/selftests/mm/droppable.c index f3d9ecf96890..44940f75c461 100644 --- a/tools/testing/selftests/mm/droppable.c +++ b/tools/testing/selftests/mm/droppable.c @@ -13,7 +13,7 @@ #include <sys/mman.h> #include <linux/mman.h> -#include "../kselftest.h" +#include "kselftest.h" int main(int argc, char *argv[]) { diff --git a/tools/testing/selftests/mm/guard-regions.c b/tools/testing/selftests/mm/guard-regions.c index 795bf3f39f44..dbd21d66d383 100644 --- a/tools/testing/selftests/mm/guard-regions.c +++ b/tools/testing/selftests/mm/guard-regions.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #define _GNU_SOURCE -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <asm-generic/mman.h> /* Force the import of the tools version. */ #include <assert.h> #include <errno.h> diff --git a/tools/testing/selftests/mm/gup_longterm.c b/tools/testing/selftests/mm/gup_longterm.c index 268dadb8ce43..6279893a0adc 100644 --- a/tools/testing/selftests/mm/gup_longterm.c +++ b/tools/testing/selftests/mm/gup_longterm.c @@ -27,7 +27,7 @@ #endif /* LOCAL_CONFIG_HAVE_LIBURING */ #include "../../../../mm/gup_test.h" -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" static size_t pagesize; diff --git a/tools/testing/selftests/mm/gup_test.c b/tools/testing/selftests/mm/gup_test.c index 40c1538a17b4..fb8f9ae49efa 100644 --- a/tools/testing/selftests/mm/gup_test.c +++ b/tools/testing/selftests/mm/gup_test.c @@ -12,7 +12,7 @@ #include <pthread.h> #include <assert.h> #include <mm/gup_test.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" #define MB (1UL << 20) diff --git a/tools/testing/selftests/mm/hmm-tests.c b/tools/testing/selftests/mm/hmm-tests.c index 5a1525f72daa..e8328c89d855 100644 --- a/tools/testing/selftests/mm/hmm-tests.c +++ b/tools/testing/selftests/mm/hmm-tests.c @@ -10,7 +10,7 @@ * bugs. */ -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <errno.h> #include <fcntl.h> diff --git a/tools/testing/selftests/mm/hugepage-mmap.c b/tools/testing/selftests/mm/hugepage-mmap.c index 3b1b532f1cbb..d543419de040 100644 --- a/tools/testing/selftests/mm/hugepage-mmap.c +++ b/tools/testing/selftests/mm/hugepage-mmap.c @@ -15,7 +15,7 @@ #include <unistd.h> #include <sys/mman.h> #include <fcntl.h> -#include "../kselftest.h" +#include "kselftest.h" #define LENGTH (256UL*1024*1024) #define PROTECTION (PROT_READ | PROT_WRITE) diff --git a/tools/testing/selftests/mm/hugepage-mremap.c b/tools/testing/selftests/mm/hugepage-mremap.c index 2bd1dac75c3f..b8f7d92e5a35 100644 --- a/tools/testing/selftests/mm/hugepage-mremap.c +++ b/tools/testing/selftests/mm/hugepage-mremap.c @@ -24,7 +24,7 @@ #include <sys/ioctl.h> #include <string.h> #include <stdbool.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" #define DEFAULT_LENGTH_MB 10UL diff --git a/tools/testing/selftests/mm/hugetlb-madvise.c b/tools/testing/selftests/mm/hugetlb-madvise.c index c5940c0595be..05d9d2805ae4 100644 --- a/tools/testing/selftests/mm/hugetlb-madvise.c +++ b/tools/testing/selftests/mm/hugetlb-madvise.c @@ -19,7 +19,7 @@ #include <sys/mman.h> #include <fcntl.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" #define MIN_FREE_PAGES 20 #define NR_HUGE_PAGES 10 /* common number of pages to map/allocate */ diff --git a/tools/testing/selftests/mm/hugetlb-read-hwpoison.c b/tools/testing/selftests/mm/hugetlb-read-hwpoison.c index ba6cc6f9cabc..46230462ad48 100644 --- a/tools/testing/selftests/mm/hugetlb-read-hwpoison.c +++ b/tools/testing/selftests/mm/hugetlb-read-hwpoison.c @@ -11,7 +11,7 @@ #include <errno.h> #include <stdbool.h> -#include "../kselftest.h" +#include "kselftest.h" #define PREFIX " ... " #define ERROR_PREFIX " !!! " diff --git a/tools/testing/selftests/mm/hugetlb-soft-offline.c b/tools/testing/selftests/mm/hugetlb-soft-offline.c index f086f0e04756..a8bc02688085 100644 --- a/tools/testing/selftests/mm/hugetlb-soft-offline.c +++ b/tools/testing/selftests/mm/hugetlb-soft-offline.c @@ -24,7 +24,7 @@ #include <sys/statfs.h> #include <sys/types.h> -#include "../kselftest.h" +#include "kselftest.h" #ifndef MADV_SOFT_OFFLINE #define MADV_SOFT_OFFLINE 101 diff --git a/tools/testing/selftests/mm/hugetlb_dio.c b/tools/testing/selftests/mm/hugetlb_dio.c index db63abe5ee5e..9ac62eb4c97d 100644 --- a/tools/testing/selftests/mm/hugetlb_dio.c +++ b/tools/testing/selftests/mm/hugetlb_dio.c @@ -18,7 +18,7 @@ #include <string.h> #include <sys/mman.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" void run_dio_using_hugetlb(unsigned int start_off, unsigned int end_off) { diff --git a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c index e2640529dbb2..b4b257775b74 100644 --- a/tools/testing/selftests/mm/hugetlb_fault_after_madv.c +++ b/tools/testing/selftests/mm/hugetlb_fault_after_madv.c @@ -9,7 +9,7 @@ #include <signal.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" #define INLOOP_ITER 100 diff --git a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c index 8f122a0f0828..efd774b41389 100644 --- a/tools/testing/selftests/mm/hugetlb_madv_vs_map.c +++ b/tools/testing/selftests/mm/hugetlb_madv_vs_map.c @@ -25,7 +25,7 @@ #include <unistd.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" #define INLOOP_ITER 100 diff --git a/tools/testing/selftests/mm/ksm_functional_tests.c b/tools/testing/selftests/mm/ksm_functional_tests.c index 95afa5cfc062..8d874c4754f3 100644 --- a/tools/testing/selftests/mm/ksm_functional_tests.c +++ b/tools/testing/selftests/mm/ksm_functional_tests.c @@ -21,7 +21,7 @@ #include <sys/wait.h> #include <linux/userfaultfd.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" #define KiB 1024u diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c index b77462b5c240..a0b48b839d54 100644 --- a/tools/testing/selftests/mm/ksm_tests.c +++ b/tools/testing/selftests/mm/ksm_tests.c @@ -12,7 +12,7 @@ #include <stdint.h> #include <err.h> -#include "../kselftest.h" +#include "kselftest.h" #include <include/vdso/time64.h> #include "vm_util.h" #include "thp_settings.h" diff --git a/tools/testing/selftests/mm/madv_populate.c b/tools/testing/selftests/mm/madv_populate.c index d8d11bc67ddc..88050e0f829a 100644 --- a/tools/testing/selftests/mm/madv_populate.c +++ b/tools/testing/selftests/mm/madv_populate.c @@ -17,7 +17,7 @@ #include <linux/mman.h> #include <sys/mman.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" /* diff --git a/tools/testing/selftests/mm/map_fixed_noreplace.c b/tools/testing/selftests/mm/map_fixed_noreplace.c index 1e9980b8993c..11241edde7fe 100644 --- a/tools/testing/selftests/mm/map_fixed_noreplace.c +++ b/tools/testing/selftests/mm/map_fixed_noreplace.c @@ -12,7 +12,7 @@ #include <stdio.h> #include <stdlib.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" static void dump_maps(void) { diff --git a/tools/testing/selftests/mm/map_hugetlb.c b/tools/testing/selftests/mm/map_hugetlb.c index b47399feab53..aa409107611b 100644 --- a/tools/testing/selftests/mm/map_hugetlb.c +++ b/tools/testing/selftests/mm/map_hugetlb.c @@ -11,7 +11,7 @@ #include <sys/mman.h> #include <fcntl.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" #define LENGTH (256UL*1024*1024) #define PROTECTION (PROT_READ | PROT_WRITE) diff --git a/tools/testing/selftests/mm/map_populate.c b/tools/testing/selftests/mm/map_populate.c index 9df2636c829b..712327f4e932 100644 --- a/tools/testing/selftests/mm/map_populate.c +++ b/tools/testing/selftests/mm/map_populate.c @@ -16,7 +16,7 @@ #include <stdlib.h> #include <string.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" diff --git a/tools/testing/selftests/mm/mdwe_test.c b/tools/testing/selftests/mm/mdwe_test.c index 200bedcdc32e..647779653da0 100644 --- a/tools/testing/selftests/mm/mdwe_test.c +++ b/tools/testing/selftests/mm/mdwe_test.c @@ -14,7 +14,7 @@ #include <sys/wait.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #ifndef __aarch64__ # define PROT_BTI 0 diff --git a/tools/testing/selftests/mm/memfd_secret.c b/tools/testing/selftests/mm/memfd_secret.c index 9a0597310a76..aac4f795c327 100644 --- a/tools/testing/selftests/mm/memfd_secret.c +++ b/tools/testing/selftests/mm/memfd_secret.c @@ -22,7 +22,7 @@ #include <stdio.h> #include <fcntl.h> -#include "../kselftest.h" +#include "kselftest.h" #define fail(fmt, ...) ksft_test_result_fail(fmt, ##__VA_ARGS__) #define pass(fmt, ...) ksft_test_result_pass(fmt, ##__VA_ARGS__) diff --git a/tools/testing/selftests/mm/merge.c b/tools/testing/selftests/mm/merge.c index cc4253f47f10..363c1033cc7d 100644 --- a/tools/testing/selftests/mm/merge.c +++ b/tools/testing/selftests/mm/merge.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #define _GNU_SOURCE -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <linux/prctl.h> #include <fcntl.h> #include <stdio.h> diff --git a/tools/testing/selftests/mm/migration.c b/tools/testing/selftests/mm/migration.c index ea945eebec2f..ee24b88c2b24 100644 --- a/tools/testing/selftests/mm/migration.c +++ b/tools/testing/selftests/mm/migration.c @@ -4,7 +4,7 @@ * paths in the kernel. */ -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "thp_settings.h" #include <strings.h> diff --git a/tools/testing/selftests/mm/mkdirty.c b/tools/testing/selftests/mm/mkdirty.c index 09feeb453646..68dd447a5454 100644 --- a/tools/testing/selftests/mm/mkdirty.c +++ b/tools/testing/selftests/mm/mkdirty.c @@ -22,7 +22,7 @@ #include <linux/userfaultfd.h> #include <linux/mempolicy.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" static size_t pagesize; diff --git a/tools/testing/selftests/mm/mlock-random-test.c b/tools/testing/selftests/mm/mlock-random-test.c index b8d7e966f44c..9d349c151360 100644 --- a/tools/testing/selftests/mm/mlock-random-test.c +++ b/tools/testing/selftests/mm/mlock-random-test.c @@ -13,7 +13,7 @@ #include <sys/ipc.h> #include <sys/shm.h> #include <time.h> -#include "../kselftest.h" +#include "kselftest.h" #include "mlock2.h" #define CHUNK_UNIT (128 * 1024) diff --git a/tools/testing/selftests/mm/mlock2-tests.c b/tools/testing/selftests/mm/mlock2-tests.c index 3e90ff37e336..b474f2b20def 100644 --- a/tools/testing/selftests/mm/mlock2-tests.c +++ b/tools/testing/selftests/mm/mlock2-tests.c @@ -7,7 +7,7 @@ #include <sys/time.h> #include <sys/resource.h> #include <stdbool.h> -#include "../kselftest.h" +#include "kselftest.h" #include "mlock2.h" struct vm_boundaries { diff --git a/tools/testing/selftests/mm/mrelease_test.c b/tools/testing/selftests/mm/mrelease_test.c index 100370a7111d..64e8d00ae944 100644 --- a/tools/testing/selftests/mm/mrelease_test.c +++ b/tools/testing/selftests/mm/mrelease_test.c @@ -12,7 +12,7 @@ #include <unistd.h> #include <asm-generic/unistd.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" #define MB(x) (x << 20) #define MAX_SIZE_MB 1024 diff --git a/tools/testing/selftests/mm/mremap_dontunmap.c b/tools/testing/selftests/mm/mremap_dontunmap.c index 1d75084b9ca5..a4f75d836733 100644 --- a/tools/testing/selftests/mm/mremap_dontunmap.c +++ b/tools/testing/selftests/mm/mremap_dontunmap.c @@ -14,7 +14,7 @@ #include <string.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" unsigned long page_size; char *page_buffer; diff --git a/tools/testing/selftests/mm/mremap_test.c b/tools/testing/selftests/mm/mremap_test.c index 5f073504e0b1..308576437228 100644 --- a/tools/testing/selftests/mm/mremap_test.c +++ b/tools/testing/selftests/mm/mremap_test.c @@ -16,7 +16,7 @@ #include <time.h> #include <stdbool.h> -#include "../kselftest.h" +#include "kselftest.h" #define EXPECT_SUCCESS 0 #define EXPECT_FAILURE 1 diff --git a/tools/testing/selftests/mm/mseal_test.c b/tools/testing/selftests/mm/mseal_test.c index 005f29c86484..faad4833366a 100644 --- a/tools/testing/selftests/mm/mseal_test.c +++ b/tools/testing/selftests/mm/mseal_test.c @@ -8,7 +8,7 @@ #include <sys/time.h> #include <sys/resource.h> #include <stdbool.h> -#include "../kselftest.h" +#include "kselftest.h" #include <syscall.h> #include <errno.h> #include <stdio.h> diff --git a/tools/testing/selftests/mm/on-fault-limit.c b/tools/testing/selftests/mm/on-fault-limit.c index 431c1277d83a..fc4117453c84 100644 --- a/tools/testing/selftests/mm/on-fault-limit.c +++ b/tools/testing/selftests/mm/on-fault-limit.c @@ -5,7 +5,7 @@ #include <string.h> #include <sys/time.h> #include <sys/resource.h> -#include "../kselftest.h" +#include "kselftest.h" static void test_limit(void) { diff --git a/tools/testing/selftests/mm/pagemap_ioctl.c b/tools/testing/selftests/mm/pagemap_ioctl.c index 4fc8e578ec7c..2cb5441f29c7 100644 --- a/tools/testing/selftests/mm/pagemap_ioctl.c +++ b/tools/testing/selftests/mm/pagemap_ioctl.c @@ -8,7 +8,7 @@ #include <errno.h> #include <malloc.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" #include <linux/types.h> #include <linux/memfd.h> #include <linux/userfaultfd.h> diff --git a/tools/testing/selftests/mm/pfnmap.c b/tools/testing/selftests/mm/pfnmap.c index 88659f0a90ea..f546dfb10cae 100644 --- a/tools/testing/selftests/mm/pfnmap.c +++ b/tools/testing/selftests/mm/pfnmap.c @@ -22,7 +22,7 @@ #include <sys/mman.h> #include <sys/wait.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "vm_util.h" static sigjmp_buf sigjmp_buf_env; diff --git a/tools/testing/selftests/mm/pkey-helpers.h b/tools/testing/selftests/mm/pkey-helpers.h index fa15f006fa68..7c29f075e40b 100644 --- a/tools/testing/selftests/mm/pkey-helpers.h +++ b/tools/testing/selftests/mm/pkey-helpers.h @@ -16,7 +16,7 @@ #include <linux/mman.h> #include <linux/types.h> -#include "../kselftest.h" +#include "kselftest.h" /* Define some kernel-like types */ typedef __u8 u8; diff --git a/tools/testing/selftests/mm/prctl_thp_disable.c b/tools/testing/selftests/mm/prctl_thp_disable.c index 84b4a4b345af..ca27200596a4 100644 --- a/tools/testing/selftests/mm/prctl_thp_disable.c +++ b/tools/testing/selftests/mm/prctl_thp_disable.c @@ -13,7 +13,7 @@ #include <sys/prctl.h> #include <sys/wait.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "thp_settings.h" #include "vm_util.h" diff --git a/tools/testing/selftests/mm/process_madv.c b/tools/testing/selftests/mm/process_madv.c index 471cae8427f1..cd4610baf5d7 100644 --- a/tools/testing/selftests/mm/process_madv.c +++ b/tools/testing/selftests/mm/process_madv.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later #define _GNU_SOURCE -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <errno.h> #include <setjmp.h> #include <signal.h> diff --git a/tools/testing/selftests/mm/rmap.c b/tools/testing/selftests/mm/rmap.c index 13f7bccfd0a9..53f2058b0ef2 100644 --- a/tools/testing/selftests/mm/rmap.c +++ b/tools/testing/selftests/mm/rmap.c @@ -5,7 +5,7 @@ * Author(s): Wei Yang <richard.weiyang@gmail.com> */ -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <strings.h> #include <pthread.h> #include <numa.h> diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c index c3a9585de98c..59c0dbe99a9b 100644 --- a/tools/testing/selftests/mm/soft-dirty.c +++ b/tools/testing/selftests/mm/soft-dirty.c @@ -7,7 +7,7 @@ #include <malloc.h> #include <sys/mman.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" #include "thp_settings.h" diff --git a/tools/testing/selftests/mm/split_huge_page_test.c b/tools/testing/selftests/mm/split_huge_page_test.c index 743af3c05190..40799f3f0213 100644 --- a/tools/testing/selftests/mm/split_huge_page_test.c +++ b/tools/testing/selftests/mm/split_huge_page_test.c @@ -20,7 +20,7 @@ #include <stdbool.h> #include <time.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" uint64_t pagesize; unsigned int pageshift; diff --git a/tools/testing/selftests/mm/thuge-gen.c b/tools/testing/selftests/mm/thuge-gen.c index 4f5e290ff1a6..77813d34dcc2 100644 --- a/tools/testing/selftests/mm/thuge-gen.c +++ b/tools/testing/selftests/mm/thuge-gen.c @@ -27,7 +27,7 @@ #include <stdarg.h> #include <string.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" #if !defined(MAP_HUGETLB) #define MAP_HUGETLB 0x40000 diff --git a/tools/testing/selftests/mm/transhuge-stress.c b/tools/testing/selftests/mm/transhuge-stress.c index 68201192e37c..bcad47c09518 100644 --- a/tools/testing/selftests/mm/transhuge-stress.c +++ b/tools/testing/selftests/mm/transhuge-stress.c @@ -16,7 +16,7 @@ #include <string.h> #include <sys/mman.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" int backing_fd = -1; int mmap_flags = MAP_ANONYMOUS | MAP_NORESERVE | MAP_PRIVATE; diff --git a/tools/testing/selftests/mm/uffd-common.h b/tools/testing/selftests/mm/uffd-common.h index 37d3ca55905f..844a85ab31eb 100644 --- a/tools/testing/selftests/mm/uffd-common.h +++ b/tools/testing/selftests/mm/uffd-common.h @@ -35,7 +35,7 @@ #include <sys/random.h> #include <stdatomic.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" #define UFFD_FLAGS (O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY) diff --git a/tools/testing/selftests/mm/uffd-wp-mremap.c b/tools/testing/selftests/mm/uffd-wp-mremap.c index 4e4a591cf527..17186d4a4147 100644 --- a/tools/testing/selftests/mm/uffd-wp-mremap.c +++ b/tools/testing/selftests/mm/uffd-wp-mremap.c @@ -7,7 +7,7 @@ #include <assert.h> #include <linux/mman.h> #include <sys/mman.h> -#include "../kselftest.h" +#include "kselftest.h" #include "thp_settings.h" #include "uffd-common.h" diff --git a/tools/testing/selftests/mm/va_high_addr_switch.c b/tools/testing/selftests/mm/va_high_addr_switch.c index 306eba825107..02f290a69132 100644 --- a/tools/testing/selftests/mm/va_high_addr_switch.c +++ b/tools/testing/selftests/mm/va_high_addr_switch.c @@ -10,7 +10,7 @@ #include <string.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" /* * The hint addr value is used to allocate addresses diff --git a/tools/testing/selftests/mm/virtual_address_range.c b/tools/testing/selftests/mm/virtual_address_range.c index 81b33d8f78f4..4f0923825ed7 100644 --- a/tools/testing/selftests/mm/virtual_address_range.c +++ b/tools/testing/selftests/mm/virtual_address_range.c @@ -16,7 +16,7 @@ #include <fcntl.h> #include "vm_util.h" -#include "../kselftest.h" +#include "kselftest.h" /* * Maximum address range mapped with a single mmap() diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c index 605cb58ea5c3..d954bf91afd5 100644 --- a/tools/testing/selftests/mm/vm_util.c +++ b/tools/testing/selftests/mm/vm_util.c @@ -9,7 +9,7 @@ #include <linux/fs.h> #include <sys/syscall.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vm_util.h" #define PMD_SIZE_FILE_PATH "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size" diff --git a/tools/testing/selftests/mm/vm_util.h b/tools/testing/selftests/mm/vm_util.h index a8abdf414d46..6ad32b1830f1 100644 --- a/tools/testing/selftests/mm/vm_util.h +++ b/tools/testing/selftests/mm/vm_util.h @@ -6,7 +6,7 @@ #include <stdarg.h> #include <strings.h> /* ffsl() */ #include <unistd.h> /* _SC_PAGESIZE */ -#include "../kselftest.h" +#include "kselftest.h" #include <linux/fs.h> #define BIT_ULL(nr) (1ULL << (nr)) diff --git a/tools/testing/selftests/mount_setattr/mount_setattr_test.c b/tools/testing/selftests/mount_setattr/mount_setattr_test.c index a688871a98eb..7aec3ae82a44 100644 --- a/tools/testing/selftests/mount_setattr/mount_setattr_test.c +++ b/tools/testing/selftests/mount_setattr/mount_setattr_test.c @@ -21,7 +21,7 @@ #include <linux/mount.h> #include "../filesystems/wrappers.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #ifndef CLONE_NEWNS #define CLONE_NEWNS 0x00020000 diff --git a/tools/testing/selftests/move_mount_set_group/move_mount_set_group_test.c b/tools/testing/selftests/move_mount_set_group/move_mount_set_group_test.c index bcf51d785a37..12434415ec36 100644 --- a/tools/testing/selftests/move_mount_set_group/move_mount_set_group_test.c +++ b/tools/testing/selftests/move_mount_set_group/move_mount_set_group_test.c @@ -15,7 +15,7 @@ #include <stdarg.h> #include <sys/syscall.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #ifndef CLONE_NEWNS #define CLONE_NEWNS 0x00020000 diff --git a/tools/testing/selftests/mqueue/mq_open_tests.c b/tools/testing/selftests/mqueue/mq_open_tests.c index 9403ac01ba11..b16029c40c0f 100644 --- a/tools/testing/selftests/mqueue/mq_open_tests.c +++ b/tools/testing/selftests/mqueue/mq_open_tests.c @@ -33,7 +33,7 @@ #include <mqueue.h> #include <error.h> -#include "../kselftest.h" +#include "kselftest.h" static char *usage = "Usage:\n" diff --git a/tools/testing/selftests/mqueue/mq_perf_tests.c b/tools/testing/selftests/mqueue/mq_perf_tests.c index fb898850867c..303c46eebd94 100644 --- a/tools/testing/selftests/mqueue/mq_perf_tests.c +++ b/tools/testing/selftests/mqueue/mq_perf_tests.c @@ -40,7 +40,7 @@ #include <popt.h> #include <error.h> -#include "../kselftest.h" +#include "kselftest.h" static char *usage = "Usage:\n" diff --git a/tools/testing/selftests/mseal_system_mappings/sysmap_is_sealed.c b/tools/testing/selftests/mseal_system_mappings/sysmap_is_sealed.c index 0d2af30c3bf5..cb0ca6ed7ebe 100644 --- a/tools/testing/selftests/mseal_system_mappings/sysmap_is_sealed.c +++ b/tools/testing/selftests/mseal_system_mappings/sysmap_is_sealed.c @@ -11,8 +11,8 @@ #include <string.h> #include <stdbool.h> -#include "../kselftest.h" -#include "../kselftest_harness.h" +#include "kselftest.h" +#include "kselftest_harness.h" #define VMFLAGS "VmFlags:" #define MSEAL_FLAGS "sl" diff --git a/tools/testing/selftests/namespaces/file_handle_test.c b/tools/testing/selftests/namespaces/file_handle_test.c index f1bc5773f552..064b41ad96b2 100644 --- a/tools/testing/selftests/namespaces/file_handle_test.c +++ b/tools/testing/selftests/namespaces/file_handle_test.c @@ -14,7 +14,7 @@ #include <sys/wait.h> #include <unistd.h> #include <linux/unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #ifndef FD_NSFS_ROOT #define FD_NSFS_ROOT -10003 /* Root of the nsfs filesystem */ diff --git a/tools/testing/selftests/namespaces/init_ino_test.c b/tools/testing/selftests/namespaces/init_ino_test.c index 5b6993c3740b..e4394a2fa0a9 100644 --- a/tools/testing/selftests/namespaces/init_ino_test.c +++ b/tools/testing/selftests/namespaces/init_ino_test.c @@ -11,7 +11,7 @@ #include <string.h> #include <linux/nsfs.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" struct ns_info { const char *name; diff --git a/tools/testing/selftests/namespaces/nsid_test.c b/tools/testing/selftests/namespaces/nsid_test.c index 527ade0a8673..b4a14c6693a5 100644 --- a/tools/testing/selftests/namespaces/nsid_test.c +++ b/tools/testing/selftests/namespaces/nsid_test.c @@ -20,7 +20,7 @@ #include <linux/fs.h> #include <linux/limits.h> #include <linux/nsfs.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" /* Fixture for tests that create child processes */ FIXTURE(nsid) { diff --git a/tools/testing/selftests/nci/nci_dev.c b/tools/testing/selftests/nci/nci_dev.c index 6dec59d64083..312f84ee0444 100644 --- a/tools/testing/selftests/nci/nci_dev.c +++ b/tools/testing/selftests/nci/nci_dev.c @@ -16,7 +16,7 @@ #include <sys/socket.h> #include <linux/nfc.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define GENLMSG_DATA(glh) ((void *)(NLMSG_DATA(glh) + GENL_HDRLEN)) #define GENLMSG_PAYLOAD(glh) (NLMSG_PAYLOAD(glh, 0) - GENL_HDRLEN) diff --git a/tools/testing/selftests/net/af_unix/diag_uid.c b/tools/testing/selftests/net/af_unix/diag_uid.c index 79a3dd75590e..da7d50cedee6 100644 --- a/tools/testing/selftests/net/af_unix/diag_uid.c +++ b/tools/testing/selftests/net/af_unix/diag_uid.c @@ -14,7 +14,7 @@ #include <sys/types.h> #include <sys/un.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" FIXTURE(diag_uid) { diff --git a/tools/testing/selftests/net/af_unix/msg_oob.c b/tools/testing/selftests/net/af_unix/msg_oob.c index b5f474969917..1b499d56656c 100644 --- a/tools/testing/selftests/net/af_unix/msg_oob.c +++ b/tools/testing/selftests/net/af_unix/msg_oob.c @@ -11,7 +11,7 @@ #include <sys/signalfd.h> #include <sys/socket.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define BUF_SZ 32 diff --git a/tools/testing/selftests/net/af_unix/scm_inq.c b/tools/testing/selftests/net/af_unix/scm_inq.c index fc467714387e..3a86be9bda17 100644 --- a/tools/testing/selftests/net/af_unix/scm_inq.c +++ b/tools/testing/selftests/net/af_unix/scm_inq.c @@ -6,7 +6,7 @@ #include <sys/socket.h> #include <sys/types.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define NR_CHUNKS 100 #define MSG_LEN 256 diff --git a/tools/testing/selftests/net/af_unix/scm_pidfd.c b/tools/testing/selftests/net/af_unix/scm_pidfd.c index ef2921988e5f..2c18b92a2603 100644 --- a/tools/testing/selftests/net/af_unix/scm_pidfd.c +++ b/tools/testing/selftests/net/af_unix/scm_pidfd.c @@ -16,7 +16,7 @@ #include <sys/wait.h> #include "../../pidfd/pidfd.h" -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define clean_errno() (errno == 0 ? "None" : strerror(errno)) #define log_err(MSG, ...) \ diff --git a/tools/testing/selftests/net/af_unix/scm_rights.c b/tools/testing/selftests/net/af_unix/scm_rights.c index 914f99d153ce..d82a79c21c17 100644 --- a/tools/testing/selftests/net/af_unix/scm_rights.c +++ b/tools/testing/selftests/net/af_unix/scm_rights.c @@ -10,7 +10,7 @@ #include <sys/socket.h> #include <sys/un.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" FIXTURE(scm_rights) { diff --git a/tools/testing/selftests/net/af_unix/unix_connect.c b/tools/testing/selftests/net/af_unix/unix_connect.c index d799fd8f5c7c..870ca96fa8ea 100644 --- a/tools/testing/selftests/net/af_unix/unix_connect.c +++ b/tools/testing/selftests/net/af_unix/unix_connect.c @@ -10,7 +10,7 @@ #include <sys/socket.h> #include <sys/un.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" FIXTURE(unix_connect) { diff --git a/tools/testing/selftests/net/bind_timewait.c b/tools/testing/selftests/net/bind_timewait.c index cb9fdf51ea59..40126f9b901e 100644 --- a/tools/testing/selftests/net/bind_timewait.c +++ b/tools/testing/selftests/net/bind_timewait.c @@ -4,7 +4,7 @@ #include <sys/socket.h> #include <netinet/in.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" FIXTURE(bind_timewait) { diff --git a/tools/testing/selftests/net/bind_wildcard.c b/tools/testing/selftests/net/bind_wildcard.c index b7b54d646b93..7d11548b2c61 100644 --- a/tools/testing/selftests/net/bind_wildcard.c +++ b/tools/testing/selftests/net/bind_wildcard.c @@ -4,7 +4,7 @@ #include <sys/socket.h> #include <netinet/in.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static const __u32 in4addr_any = INADDR_ANY; static const __u32 in4addr_loopback = INADDR_LOOPBACK; diff --git a/tools/testing/selftests/net/can/test_raw_filter.c b/tools/testing/selftests/net/can/test_raw_filter.c index 4101c36390fd..bb8ae8854273 100644 --- a/tools/testing/selftests/net/can/test_raw_filter.c +++ b/tools/testing/selftests/net/can/test_raw_filter.c @@ -19,7 +19,7 @@ #include <linux/can.h> #include <linux/can/raw.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define ID 0x123 diff --git a/tools/testing/selftests/net/cmsg_sender.c b/tools/testing/selftests/net/cmsg_sender.c index ded9b925865e..67a72b1a2f3d 100644 --- a/tools/testing/selftests/net/cmsg_sender.c +++ b/tools/testing/selftests/net/cmsg_sender.c @@ -16,7 +16,7 @@ #include <linux/udp.h> #include <sys/socket.h> -#include "../kselftest.h" +#include "kselftest.h" enum { ERN_SUCCESS = 0, diff --git a/tools/testing/selftests/net/epoll_busy_poll.c b/tools/testing/selftests/net/epoll_busy_poll.c index 16e457c2f877..adf8dd0b5e0b 100644 --- a/tools/testing/selftests/net/epoll_busy_poll.c +++ b/tools/testing/selftests/net/epoll_busy_poll.c @@ -23,7 +23,7 @@ #include <sys/ioctl.h> #include <sys/socket.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" /* if the headers haven't been updated, we need to define some things */ #if !defined(EPOLL_IOC_TYPE) diff --git a/tools/testing/selftests/net/ip_local_port_range.c b/tools/testing/selftests/net/ip_local_port_range.c index 29451d2244b7..e6834a6cfc8f 100644 --- a/tools/testing/selftests/net/ip_local_port_range.c +++ b/tools/testing/selftests/net/ip_local_port_range.c @@ -10,7 +10,7 @@ #include <fcntl.h> #include <netinet/ip.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #ifndef IP_LOCAL_PORT_RANGE #define IP_LOCAL_PORT_RANGE 51 diff --git a/tools/testing/selftests/net/ipsec.c b/tools/testing/selftests/net/ipsec.c index 9b44a091802c..0ccf484b1d9d 100644 --- a/tools/testing/selftests/net/ipsec.c +++ b/tools/testing/selftests/net/ipsec.c @@ -34,7 +34,7 @@ #include <time.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #define printk(fmt, ...) \ ksft_print_msg("%d[%u] " fmt "\n", getpid(), __LINE__, ##__VA_ARGS__) diff --git a/tools/testing/selftests/net/ipv6_fragmentation.c b/tools/testing/selftests/net/ipv6_fragmentation.c index 267ef62b5c72..672c9fe086a7 100644 --- a/tools/testing/selftests/net/ipv6_fragmentation.c +++ b/tools/testing/selftests/net/ipv6_fragmentation.c @@ -34,7 +34,7 @@ #include <sys/ioctl.h> #include <sys/socket.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #define MTU 1500 #define LARGER_THAN_MTU 8192 diff --git a/tools/testing/selftests/net/netfilter/conntrack_dump_flush.c b/tools/testing/selftests/net/netfilter/conntrack_dump_flush.c index 5f827e10717d..5cecb8a1bc94 100644 --- a/tools/testing/selftests/net/netfilter/conntrack_dump_flush.c +++ b/tools/testing/selftests/net/netfilter/conntrack_dump_flush.c @@ -10,7 +10,7 @@ #include <linux/netfilter/nfnetlink.h> #include <linux/netfilter/nfnetlink_conntrack.h> #include <linux/netfilter/nf_conntrack_tcp.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define TEST_ZONE_ID 123 #define NF_CT_DEFAULT_ZONE_ID 0 diff --git a/tools/testing/selftests/net/netlink-dumps.c b/tools/testing/selftests/net/netlink-dumps.c index 679b6c77ace7..51129c564d0a 100644 --- a/tools/testing/selftests/net/netlink-dumps.c +++ b/tools/testing/selftests/net/netlink-dumps.c @@ -18,7 +18,7 @@ #include <linux/mqueue.h> #include <linux/rtnetlink.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <ynl.h> diff --git a/tools/testing/selftests/net/ovpn/ovpn-cli.c b/tools/testing/selftests/net/ovpn/ovpn-cli.c index 0a5226196a2e..0f3babf19fd0 100644 --- a/tools/testing/selftests/net/ovpn/ovpn-cli.c +++ b/tools/testing/selftests/net/ovpn/ovpn-cli.c @@ -32,7 +32,7 @@ #include <sys/socket.h> -#include "../../kselftest.h" +#include "kselftest.h" /* defines to make checkpatch happy */ #define strscpy strncpy diff --git a/tools/testing/selftests/net/proc_net_pktgen.c b/tools/testing/selftests/net/proc_net_pktgen.c index 69444fb29577..fab3b5c2e25d 100644 --- a/tools/testing/selftests/net/proc_net_pktgen.c +++ b/tools/testing/selftests/net/proc_net_pktgen.c @@ -10,7 +10,7 @@ #include <stdlib.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static const char ctrl_cmd_stop[] = "stop"; static const char ctrl_cmd_start[] = "start"; diff --git a/tools/testing/selftests/net/psock_fanout.c b/tools/testing/selftests/net/psock_fanout.c index 84c524357075..ab8d8b7e6cb0 100644 --- a/tools/testing/selftests/net/psock_fanout.c +++ b/tools/testing/selftests/net/psock_fanout.c @@ -54,7 +54,7 @@ #include <unistd.h> #include "psock_lib.h" -#include "../kselftest.h" +#include "kselftest.h" #define RING_NUM_FRAMES 20 diff --git a/tools/testing/selftests/net/psock_tpacket.c b/tools/testing/selftests/net/psock_tpacket.c index 2938045c5cf9..7caf3135448d 100644 --- a/tools/testing/selftests/net/psock_tpacket.c +++ b/tools/testing/selftests/net/psock_tpacket.c @@ -46,7 +46,7 @@ #include "psock_lib.h" -#include "../kselftest.h" +#include "kselftest.h" #ifndef bug_on # define bug_on(cond) assert(!(cond)) diff --git a/tools/testing/selftests/net/reuseaddr_ports_exhausted.c b/tools/testing/selftests/net/reuseaddr_ports_exhausted.c index 7b9bf8a7bbe1..5aad27a0d13a 100644 --- a/tools/testing/selftests/net/reuseaddr_ports_exhausted.c +++ b/tools/testing/selftests/net/reuseaddr_ports_exhausted.c @@ -22,7 +22,7 @@ #include <sys/socket.h> #include <sys/types.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" struct reuse_opts { int reuseaddr[2]; diff --git a/tools/testing/selftests/net/reuseport_bpf.c b/tools/testing/selftests/net/reuseport_bpf.c index 65aea27d761c..b6634d6da3d6 100644 --- a/tools/testing/selftests/net/reuseport_bpf.c +++ b/tools/testing/selftests/net/reuseport_bpf.c @@ -24,7 +24,7 @@ #include <sys/resource.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" struct test_params { int recv_family; diff --git a/tools/testing/selftests/net/reuseport_bpf_numa.c b/tools/testing/selftests/net/reuseport_bpf_numa.c index c9ba36aa688e..2ffd957ffb15 100644 --- a/tools/testing/selftests/net/reuseport_bpf_numa.c +++ b/tools/testing/selftests/net/reuseport_bpf_numa.c @@ -23,7 +23,7 @@ #include <unistd.h> #include <numa.h> -#include "../kselftest.h" +#include "kselftest.h" static const int PORT = 8888; diff --git a/tools/testing/selftests/net/rxtimestamp.c b/tools/testing/selftests/net/rxtimestamp.c index 16ac4df55fdb..b81ed0352d6c 100644 --- a/tools/testing/selftests/net/rxtimestamp.c +++ b/tools/testing/selftests/net/rxtimestamp.c @@ -18,7 +18,7 @@ #include <linux/net_tstamp.h> #include <linux/errqueue.h> -#include "../kselftest.h" +#include "kselftest.h" struct options { int so_timestamp; diff --git a/tools/testing/selftests/net/sk_so_peek_off.c b/tools/testing/selftests/net/sk_so_peek_off.c index d87dd8d8d491..2a3f5c604f52 100644 --- a/tools/testing/selftests/net/sk_so_peek_off.c +++ b/tools/testing/selftests/net/sk_so_peek_off.c @@ -8,7 +8,7 @@ #include <sys/types.h> #include <netinet/in.h> #include <arpa/inet.h> -#include "../kselftest.h" +#include "kselftest.h" static char *afstr(int af, int proto) { diff --git a/tools/testing/selftests/net/so_incoming_cpu.c b/tools/testing/selftests/net/so_incoming_cpu.c index e9fa14e10732..4740701f1a9a 100644 --- a/tools/testing/selftests/net/so_incoming_cpu.c +++ b/tools/testing/selftests/net/so_incoming_cpu.c @@ -9,7 +9,7 @@ #include <sys/socket.h> #include <sys/sysinfo.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" FIXTURE(so_incoming_cpu) { diff --git a/tools/testing/selftests/net/socket.c b/tools/testing/selftests/net/socket.c index be1080003c61..9e270548dad8 100644 --- a/tools/testing/selftests/net/socket.c +++ b/tools/testing/selftests/net/socket.c @@ -7,7 +7,7 @@ #include <sys/socket.h> #include <netinet/in.h> -#include "../kselftest.h" +#include "kselftest.h" struct socket_testcase { int domain; diff --git a/tools/testing/selftests/net/tap.c b/tools/testing/selftests/net/tap.c index 247c3b3ac1c9..9ec1c9b50e77 100644 --- a/tools/testing/selftests/net/tap.c +++ b/tools/testing/selftests/net/tap.c @@ -17,7 +17,7 @@ #include <linux/virtio_net.h> #include <netinet/ip.h> #include <netinet/udp.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static const char param_dev_tap_name[] = "xmacvtap0"; static const char param_dev_dummy_name[] = "xdummy0"; diff --git a/tools/testing/selftests/net/tcp_ao/lib/setup.c b/tools/testing/selftests/net/tcp_ao/lib/setup.c index a27cc03c9fbd..49aec2922a31 100644 --- a/tools/testing/selftests/net/tcp_ao/lib/setup.c +++ b/tools/testing/selftests/net/tcp_ao/lib/setup.c @@ -9,7 +9,7 @@ * Can't be included in the header: it defines static variables which * will be unique to every object. Let's include it only once here. */ -#include "../../../kselftest.h" +#include "kselftest.h" /* Prevent overriding of one thread's output by another */ static pthread_mutex_t ksft_print_lock = PTHREAD_MUTEX_INITIALIZER; diff --git a/tools/testing/selftests/net/tcp_fastopen_backup_key.c b/tools/testing/selftests/net/tcp_fastopen_backup_key.c index c1cb0c75156a..4b3f9b5e50fe 100644 --- a/tools/testing/selftests/net/tcp_fastopen_backup_key.c +++ b/tools/testing/selftests/net/tcp_fastopen_backup_key.c @@ -26,7 +26,7 @@ #include <fcntl.h> #include <time.h> -#include "../kselftest.h" +#include "kselftest.h" #ifndef TCP_FASTOPEN_KEY #define TCP_FASTOPEN_KEY 33 diff --git a/tools/testing/selftests/net/tcp_port_share.c b/tools/testing/selftests/net/tcp_port_share.c index 4c39d599dfce..6146b62610df 100644 --- a/tools/testing/selftests/net/tcp_port_share.c +++ b/tools/testing/selftests/net/tcp_port_share.c @@ -10,7 +10,7 @@ #include <sched.h> #include <stdlib.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define DST_PORT 30000 #define SRC_PORT 40000 diff --git a/tools/testing/selftests/net/tls.c b/tools/testing/selftests/net/tls.c index da1b50b30719..a3ef4b57eb5f 100644 --- a/tools/testing/selftests/net/tls.c +++ b/tools/testing/selftests/net/tls.c @@ -21,7 +21,7 @@ #include <sys/socket.h> #include <sys/stat.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define TLS_PAYLOAD_MAX_LEN 16384 #define SOL_TLS 282 diff --git a/tools/testing/selftests/net/tun.c b/tools/testing/selftests/net/tun.c index fa83918b62d1..0efc67b0357a 100644 --- a/tools/testing/selftests/net/tun.c +++ b/tools/testing/selftests/net/tun.c @@ -15,7 +15,7 @@ #include <sys/ioctl.h> #include <sys/socket.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static int tun_attach(int fd, char *dev) { diff --git a/tools/testing/selftests/net/udpgso_bench_tx.c b/tools/testing/selftests/net/udpgso_bench_tx.c index 477392715a9a..86d80cce55b4 100644 --- a/tools/testing/selftests/net/udpgso_bench_tx.c +++ b/tools/testing/selftests/net/udpgso_bench_tx.c @@ -25,7 +25,7 @@ #include <sys/types.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #ifndef ETH_MAX_MTU #define ETH_MAX_MTU 0xFFFFU diff --git a/tools/testing/selftests/openat2/helpers.h b/tools/testing/selftests/openat2/helpers.h index 7056340b9339..510e60602511 100644 --- a/tools/testing/selftests/openat2/helpers.h +++ b/tools/testing/selftests/openat2/helpers.h @@ -12,7 +12,7 @@ #include <stdbool.h> #include <errno.h> #include <linux/types.h> -#include "../kselftest.h" +#include "kselftest.h" #define ARRAY_LEN(X) (sizeof (X) / sizeof (*(X))) #define BUILD_BUG_ON(e) ((void)(sizeof(struct { int:(-!!(e)); }))) diff --git a/tools/testing/selftests/openat2/openat2_test.c b/tools/testing/selftests/openat2/openat2_test.c index 5790ab446527..0e161ef9e9e4 100644 --- a/tools/testing/selftests/openat2/openat2_test.c +++ b/tools/testing/selftests/openat2/openat2_test.c @@ -15,7 +15,7 @@ #include <stdbool.h> #include <string.h> -#include "../kselftest.h" +#include "kselftest.h" #include "helpers.h" /* diff --git a/tools/testing/selftests/openat2/rename_attack_test.c b/tools/testing/selftests/openat2/rename_attack_test.c index 0a770728b436..aa5699e45729 100644 --- a/tools/testing/selftests/openat2/rename_attack_test.c +++ b/tools/testing/selftests/openat2/rename_attack_test.c @@ -22,7 +22,7 @@ #include <limits.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #include "helpers.h" /* Construct a test directory with the following structure: diff --git a/tools/testing/selftests/openat2/resolve_test.c b/tools/testing/selftests/openat2/resolve_test.c index bbafad440893..a76ef15ceb90 100644 --- a/tools/testing/selftests/openat2/resolve_test.c +++ b/tools/testing/selftests/openat2/resolve_test.c @@ -14,7 +14,7 @@ #include <stdbool.h> #include <string.h> -#include "../kselftest.h" +#include "kselftest.h" #include "helpers.h" /* diff --git a/tools/testing/selftests/pci_endpoint/pci_endpoint_test.c b/tools/testing/selftests/pci_endpoint/pci_endpoint_test.c index cd9075444c32..23aac6f97061 100644 --- a/tools/testing/selftests/pci_endpoint/pci_endpoint_test.c +++ b/tools/testing/selftests/pci_endpoint/pci_endpoint_test.c @@ -20,7 +20,7 @@ #include "../../../../include/uapi/linux/pcitest.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define pci_ep_ioctl(cmd, arg) \ ({ \ diff --git a/tools/testing/selftests/perf_events/mmap.c b/tools/testing/selftests/perf_events/mmap.c index ea0427aac1f9..d1fa8ec58987 100644 --- a/tools/testing/selftests/perf_events/mmap.c +++ b/tools/testing/selftests/perf_events/mmap.c @@ -14,7 +14,7 @@ #include <linux/perf_event.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define RB_SIZE 0x3000 #define AUX_SIZE 0x10000 diff --git a/tools/testing/selftests/perf_events/remove_on_exec.c b/tools/testing/selftests/perf_events/remove_on_exec.c index 5814611a1dc7..89e7b06835df 100644 --- a/tools/testing/selftests/perf_events/remove_on_exec.c +++ b/tools/testing/selftests/perf_events/remove_on_exec.c @@ -30,7 +30,7 @@ #include <sys/syscall.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static volatile int signal_count; diff --git a/tools/testing/selftests/perf_events/sigtrap_threads.c b/tools/testing/selftests/perf_events/sigtrap_threads.c index d1d8483ac628..b5cf8355345d 100644 --- a/tools/testing/selftests/perf_events/sigtrap_threads.c +++ b/tools/testing/selftests/perf_events/sigtrap_threads.c @@ -31,7 +31,7 @@ #include <sys/syscall.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define NUM_THREADS 5 diff --git a/tools/testing/selftests/perf_events/watermark_signal.c b/tools/testing/selftests/perf_events/watermark_signal.c index b3a72f0ac522..0f64b9b17081 100644 --- a/tools/testing/selftests/perf_events/watermark_signal.c +++ b/tools/testing/selftests/perf_events/watermark_signal.c @@ -15,7 +15,7 @@ #include <sys/wait.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static int sigio_count; diff --git a/tools/testing/selftests/pid_namespace/pid_max.c b/tools/testing/selftests/pid_namespace/pid_max.c index 96f274f0582b..c9519e7385b6 100644 --- a/tools/testing/selftests/pid_namespace/pid_max.c +++ b/tools/testing/selftests/pid_namespace/pid_max.c @@ -13,7 +13,7 @@ #include <sys/mount.h> #include <sys/wait.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "../pidfd/pidfd.h" #define __STACK_SIZE (8 * 1024 * 1024) diff --git a/tools/testing/selftests/pid_namespace/regression_enomem.c b/tools/testing/selftests/pid_namespace/regression_enomem.c index 7d84097ad45c..059e7ec5b4fd 100644 --- a/tools/testing/selftests/pid_namespace/regression_enomem.c +++ b/tools/testing/selftests/pid_namespace/regression_enomem.c @@ -11,7 +11,7 @@ #include <syscall.h> #include <sys/wait.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "../pidfd/pidfd.h" /* diff --git a/tools/testing/selftests/pidfd/pidfd.h b/tools/testing/selftests/pidfd/pidfd.h index d60f10a873bb..9085c1a3c005 100644 --- a/tools/testing/selftests/pidfd/pidfd.h +++ b/tools/testing/selftests/pidfd/pidfd.h @@ -25,7 +25,7 @@ #undef SCHED_FLAG_KEEP_ALL #undef SCHED_FLAG_UTIL_CLAMP -#include "../kselftest.h" +#include "kselftest.h" #include "../clone3/clone3_selftests.h" #ifndef FD_PIDFS_ROOT diff --git a/tools/testing/selftests/pidfd/pidfd_bind_mount.c b/tools/testing/selftests/pidfd/pidfd_bind_mount.c index c094aeb1c620..1fdf49939524 100644 --- a/tools/testing/selftests/pidfd/pidfd_bind_mount.c +++ b/tools/testing/selftests/pidfd/pidfd_bind_mount.c @@ -14,7 +14,7 @@ #include <unistd.h> #include "pidfd.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "../filesystems/wrappers.h" FIXTURE(pidfd_bind_mount) { diff --git a/tools/testing/selftests/pidfd/pidfd_fdinfo_test.c b/tools/testing/selftests/pidfd/pidfd_fdinfo_test.c index f718aac75068..9935e9471c77 100644 --- a/tools/testing/selftests/pidfd/pidfd_fdinfo_test.c +++ b/tools/testing/selftests/pidfd/pidfd_fdinfo_test.c @@ -16,7 +16,7 @@ #include <sys/mount.h> #include "pidfd.h" -#include "../kselftest.h" +#include "kselftest.h" struct error { int code; diff --git a/tools/testing/selftests/pidfd/pidfd_file_handle_test.c b/tools/testing/selftests/pidfd/pidfd_file_handle_test.c index 6bd2e9c9565b..68918734dcf3 100644 --- a/tools/testing/selftests/pidfd/pidfd_file_handle_test.c +++ b/tools/testing/selftests/pidfd/pidfd_file_handle_test.c @@ -20,7 +20,7 @@ #include <sys/stat.h> #include "pidfd.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" FIXTURE(file_handle) { diff --git a/tools/testing/selftests/pidfd/pidfd_getfd_test.c b/tools/testing/selftests/pidfd/pidfd_getfd_test.c index cd51d547b751..ea45b37001b0 100644 --- a/tools/testing/selftests/pidfd/pidfd_getfd_test.c +++ b/tools/testing/selftests/pidfd/pidfd_getfd_test.c @@ -19,7 +19,7 @@ #include <linux/kcmp.h> #include "pidfd.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" /* * UNKNOWN_FD is an fd number that should never exist in the child, as it is diff --git a/tools/testing/selftests/pidfd/pidfd_info_test.c b/tools/testing/selftests/pidfd/pidfd_info_test.c index cb5430a2fd75..6571e04acd88 100644 --- a/tools/testing/selftests/pidfd/pidfd_info_test.c +++ b/tools/testing/selftests/pidfd/pidfd_info_test.c @@ -21,7 +21,7 @@ #include <sys/stat.h> #include "pidfd.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" FIXTURE(pidfd_info) { diff --git a/tools/testing/selftests/pidfd/pidfd_open_test.c b/tools/testing/selftests/pidfd/pidfd_open_test.c index cd3de40e4977..318e6f09c8e0 100644 --- a/tools/testing/selftests/pidfd/pidfd_open_test.c +++ b/tools/testing/selftests/pidfd/pidfd_open_test.c @@ -20,7 +20,7 @@ #include <unistd.h> #include "pidfd.h" -#include "../kselftest.h" +#include "kselftest.h" static int safe_int(const char *numstr, int *converted) { diff --git a/tools/testing/selftests/pidfd/pidfd_poll_test.c b/tools/testing/selftests/pidfd/pidfd_poll_test.c index 55d74a50358f..232304f818c7 100644 --- a/tools/testing/selftests/pidfd/pidfd_poll_test.c +++ b/tools/testing/selftests/pidfd/pidfd_poll_test.c @@ -14,7 +14,7 @@ #include <unistd.h> #include "pidfd.h" -#include "../kselftest.h" +#include "kselftest.h" static bool timeout; diff --git a/tools/testing/selftests/pidfd/pidfd_setattr_test.c b/tools/testing/selftests/pidfd/pidfd_setattr_test.c index d7de05edc4b3..e8562a2992f3 100644 --- a/tools/testing/selftests/pidfd/pidfd_setattr_test.c +++ b/tools/testing/selftests/pidfd/pidfd_setattr_test.c @@ -22,7 +22,7 @@ #include <sys/xattr.h> #include "pidfd.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" FIXTURE(pidfs_setattr) { diff --git a/tools/testing/selftests/pidfd/pidfd_setns_test.c b/tools/testing/selftests/pidfd/pidfd_setns_test.c index e6a079b3d5e2..107edecff224 100644 --- a/tools/testing/selftests/pidfd/pidfd_setns_test.c +++ b/tools/testing/selftests/pidfd/pidfd_setns_test.c @@ -18,7 +18,7 @@ #include <sys/stat.h> #include "pidfd.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" enum { PIDFD_NS_USER, diff --git a/tools/testing/selftests/pidfd/pidfd_test.c b/tools/testing/selftests/pidfd/pidfd_test.c index fcd85cad9f18..932cbd8caa77 100644 --- a/tools/testing/selftests/pidfd/pidfd_test.c +++ b/tools/testing/selftests/pidfd/pidfd_test.c @@ -20,7 +20,7 @@ #include <unistd.h> #include "pidfd.h" -#include "../kselftest.h" +#include "kselftest.h" #define str(s) _str(s) #define _str(s) #s diff --git a/tools/testing/selftests/pidfd/pidfd_wait.c b/tools/testing/selftests/pidfd/pidfd_wait.c index 1e2d49751cde..4bf702d62c1c 100644 --- a/tools/testing/selftests/pidfd/pidfd_wait.c +++ b/tools/testing/selftests/pidfd/pidfd_wait.c @@ -17,7 +17,7 @@ #include <unistd.h> #include "pidfd.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define ptr_to_u64(ptr) ((__u64)((uintptr_t)(ptr))) diff --git a/tools/testing/selftests/pidfd/pidfd_xattr_test.c b/tools/testing/selftests/pidfd/pidfd_xattr_test.c index 5cf7bb0e4bf2..fd57511af7e4 100644 --- a/tools/testing/selftests/pidfd/pidfd_xattr_test.c +++ b/tools/testing/selftests/pidfd/pidfd_xattr_test.c @@ -22,7 +22,7 @@ #include <sys/xattr.h> #include "pidfd.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" FIXTURE(pidfs_xattr) { diff --git a/tools/testing/selftests/prctl/set-anon-vma-name-test.c b/tools/testing/selftests/prctl/set-anon-vma-name-test.c index 4275cb256dce..ac6721b184a6 100644 --- a/tools/testing/selftests/prctl/set-anon-vma-name-test.c +++ b/tools/testing/selftests/prctl/set-anon-vma-name-test.c @@ -10,7 +10,7 @@ #include <sys/mman.h> #include <string.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define AREA_SIZE 1024 diff --git a/tools/testing/selftests/prctl/set-process-name.c b/tools/testing/selftests/prctl/set-process-name.c index 562f707ba771..3f7b146d36df 100644 --- a/tools/testing/selftests/prctl/set-process-name.c +++ b/tools/testing/selftests/prctl/set-process-name.c @@ -7,7 +7,7 @@ #include <sys/prctl.h> #include <string.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define CHANGE_NAME "changename" #define EMPTY_NAME "" diff --git a/tools/testing/selftests/proc/proc-maps-race.c b/tools/testing/selftests/proc/proc-maps-race.c index a546475db550..a734553718da 100644 --- a/tools/testing/selftests/proc/proc-maps-race.c +++ b/tools/testing/selftests/proc/proc-maps-race.c @@ -23,7 +23,7 @@ * */ #define _GNU_SOURCE -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <errno.h> #include <fcntl.h> #include <pthread.h> diff --git a/tools/testing/selftests/proc/proc-pid-vm.c b/tools/testing/selftests/proc/proc-pid-vm.c index 978cbcb3eb11..4e6a3e53f975 100644 --- a/tools/testing/selftests/proc/proc-pid-vm.c +++ b/tools/testing/selftests/proc/proc-pid-vm.c @@ -51,7 +51,7 @@ #define __maybe_unused __attribute__((__unused__)) #endif -#include "../kselftest.h" +#include "kselftest.h" static inline long sys_execveat(int dirfd, const char *pathname, char **argv, char **envp, int flags) { diff --git a/tools/testing/selftests/proc/proc-pidns.c b/tools/testing/selftests/proc/proc-pidns.c index 52500597f951..25b9a2933c45 100644 --- a/tools/testing/selftests/proc/proc-pidns.c +++ b/tools/testing/selftests/proc/proc-pidns.c @@ -16,7 +16,7 @@ #include <sys/stat.h> #include <sys/prctl.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define ASSERT_ERRNO(expected, _t, seen) \ __EXPECT(expected, #expected, \ diff --git a/tools/testing/selftests/ptrace/get_set_sud.c b/tools/testing/selftests/ptrace/get_set_sud.c index 5297b10d25c3..2e619c7599bb 100644 --- a/tools/testing/selftests/ptrace/get_set_sud.c +++ b/tools/testing/selftests/ptrace/get_set_sud.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 #define _GNU_SOURCE -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <stdio.h> #include <string.h> #include <errno.h> diff --git a/tools/testing/selftests/ptrace/get_syscall_info.c b/tools/testing/selftests/ptrace/get_syscall_info.c index 5bcd1c7b5be6..3f5c3a9fdaba 100644 --- a/tools/testing/selftests/ptrace/get_syscall_info.c +++ b/tools/testing/selftests/ptrace/get_syscall_info.c @@ -7,7 +7,7 @@ * matches userspace expectations. */ -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <err.h> #include <signal.h> #include <asm/unistd.h> diff --git a/tools/testing/selftests/ptrace/set_syscall_info.c b/tools/testing/selftests/ptrace/set_syscall_info.c index 4198248ef874..1cc411a41cd6 100644 --- a/tools/testing/selftests/ptrace/set_syscall_info.c +++ b/tools/testing/selftests/ptrace/set_syscall_info.c @@ -7,7 +7,7 @@ * matches userspace expectations. */ -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <err.h> #include <fcntl.h> #include <signal.h> diff --git a/tools/testing/selftests/ptrace/vmaccess.c b/tools/testing/selftests/ptrace/vmaccess.c index 4db327b44586..3801b5831527 100644 --- a/tools/testing/selftests/ptrace/vmaccess.c +++ b/tools/testing/selftests/ptrace/vmaccess.c @@ -7,7 +7,7 @@ * when de_thread is blocked with ->cred_guard_mutex held. */ -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include <stdio.h> #include <fcntl.h> #include <pthread.h> diff --git a/tools/testing/selftests/resctrl/resctrl.h b/tools/testing/selftests/resctrl/resctrl.h index cd3adfc14969..3c51bdac2dfa 100644 --- a/tools/testing/selftests/resctrl/resctrl.h +++ b/tools/testing/selftests/resctrl/resctrl.h @@ -23,7 +23,7 @@ #include <asm/unistd.h> #include <linux/perf_event.h> #include <linux/compiler.h> -#include "../kselftest.h" +#include "kselftest.h" #define MB (1024 * 1024) #define RESCTRL_PATH "/sys/fs/resctrl" diff --git a/tools/testing/selftests/ring-buffer/map_test.c b/tools/testing/selftests/ring-buffer/map_test.c index a58f520f2f41..f24677737066 100644 --- a/tools/testing/selftests/ring-buffer/map_test.c +++ b/tools/testing/selftests/ring-buffer/map_test.c @@ -17,7 +17,7 @@ #include <sys/ioctl.h> #include "../user_events/user_events_selftests.h" /* share tracefs setup */ -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define TRACEFS_ROOT "/sys/kernel/tracing" diff --git a/tools/testing/selftests/riscv/abi/pointer_masking.c b/tools/testing/selftests/riscv/abi/pointer_masking.c index 059d2e87eb1f..2d540af7b558 100644 --- a/tools/testing/selftests/riscv/abi/pointer_masking.c +++ b/tools/testing/selftests/riscv/abi/pointer_masking.c @@ -9,7 +9,7 @@ #include <sys/wait.h> #include <unistd.h> -#include "../../kselftest.h" +#include "kselftest.h" #ifndef PR_PMLEN_SHIFT #define PR_PMLEN_SHIFT 24 diff --git a/tools/testing/selftests/riscv/hwprobe/cbo.c b/tools/testing/selftests/riscv/hwprobe/cbo.c index 6d99726aceac..f254b2edd6ce 100644 --- a/tools/testing/selftests/riscv/hwprobe/cbo.c +++ b/tools/testing/selftests/riscv/hwprobe/cbo.c @@ -18,7 +18,7 @@ #include <getopt.h> #include "hwprobe.h" -#include "../../kselftest.h" +#include "kselftest.h" #define MK_CBO(fn) le32_bswap((uint32_t)(fn) << 20 | 10 << 15 | 2 << 12 | 0 << 7 | 15) #define MK_PREFETCH(fn) \ diff --git a/tools/testing/selftests/riscv/hwprobe/hwprobe.c b/tools/testing/selftests/riscv/hwprobe/hwprobe.c index fd73c87804f3..54c435af9923 100644 --- a/tools/testing/selftests/riscv/hwprobe/hwprobe.c +++ b/tools/testing/selftests/riscv/hwprobe/hwprobe.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only #include "hwprobe.h" -#include "../../kselftest.h" +#include "kselftest.h" int main(int argc, char **argv) { diff --git a/tools/testing/selftests/riscv/hwprobe/which-cpus.c b/tools/testing/selftests/riscv/hwprobe/which-cpus.c index 82c121412dfc..3ab53067e8dd 100644 --- a/tools/testing/selftests/riscv/hwprobe/which-cpus.c +++ b/tools/testing/selftests/riscv/hwprobe/which-cpus.c @@ -14,7 +14,7 @@ #include <assert.h> #include "hwprobe.h" -#include "../../kselftest.h" +#include "kselftest.h" static void help(void) { diff --git a/tools/testing/selftests/riscv/mm/mmap_bottomup.c b/tools/testing/selftests/riscv/mm/mmap_bottomup.c index f9ccae50349b..461a65c9be00 100644 --- a/tools/testing/selftests/riscv/mm/mmap_bottomup.c +++ b/tools/testing/selftests/riscv/mm/mmap_bottomup.c @@ -2,7 +2,7 @@ #include <sys/mman.h> #include <mmap_test.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" TEST(infinite_rlimit) { diff --git a/tools/testing/selftests/riscv/mm/mmap_default.c b/tools/testing/selftests/riscv/mm/mmap_default.c index 3f53b6ecc326..58db7d172af2 100644 --- a/tools/testing/selftests/riscv/mm/mmap_default.c +++ b/tools/testing/selftests/riscv/mm/mmap_default.c @@ -2,7 +2,7 @@ #include <sys/mman.h> #include <mmap_test.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" TEST(default_rlimit) { diff --git a/tools/testing/selftests/riscv/mm/mmap_test.h b/tools/testing/selftests/riscv/mm/mmap_test.h index 75918d15919f..266a6becdeba 100644 --- a/tools/testing/selftests/riscv/mm/mmap_test.h +++ b/tools/testing/selftests/riscv/mm/mmap_test.h @@ -5,7 +5,7 @@ #include <sys/resource.h> #include <stddef.h> #include <strings.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define TOP_DOWN 0 #define BOTTOM_UP 1 diff --git a/tools/testing/selftests/riscv/sigreturn/sigreturn.c b/tools/testing/selftests/riscv/sigreturn/sigreturn.c index ed351a1cb917..e10873d95fed 100644 --- a/tools/testing/selftests/riscv/sigreturn/sigreturn.c +++ b/tools/testing/selftests/riscv/sigreturn/sigreturn.c @@ -4,7 +4,7 @@ #include <stdlib.h> #include <ucontext.h> #include <linux/ptrace.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #define RISCV_V_MAGIC 0x53465457 #define DEFAULT_VALUE 2 diff --git a/tools/testing/selftests/riscv/vector/v_initval.c b/tools/testing/selftests/riscv/vector/v_initval.c index be9e1d18ad29..5fd2382e15a2 100644 --- a/tools/testing/selftests/riscv/vector/v_initval.c +++ b/tools/testing/selftests/riscv/vector/v_initval.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #include "v_helpers.h" #define NEXT_PROGRAM "./v_exec_initval_nolibc" diff --git a/tools/testing/selftests/riscv/vector/vstate_prctl.c b/tools/testing/selftests/riscv/vector/vstate_prctl.c index 62fbb17a0556..d607af3900c1 100644 --- a/tools/testing/selftests/riscv/vector/vstate_prctl.c +++ b/tools/testing/selftests/riscv/vector/vstate_prctl.c @@ -6,7 +6,7 @@ #include <sys/types.h> #include <stdlib.h> -#include "../../kselftest_harness.h" +#include "kselftest_harness.h" #include "v_helpers.h" #define NEXT_PROGRAM "./vstate_exec_nolibc" diff --git a/tools/testing/selftests/rseq/basic_percpu_ops_test.c b/tools/testing/selftests/rseq/basic_percpu_ops_test.c index 2348d2c20d0a..1193612bf327 100644 --- a/tools/testing/selftests/rseq/basic_percpu_ops_test.c +++ b/tools/testing/selftests/rseq/basic_percpu_ops_test.c @@ -9,7 +9,7 @@ #include <string.h> #include <stddef.h> -#include "../kselftest.h" +#include "kselftest.h" #include "rseq.h" #ifdef BUILDOPT_RSEQ_PERCPU_MM_CID diff --git a/tools/testing/selftests/rseq/rseq.c b/tools/testing/selftests/rseq/rseq.c index dcac5cbe7933..a736727b83c1 100644 --- a/tools/testing/selftests/rseq/rseq.c +++ b/tools/testing/selftests/rseq/rseq.c @@ -33,7 +33,7 @@ #include <linux/compiler.h> -#include "../kselftest.h" +#include "kselftest.h" #include "rseq.h" /* diff --git a/tools/testing/selftests/rtc/rtctest.c b/tools/testing/selftests/rtc/rtctest.c index be175c0e6ae3..8047d9879039 100644 --- a/tools/testing/selftests/rtc/rtctest.c +++ b/tools/testing/selftests/rtc/rtctest.c @@ -16,7 +16,7 @@ #include <time.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define NUM_UIE 3 #define ALARM_DELTA 3 diff --git a/tools/testing/selftests/seccomp/seccomp_benchmark.c b/tools/testing/selftests/seccomp/seccomp_benchmark.c index 5822e25e0217..ea4068cdefd6 100644 --- a/tools/testing/selftests/seccomp/seccomp_benchmark.c +++ b/tools/testing/selftests/seccomp/seccomp_benchmark.c @@ -20,7 +20,7 @@ #include <sys/syscall.h> #include <sys/types.h> -#include "../kselftest.h" +#include "kselftest.h" unsigned long long timing(clockid_t clk_id, unsigned long long samples) { diff --git a/tools/testing/selftests/seccomp/seccomp_bpf.c b/tools/testing/selftests/seccomp/seccomp_bpf.c index 874f17763536..32e2d4df397b 100644 --- a/tools/testing/selftests/seccomp/seccomp_bpf.c +++ b/tools/testing/selftests/seccomp/seccomp_bpf.c @@ -54,7 +54,7 @@ #include <sys/syscall.h> #include <poll.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "../clone3/clone3_selftests.h" /* Attempt to de-conflict with the selftests tree. */ diff --git a/tools/testing/selftests/sgx/main.c b/tools/testing/selftests/sgx/main.c index 9820b3809c69..13b84e54ce38 100644 --- a/tools/testing/selftests/sgx/main.c +++ b/tools/testing/selftests/sgx/main.c @@ -18,7 +18,7 @@ #include <sys/types.h> #include <sys/auxv.h> #include "defines.h" -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "main.h" static const uint64_t MAGIC = 0x1122334455667788ULL; diff --git a/tools/testing/selftests/signal/mangle_uc_sigmask.c b/tools/testing/selftests/signal/mangle_uc_sigmask.c index b79ab92178a8..11dbc14bbc8e 100644 --- a/tools/testing/selftests/signal/mangle_uc_sigmask.c +++ b/tools/testing/selftests/signal/mangle_uc_sigmask.c @@ -39,7 +39,7 @@ #include <signal.h> #include <ucontext.h> -#include "../kselftest.h" +#include "kselftest.h" void handler_verify_ucontext(int signo, siginfo_t *info, void *uc) { diff --git a/tools/testing/selftests/signal/sas.c b/tools/testing/selftests/signal/sas.c index 07227fab1cc9..306b996ab365 100644 --- a/tools/testing/selftests/signal/sas.c +++ b/tools/testing/selftests/signal/sas.c @@ -19,7 +19,7 @@ #include <errno.h> #include <sys/auxv.h> -#include "../kselftest.h" +#include "kselftest.h" #include "current_stack_pointer.h" #ifndef SS_AUTODISARM diff --git a/tools/testing/selftests/sparc64/drivers/adi-test.c b/tools/testing/selftests/sparc64/drivers/adi-test.c index 84e5d9fd20b0..b986714e7a52 100644 --- a/tools/testing/selftests/sparc64/drivers/adi-test.c +++ b/tools/testing/selftests/sparc64/drivers/adi-test.c @@ -16,7 +16,7 @@ #include <sys/stat.h> #include <unistd.h> -#include "../../kselftest.h" +#include "kselftest.h" #define DEBUG_LEVEL_1_BIT (0x0001) #define DEBUG_LEVEL_2_BIT (0x0002) diff --git a/tools/testing/selftests/sync/sync_test.c b/tools/testing/selftests/sync/sync_test.c index 93db5aa246a3..2b44e5d88b63 100644 --- a/tools/testing/selftests/sync/sync_test.c +++ b/tools/testing/selftests/sync/sync_test.c @@ -34,7 +34,7 @@ #include <errno.h> #include <string.h> -#include "../kselftest.h" +#include "kselftest.h" #include "synctest.h" static int run_test(int (*test)(void), char *name) diff --git a/tools/testing/selftests/syscall_user_dispatch/sud_test.c b/tools/testing/selftests/syscall_user_dispatch/sud_test.c index 2eb2c06303f2..b855c6000287 100644 --- a/tools/testing/selftests/syscall_user_dispatch/sud_test.c +++ b/tools/testing/selftests/syscall_user_dispatch/sud_test.c @@ -14,7 +14,7 @@ #include <stdlib.h> #include <asm/unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #ifndef PR_SET_SYSCALL_USER_DISPATCH # define PR_SET_SYSCALL_USER_DISPATCH 59 diff --git a/tools/testing/selftests/tdx/tdx_guest_test.c b/tools/testing/selftests/tdx/tdx_guest_test.c index 81d8cb88ea1a..dfaefa685519 100644 --- a/tools/testing/selftests/tdx/tdx_guest_test.c +++ b/tools/testing/selftests/tdx/tdx_guest_test.c @@ -13,7 +13,7 @@ #include <fcntl.h> #include <linux/tdx-guest.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define TDX_GUEST_DEVNAME "/dev/tdx_guest" #define HEX_DUMP_SIZE 8 diff --git a/tools/testing/selftests/timens/timens.h b/tools/testing/selftests/timens/timens.h index d4fc52d47146..7ca4b46ca61d 100644 --- a/tools/testing/selftests/timens/timens.h +++ b/tools/testing/selftests/timens/timens.h @@ -7,7 +7,7 @@ #include <stdlib.h> #include <stdbool.h> -#include "../kselftest.h" +#include "kselftest.h" #ifndef CLONE_NEWTIME # define CLONE_NEWTIME 0x00000080 diff --git a/tools/testing/selftests/timers/adjtick.c b/tools/testing/selftests/timers/adjtick.c index 777d9494b683..5b3ef708d6e9 100644 --- a/tools/testing/selftests/timers/adjtick.c +++ b/tools/testing/selftests/timers/adjtick.c @@ -24,7 +24,7 @@ #include <time.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" #define MILLION 1000000 diff --git a/tools/testing/selftests/timers/alarmtimer-suspend.c b/tools/testing/selftests/timers/alarmtimer-suspend.c index a9ef76ea6051..aa66c805f6a4 100644 --- a/tools/testing/selftests/timers/alarmtimer-suspend.c +++ b/tools/testing/selftests/timers/alarmtimer-suspend.c @@ -30,7 +30,7 @@ #include <pthread.h> #include <include/vdso/time64.h> #include <errno.h> -#include "../kselftest.h" +#include "kselftest.h" #define UNREASONABLE_LAT (NSEC_PER_SEC * 5) /* hopefully we resume in 5 secs */ diff --git a/tools/testing/selftests/timers/change_skew.c b/tools/testing/selftests/timers/change_skew.c index 18e794a46c23..387fda10fcd8 100644 --- a/tools/testing/selftests/timers/change_skew.c +++ b/tools/testing/selftests/timers/change_skew.c @@ -28,7 +28,7 @@ #include <sys/time.h> #include <sys/timex.h> #include <time.h> -#include "../kselftest.h" +#include "kselftest.h" int change_skew_test(int ppm) { diff --git a/tools/testing/selftests/timers/clocksource-switch.c b/tools/testing/selftests/timers/clocksource-switch.c index 83faa4e354e3..db62a764c29e 100644 --- a/tools/testing/selftests/timers/clocksource-switch.c +++ b/tools/testing/selftests/timers/clocksource-switch.c @@ -34,7 +34,7 @@ #include <sys/wait.h> #include <time.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" int get_clocksources(char list[][30]) diff --git a/tools/testing/selftests/timers/freq-step.c b/tools/testing/selftests/timers/freq-step.c index 73b636f89fdc..cfa46dafe3e8 100644 --- a/tools/testing/selftests/timers/freq-step.c +++ b/tools/testing/selftests/timers/freq-step.c @@ -15,7 +15,7 @@ #include <time.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" #define SAMPLES 100 #define SAMPLE_READINGS 10 diff --git a/tools/testing/selftests/timers/inconsistency-check.c b/tools/testing/selftests/timers/inconsistency-check.c index 9d1573769d55..e53e63e18683 100644 --- a/tools/testing/selftests/timers/inconsistency-check.c +++ b/tools/testing/selftests/timers/inconsistency-check.c @@ -29,7 +29,7 @@ #include <string.h> #include <signal.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" /* CLOCK_HWSPECIFIC == CLOCK_SGI_CYCLE (Deprecated) */ #define CLOCK_HWSPECIFIC 10 diff --git a/tools/testing/selftests/timers/leap-a-day.c b/tools/testing/selftests/timers/leap-a-day.c index 04004a7c0934..3568cfb3e815 100644 --- a/tools/testing/selftests/timers/leap-a-day.c +++ b/tools/testing/selftests/timers/leap-a-day.c @@ -49,7 +49,7 @@ #include <signal.h> #include <unistd.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" #define CLOCK_TAI 11 diff --git a/tools/testing/selftests/timers/leapcrash.c b/tools/testing/selftests/timers/leapcrash.c index 8fd065eec904..c2d3bccb52f2 100644 --- a/tools/testing/selftests/timers/leapcrash.c +++ b/tools/testing/selftests/timers/leapcrash.c @@ -22,7 +22,7 @@ #include <sys/timex.h> #include <string.h> #include <signal.h> -#include "../kselftest.h" +#include "kselftest.h" /* clear NTP time_status & time_state */ int clear_time_state(void) diff --git a/tools/testing/selftests/timers/mqueue-lat.c b/tools/testing/selftests/timers/mqueue-lat.c index 63de2334a291..c0d9368e4fca 100644 --- a/tools/testing/selftests/timers/mqueue-lat.c +++ b/tools/testing/selftests/timers/mqueue-lat.c @@ -30,7 +30,7 @@ #include <errno.h> #include <mqueue.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" #define TARGET_TIMEOUT 100000000 /* 100ms in nanoseconds */ diff --git a/tools/testing/selftests/timers/nanosleep.c b/tools/testing/selftests/timers/nanosleep.c index 10badae13ebe..a054680b3372 100644 --- a/tools/testing/selftests/timers/nanosleep.c +++ b/tools/testing/selftests/timers/nanosleep.c @@ -28,7 +28,7 @@ #include <string.h> #include <signal.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" /* CLOCK_HWSPECIFIC == CLOCK_SGI_CYCLE (Deprecated) */ #define CLOCK_HWSPECIFIC 10 diff --git a/tools/testing/selftests/timers/nsleep-lat.c b/tools/testing/selftests/timers/nsleep-lat.c index de23dc0c9f97..a7ba1eb1e21b 100644 --- a/tools/testing/selftests/timers/nsleep-lat.c +++ b/tools/testing/selftests/timers/nsleep-lat.c @@ -25,7 +25,7 @@ #include <string.h> #include <signal.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" #define UNRESONABLE_LATENCY 40000000 /* 40ms in nanosecs */ diff --git a/tools/testing/selftests/timers/posix_timers.c b/tools/testing/selftests/timers/posix_timers.c index a563c438ac79..38512623622a 100644 --- a/tools/testing/selftests/timers/posix_timers.c +++ b/tools/testing/selftests/timers/posix_timers.c @@ -20,7 +20,7 @@ #include <pthread.h> #include <stdbool.h> -#include "../kselftest.h" +#include "kselftest.h" #define DELAY 2 diff --git a/tools/testing/selftests/timers/raw_skew.c b/tools/testing/selftests/timers/raw_skew.c index 957f7cd29cb1..a7bae7d80916 100644 --- a/tools/testing/selftests/timers/raw_skew.c +++ b/tools/testing/selftests/timers/raw_skew.c @@ -26,7 +26,7 @@ #include <sys/timex.h> #include <time.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" #define shift_right(x, s) ({ \ __typeof__(x) __x = (x); \ diff --git a/tools/testing/selftests/timers/rtcpie.c b/tools/testing/selftests/timers/rtcpie.c index 7c07edd0d450..4ba42d198b7e 100644 --- a/tools/testing/selftests/timers/rtcpie.c +++ b/tools/testing/selftests/timers/rtcpie.c @@ -18,7 +18,7 @@ #include <stdlib.h> #include <errno.h> -#include "../kselftest.h" +#include "kselftest.h" /* * This expects the new RTC class driver framework, working with diff --git a/tools/testing/selftests/timers/set-2038.c b/tools/testing/selftests/timers/set-2038.c index ed244315e11c..ecc171de4728 100644 --- a/tools/testing/selftests/timers/set-2038.c +++ b/tools/testing/selftests/timers/set-2038.c @@ -28,7 +28,7 @@ #include <time.h> #include <sys/time.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" #define KTIME_MAX ((long long)~((unsigned long long)1 << 63)) #define KTIME_SEC_MAX (KTIME_MAX / NSEC_PER_SEC) diff --git a/tools/testing/selftests/timers/set-tai.c b/tools/testing/selftests/timers/set-tai.c index 5b67462efcd6..e283c04284af 100644 --- a/tools/testing/selftests/timers/set-tai.c +++ b/tools/testing/selftests/timers/set-tai.c @@ -23,7 +23,7 @@ #include <string.h> #include <signal.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" int set_tai(int offset) { diff --git a/tools/testing/selftests/timers/set-timer-lat.c b/tools/testing/selftests/timers/set-timer-lat.c index 9d8437c13929..44d2e3614fa5 100644 --- a/tools/testing/selftests/timers/set-timer-lat.c +++ b/tools/testing/selftests/timers/set-timer-lat.c @@ -29,7 +29,7 @@ #include <stdlib.h> #include <pthread.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" /* CLOCK_HWSPECIFIC == CLOCK_SGI_CYCLE (Deprecated) */ #define CLOCK_HWSPECIFIC 10 diff --git a/tools/testing/selftests/timers/set-tz.c b/tools/testing/selftests/timers/set-tz.c index 20daaf1782b7..334f36c0eda6 100644 --- a/tools/testing/selftests/timers/set-tz.c +++ b/tools/testing/selftests/timers/set-tz.c @@ -23,7 +23,7 @@ #include <string.h> #include <signal.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" int set_tz(int min, int dst) { diff --git a/tools/testing/selftests/timers/skew_consistency.c b/tools/testing/selftests/timers/skew_consistency.c index 46c391d7f45d..53ee5d710ff4 100644 --- a/tools/testing/selftests/timers/skew_consistency.c +++ b/tools/testing/selftests/timers/skew_consistency.c @@ -34,7 +34,7 @@ #include <fcntl.h> #include <string.h> #include <sys/wait.h> -#include "../kselftest.h" +#include "kselftest.h" int main(int argc, char **argv) { diff --git a/tools/testing/selftests/timers/threadtest.c b/tools/testing/selftests/timers/threadtest.c index d5564bbf0e50..60b8b21bf782 100644 --- a/tools/testing/selftests/timers/threadtest.c +++ b/tools/testing/selftests/timers/threadtest.c @@ -21,7 +21,7 @@ #include <stdlib.h> #include <sys/time.h> #include <pthread.h> -#include "../kselftest.h" +#include "kselftest.h" /* serializes shared list access */ pthread_mutex_t list_lock = PTHREAD_MUTEX_INITIALIZER; diff --git a/tools/testing/selftests/timers/valid-adjtimex.c b/tools/testing/selftests/timers/valid-adjtimex.c index 6b7801055ad1..e1e56d3097d6 100644 --- a/tools/testing/selftests/timers/valid-adjtimex.c +++ b/tools/testing/selftests/timers/valid-adjtimex.c @@ -30,7 +30,7 @@ #include <signal.h> #include <unistd.h> #include <include/vdso/time64.h> -#include "../kselftest.h" +#include "kselftest.h" #define ADJ_SETOFFSET 0x0100 diff --git a/tools/testing/selftests/tmpfs/bug-link-o-tmpfile.c b/tools/testing/selftests/tmpfs/bug-link-o-tmpfile.c index 02ecfe687dc2..5cb4e404a2bd 100644 --- a/tools/testing/selftests/tmpfs/bug-link-o-tmpfile.c +++ b/tools/testing/selftests/tmpfs/bug-link-o-tmpfile.c @@ -23,7 +23,7 @@ #include <sys/mount.h> #include <unistd.h> -#include "../kselftest.h" +#include "kselftest.h" int main(void) { diff --git a/tools/testing/selftests/tty/tty_tstamp_update.c b/tools/testing/selftests/tty/tty_tstamp_update.c index 9e1a40f5db17..bc3291dcd18b 100644 --- a/tools/testing/selftests/tty/tty_tstamp_update.c +++ b/tools/testing/selftests/tty/tty_tstamp_update.c @@ -9,7 +9,7 @@ #include <unistd.h> #include <linux/limits.h> -#include "../kselftest.h" +#include "kselftest.h" #define MIN_TTY_PATH_LEN 8 diff --git a/tools/testing/selftests/uevent/uevent_filtering.c b/tools/testing/selftests/uevent/uevent_filtering.c index dbe55f3a66f4..974b076f9235 100644 --- a/tools/testing/selftests/uevent/uevent_filtering.c +++ b/tools/testing/selftests/uevent/uevent_filtering.c @@ -19,7 +19,7 @@ #include <sys/wait.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #define __DEV_FULL "/sys/devices/virtual/mem/full/uevent" #define __UEVENT_BUFFER_SIZE (2048 * 2) diff --git a/tools/testing/selftests/user_events/abi_test.c b/tools/testing/selftests/user_events/abi_test.c index 7288a05136ba..85892b3b719c 100644 --- a/tools/testing/selftests/user_events/abi_test.c +++ b/tools/testing/selftests/user_events/abi_test.c @@ -20,7 +20,7 @@ #include <string.h> #include <asm/unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "user_events_selftests.h" const char *data_file = "/sys/kernel/tracing/user_events_data"; diff --git a/tools/testing/selftests/user_events/dyn_test.c b/tools/testing/selftests/user_events/dyn_test.c index 54c9412f8dee..78e3c33f4015 100644 --- a/tools/testing/selftests/user_events/dyn_test.c +++ b/tools/testing/selftests/user_events/dyn_test.c @@ -14,7 +14,7 @@ #include <sys/stat.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "user_events_selftests.h" const char *dyn_file = "/sys/kernel/tracing/dynamic_events"; diff --git a/tools/testing/selftests/user_events/ftrace_test.c b/tools/testing/selftests/user_events/ftrace_test.c index 0bb46793dcd4..decce06b9ba8 100644 --- a/tools/testing/selftests/user_events/ftrace_test.c +++ b/tools/testing/selftests/user_events/ftrace_test.c @@ -15,7 +15,7 @@ #include <sys/uio.h> #include <unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "user_events_selftests.h" const char *data_file = "/sys/kernel/tracing/user_events_data"; diff --git a/tools/testing/selftests/user_events/perf_test.c b/tools/testing/selftests/user_events/perf_test.c index 68625362add2..cafec0e52eb3 100644 --- a/tools/testing/selftests/user_events/perf_test.c +++ b/tools/testing/selftests/user_events/perf_test.c @@ -16,7 +16,7 @@ #include <unistd.h> #include <asm/unistd.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" #include "user_events_selftests.h" const char *data_file = "/sys/kernel/tracing/user_events_data"; diff --git a/tools/testing/selftests/user_events/user_events_selftests.h b/tools/testing/selftests/user_events/user_events_selftests.h index e1c3c063c031..3b5d37e46f8a 100644 --- a/tools/testing/selftests/user_events/user_events_selftests.h +++ b/tools/testing/selftests/user_events/user_events_selftests.h @@ -9,7 +9,7 @@ #include <unistd.h> #include <errno.h> -#include "../kselftest.h" +#include "kselftest.h" static inline void tracefs_unmount(void) { diff --git a/tools/testing/selftests/vDSO/vdso_test_abi.c b/tools/testing/selftests/vDSO/vdso_test_abi.c index 238d609a457a..c620317eaeea 100644 --- a/tools/testing/selftests/vDSO/vdso_test_abi.c +++ b/tools/testing/selftests/vDSO/vdso_test_abi.c @@ -18,7 +18,7 @@ #include <unistd.h> #include <sys/syscall.h> -#include "../kselftest.h" +#include "kselftest.h" #include "vdso_config.h" #include "vdso_call.h" #include "parse_vdso.h" diff --git a/tools/testing/selftests/vDSO/vdso_test_chacha.c b/tools/testing/selftests/vDSO/vdso_test_chacha.c index 0aad682b12c8..9a5c9c05e09c 100644 --- a/tools/testing/selftests/vDSO/vdso_test_chacha.c +++ b/tools/testing/selftests/vDSO/vdso_test_chacha.c @@ -10,7 +10,7 @@ #include <string.h> #include <stdint.h> #include <stdbool.h> -#include "../kselftest.h" +#include "kselftest.h" #if defined(__aarch64__) static bool cpu_has_capabilities(void) diff --git a/tools/testing/selftests/vDSO/vdso_test_correctness.c b/tools/testing/selftests/vDSO/vdso_test_correctness.c index da651cf53c6c..055af95aa552 100644 --- a/tools/testing/selftests/vDSO/vdso_test_correctness.c +++ b/tools/testing/selftests/vDSO/vdso_test_correctness.c @@ -21,7 +21,7 @@ #include "vdso_config.h" #include "vdso_call.h" -#include "../kselftest.h" +#include "kselftest.h" static const char **name; diff --git a/tools/testing/selftests/vDSO/vdso_test_getcpu.c b/tools/testing/selftests/vDSO/vdso_test_getcpu.c index cdeaed45fb26..bea8ad54da11 100644 --- a/tools/testing/selftests/vDSO/vdso_test_getcpu.c +++ b/tools/testing/selftests/vDSO/vdso_test_getcpu.c @@ -11,7 +11,7 @@ #include <sys/auxv.h> #include <sys/time.h> -#include "../kselftest.h" +#include "kselftest.h" #include "parse_vdso.h" #include "vdso_config.h" #include "vdso_call.h" diff --git a/tools/testing/selftests/vDSO/vdso_test_getrandom.c b/tools/testing/selftests/vDSO/vdso_test_getrandom.c index dd1132508a0d..ef402001e898 100644 --- a/tools/testing/selftests/vDSO/vdso_test_getrandom.c +++ b/tools/testing/selftests/vDSO/vdso_test_getrandom.c @@ -23,7 +23,7 @@ #include <linux/random.h> #include <linux/ptrace.h> -#include "../kselftest.h" +#include "kselftest.h" #include "parse_vdso.h" #include "vdso_config.h" #include "vdso_call.h" diff --git a/tools/testing/selftests/vDSO/vdso_test_gettimeofday.c b/tools/testing/selftests/vDSO/vdso_test_gettimeofday.c index 9ce795b806f0..912edadad92c 100644 --- a/tools/testing/selftests/vDSO/vdso_test_gettimeofday.c +++ b/tools/testing/selftests/vDSO/vdso_test_gettimeofday.c @@ -16,7 +16,7 @@ #include <sys/time.h> #endif -#include "../kselftest.h" +#include "kselftest.h" #include "parse_vdso.h" #include "vdso_config.h" #include "vdso_call.h" diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_device.c b/tools/testing/selftests/vfio/lib/vfio_pci_device.c index 13fdb4b0b10f..8e34b9bfc96b 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_device.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_device.c @@ -19,7 +19,7 @@ #include <linux/types.h> #include <linux/vfio.h> -#include "../../../kselftest.h" +#include "kselftest.h" #include <libvfio.h> #define PCI_SYSFS_PATH "/sys/bus/pci/devices" diff --git a/tools/testing/selftests/vfio/lib/vfio_pci_driver.c b/tools/testing/selftests/vfio/lib/vfio_pci_driver.c index ca0e25efbfa1..6827f4a6febe 100644 --- a/tools/testing/selftests/vfio/lib/vfio_pci_driver.c +++ b/tools/testing/selftests/vfio/lib/vfio_pci_driver.c @@ -1,5 +1,5 @@ // SPDX-License-Identifier: GPL-2.0-only -#include "../../../kselftest.h" +#include "kselftest.h" #include <libvfio.h> #ifdef __x86_64__ diff --git a/tools/testing/selftests/vfio/vfio_dma_mapping_test.c b/tools/testing/selftests/vfio/vfio_dma_mapping_test.c index 5397822c3dd4..16eba2ecca47 100644 --- a/tools/testing/selftests/vfio/vfio_dma_mapping_test.c +++ b/tools/testing/selftests/vfio/vfio_dma_mapping_test.c @@ -12,7 +12,7 @@ #include <libvfio.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static const char *device_bdf; diff --git a/tools/testing/selftests/vfio/vfio_iommufd_setup_test.c b/tools/testing/selftests/vfio/vfio_iommufd_setup_test.c index caf1c6291f3d..17017ed3beac 100644 --- a/tools/testing/selftests/vfio/vfio_iommufd_setup_test.c +++ b/tools/testing/selftests/vfio/vfio_iommufd_setup_test.c @@ -11,7 +11,7 @@ #include <unistd.h> #include <libvfio.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static const char iommu_dev_path[] = "/dev/iommu"; static const char *cdev_path; diff --git a/tools/testing/selftests/vfio/vfio_pci_device_test.c b/tools/testing/selftests/vfio/vfio_pci_device_test.c index ecbb669b3765..7c0fe8ce3a61 100644 --- a/tools/testing/selftests/vfio/vfio_pci_device_test.c +++ b/tools/testing/selftests/vfio/vfio_pci_device_test.c @@ -12,7 +12,7 @@ #include <libvfio.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static const char *device_bdf; diff --git a/tools/testing/selftests/vfio/vfio_pci_driver_test.c b/tools/testing/selftests/vfio/vfio_pci_driver_test.c index f0ca8310d6a8..afa0480ddd9b 100644 --- a/tools/testing/selftests/vfio/vfio_pci_driver_test.c +++ b/tools/testing/selftests/vfio/vfio_pci_driver_test.c @@ -7,7 +7,7 @@ #include <libvfio.h> -#include "../kselftest_harness.h" +#include "kselftest_harness.h" static const char *device_bdf; diff --git a/tools/testing/selftests/wireguard/qemu/kernel.config b/tools/testing/selftests/wireguard/qemu/kernel.config index 936b18be07cf..0504c11c2de6 100644 --- a/tools/testing/selftests/wireguard/qemu/kernel.config +++ b/tools/testing/selftests/wireguard/qemu/kernel.config @@ -81,7 +81,7 @@ CONFIG_WQ_WATCHDOG=y CONFIG_DETECT_HUNG_TASK=y CONFIG_BOOTPARAM_HARDLOCKUP_PANIC=y CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC=y -CONFIG_BOOTPARAM_HUNG_TASK_PANIC=y +CONFIG_BOOTPARAM_HUNG_TASK_PANIC=1 CONFIG_PANIC_TIMEOUT=-1 CONFIG_STACKTRACE=y CONFIG_EARLY_PRINTK=y diff --git a/tools/testing/selftests/x86/corrupt_xstate_header.c b/tools/testing/selftests/x86/corrupt_xstate_header.c index 93a89a5997ca..f4d67b050275 100644 --- a/tools/testing/selftests/x86/corrupt_xstate_header.c +++ b/tools/testing/selftests/x86/corrupt_xstate_header.c @@ -17,7 +17,7 @@ #include <stdint.h> #include <sys/wait.h> -#include "../kselftest.h" /* For __cpuid_count() */ +#include "kselftest.h" /* For __cpuid_count() */ #include "helpers.h" static inline int xsave_enabled(void) diff --git a/tools/testing/selftests/x86/helpers.h b/tools/testing/selftests/x86/helpers.h index 6deaad035161..4c747a1278d9 100644 --- a/tools/testing/selftests/x86/helpers.h +++ b/tools/testing/selftests/x86/helpers.h @@ -7,7 +7,7 @@ #include <asm/processor-flags.h> -#include "../kselftest.h" +#include "kselftest.h" static inline unsigned long get_eflags(void) { diff --git a/tools/testing/selftests/x86/lam.c b/tools/testing/selftests/x86/lam.c index 0873b0e5f48b..1919fa6daec0 100644 --- a/tools/testing/selftests/x86/lam.c +++ b/tools/testing/selftests/x86/lam.c @@ -18,7 +18,7 @@ #include <sys/uio.h> #include <linux/io_uring.h> -#include "../kselftest.h" +#include "kselftest.h" #ifndef __x86_64__ # error This test is 64-bit only diff --git a/tools/testing/selftests/x86/syscall_numbering.c b/tools/testing/selftests/x86/syscall_numbering.c index 41c42b7b54a6..ca0eca7b9dce 100644 --- a/tools/testing/selftests/x86/syscall_numbering.c +++ b/tools/testing/selftests/x86/syscall_numbering.c @@ -25,7 +25,7 @@ #include <sys/mman.h> #include <linux/ptrace.h> -#include "../kselftest.h" +#include "kselftest.h" /* Common system call numbers */ #define SYS_READ 0 diff --git a/tools/testing/selftests/x86/test_mremap_vdso.c b/tools/testing/selftests/x86/test_mremap_vdso.c index 94bee6e0c813..a5edf6c5f17e 100644 --- a/tools/testing/selftests/x86/test_mremap_vdso.c +++ b/tools/testing/selftests/x86/test_mremap_vdso.c @@ -20,7 +20,7 @@ #include <sys/auxv.h> #include <sys/syscall.h> #include <sys/wait.h> -#include "../kselftest.h" +#include "kselftest.h" #define PAGE_SIZE 4096 diff --git a/tools/testing/selftests/x86/test_vsyscall.c b/tools/testing/selftests/x86/test_vsyscall.c index 918eaec8bfbe..f1c3df642352 100644 --- a/tools/testing/selftests/x86/test_vsyscall.c +++ b/tools/testing/selftests/x86/test_vsyscall.c @@ -21,7 +21,7 @@ #include <sys/uio.h> #include "helpers.h" -#include "../kselftest.h" +#include "kselftest.h" #ifdef __x86_64__ #define TOTAL_TESTS 13 diff --git a/tools/testing/selftests/x86/xstate.h b/tools/testing/selftests/x86/xstate.h index e91e3092b5d2..6ee816e7625a 100644 --- a/tools/testing/selftests/x86/xstate.h +++ b/tools/testing/selftests/x86/xstate.h @@ -4,7 +4,7 @@ #include <stdint.h> -#include "../kselftest.h" +#include "kselftest.h" #define XSAVE_HDR_OFFSET 512 #define XSAVE_HDR_SIZE 64 |