Archlinux Kernel based on different schedulers and some other performance improvements.
Shell archlinuxbaby-schedcachycachy-schedulercachyoscacule-schedkernellinux-kernelperformanceperformance-tuning
5.17-rc3-1-patch
P. Junge4ce9f5ad9fd
diff --git a/patches/linux-cachyos-rc-5.17.rc3.patch b/patches/linux-cachyos-rc-5.17.rc3.patchnew file mode 100644index 00000000..8c6be46e--- /dev/null+++ b/patches/linux-cachyos-rc-5.17.rc3.patch@@ -0,0 +1,11842 @@+From 1acd39e254766a13905a4758a1ff1da74b289e9d Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sun, 23 Jan 2022 16:49:01 +0000+Subject: [PATCH] anbox++---+ drivers/android/Kconfig | 2 +-+ drivers/android/Makefile | 7 ++++---+ drivers/android/binder_alloc.c | 2 +-+ drivers/staging/android/Kconfig | 2 +-+ drivers/staging/android/Makefile | 3 ++-+ drivers/staging/android/ashmem.c | 3 ++++ fs/file.c | 1 ++ kernel/fork.c | 1 ++ kernel/sched/core.c | 1 ++ kernel/task_work.c | 1 ++ mm/memory.c | 1 ++ mm/shmem.c | 1 ++ security/security.c | 4 +++++ 13 files changed, 22 insertions(+), 7 deletions(-)++diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig+index 53b22e26266c..f3c50236e8d1 100644+--- a/drivers/android/Kconfig++++ b/drivers/android/Kconfig+@@ -9,7 +9,7 @@ config ANDROID+ if ANDROID++ config ANDROID_BINDER_IPC+- bool "Android Binder IPC Driver"++ tristate "Android Binder IPC Driver"+ depends on MMU+ default n+ help+diff --git a/drivers/android/Makefile b/drivers/android/Makefile+index c9d3d0c99c25..55411d9a9c2a 100644+--- a/drivers/android/Makefile++++ b/drivers/android/Makefile+@@ -1,6 +1,7 @@+ # SPDX-License-Identifier: GPL-2.0-only+ ccflags-y += -I$(src) # needed for trace events++-obj-$(CONFIG_ANDROID_BINDERFS) += binderfs.o+-obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o binder_alloc.o+-obj-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o++obj-$(CONFIG_ANDROID_BINDER_IPC) += binder_linux.o++binder_linux-y := binder.o binder_alloc.o++binder_linux-$(CONFIG_ANDROID_BINDERFS) += binderfs.o++binder_linux-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o+diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c+index 47bc74a8c7b6..49ff7504646e 100644+--- a/drivers/android/binder_alloc.c++++ b/drivers/android/binder_alloc.c+@@ -38,7 +38,7 @@ enum {+ };+ static uint32_t binder_alloc_debug_mask = BINDER_DEBUG_USER_ERROR;++-module_param_named(debug_mask, binder_alloc_debug_mask,++module_param_named(alloc_debug_mask, binder_alloc_debug_mask,+ uint, 0644);++ #define binder_alloc_debug(mask, x...) \+diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig+index 70498adb1575..5c35653ed36d 100644+--- a/drivers/staging/android/Kconfig++++ b/drivers/staging/android/Kconfig+@@ -4,7 +4,7 @@ menu "Android"+ if ANDROID++ config ASHMEM+- bool "Enable the Anonymous Shared Memory Subsystem"++ tristate "Enable the Anonymous Shared Memory Subsystem"+ depends on SHMEM+ help+ The ashmem subsystem is a new shared memory allocator, similar to+diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile+index e9a55a5e6529..60cb8eacc793 100644+--- a/drivers/staging/android/Makefile++++ b/drivers/staging/android/Makefile+@@ -1,4 +1,5 @@+ # SPDX-License-Identifier: GPL-2.0+ ccflags-y += -I$(src) # needed for trace events++-obj-$(CONFIG_ASHMEM) += ashmem.o++obj-$(CONFIG_ASHMEM) += ashmem_linux.o++ashmem_linux-y += ashmem.o+diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c+index ddbde3f8430e..2a92b0a7482f 100644+--- a/drivers/staging/android/ashmem.c++++ b/drivers/staging/android/ashmem.c+@@ -24,6 +24,7 @@+ #include <linux/bitops.h>+ #include <linux/mutex.h>+ #include <linux/shmem_fs.h>++#include <linux/module.h>+ #include "ashmem.h"++ #define ASHMEM_NAME_PREFIX "dev/ashmem/"+@@ -968,3 +969,5 @@ static int __init ashmem_init(void)+ return ret;+ }+ device_initcall(ashmem_init);++++MODULE_LICENSE("GPL v2");+diff --git a/fs/file.c b/fs/file.c+index 97d212a9b814..b8f81bfa874e 100644+--- a/fs/file.c++++ b/fs/file.c+@@ -787,6 +787,7 @@ int __close_fd_get_file(unsigned int fd, struct file **res)+ *res = NULL;+ return -ENOENT;+ }++EXPORT_SYMBOL(close_fd_get_file);++ /*+ * variant of close_fd that gets a ref on the file for later fput.+diff --git a/kernel/fork.c b/kernel/fork.c+index d75a528f7b21..fe337e1b7c4b 100644+--- a/kernel/fork.c++++ b/kernel/fork.c+@@ -1152,6 +1152,7 @@ void mmput_async(struct mm_struct *mm)+ schedule_work(&mm->async_put_work);+ }+ }++EXPORT_SYMBOL_GPL(mmput_async);+ #endif++ /**+diff --git a/kernel/sched/core.c b/kernel/sched/core.c+index 2e4ae00e52d1..d0fe040e8b97 100644+--- a/kernel/sched/core.c++++ b/kernel/sched/core.c+@@ -6955,6 +6955,7 @@ int can_nice(const struct task_struct *p, const int nice)+ return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||+ capable(CAP_SYS_NICE));+ }++EXPORT_SYMBOL_GPL(can_nice);++ #ifdef __ARCH_WANT_SYS_NICE++diff --git a/kernel/task_work.c b/kernel/task_work.c+index 1698fbe6f0e1..90000404af2b 100644+--- a/kernel/task_work.c++++ b/kernel/task_work.c+@@ -60,6 +60,7 @@ int task_work_add(struct task_struct *task, struct callback_head *work,++ return 0;+ }++EXPORT_SYMBOL(task_work_add);++ /**+ * task_work_cancel_match - cancel a pending work added by task_work_add()+diff --git a/mm/memory.c b/mm/memory.c+index c125c4969913..fe59e377b671 100644+--- a/mm/memory.c++++ b/mm/memory.c+@@ -1664,6 +1664,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,+ mmu_notifier_invalidate_range_end(&range);+ tlb_finish_mmu(&tlb);+ }++EXPORT_SYMBOL_GPL(zap_page_range);++ /**+ * zap_page_range_single - remove user pages in a given range+diff --git a/mm/shmem.c b/mm/shmem.c+index a09b29ec2b45..98f73f109f19 100644+--- a/mm/shmem.c++++ b/mm/shmem.c+@@ -4140,6 +4140,7 @@ int shmem_zero_setup(struct vm_area_struct *vma)++ return 0;+ }++EXPORT_SYMBOL_GPL(shmem_zero_setup);++ /**+ * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.+diff --git a/security/security.c b/security/security.c+index 3d4eb474f35b..429ecc13a8aa 100644+--- a/security/security.c++++ b/security/security.c+@@ -751,24 +751,28 @@ int security_binder_set_context_mgr(const struct cred *mgr)+ {+ return call_int_hook(binder_set_context_mgr, 0, mgr);+ }++EXPORT_SYMBOL_GPL(security_binder_set_context_mgr);++ int security_binder_transaction(const struct cred *from,+ const struct cred *to)+ {+ return call_int_hook(binder_transaction, 0, from, to);+ }++EXPORT_SYMBOL_GPL(security_binder_transaction);++ int security_binder_transfer_binder(const struct cred *from,+ const struct cred *to)+ {+ return call_int_hook(binder_transfer_binder, 0, from, to);+ }++EXPORT_SYMBOL_GPL(security_binder_transfer_binder);++ int security_binder_transfer_file(const struct cred *from,+ const struct cred *to, struct file *file)+ {+ return call_int_hook(binder_transfer_file, 0, from, to, file);+ }++EXPORT_SYMBOL_GPL(security_binder_transfer_file);++ int security_ptrace_access_check(struct task_struct *child, unsigned int mode)+ {+--+2.34.1++From 2d64fc35bf31bba408508cec7e3ccd45c20c7bec Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sat, 29 Jan 2022 07:47:17 +0000+Subject: [PATCH] asus-ec++---+ Documentation/hwmon/asus_ec_sensors.rst | 52 +++ Documentation/hwmon/asus_wmi_ec_sensors.rst | 38 --+ MAINTAINERS | 6 ++ drivers/hwmon/Kconfig | 16 +-+ drivers/hwmon/Makefile | 1 ++ drivers/hwmon/asus-ec-sensors.c | 694 +++++++++++++++++++++ 6 files changed, 768 insertions(+), 39 deletions(-)+ create mode 100644 Documentation/hwmon/asus_ec_sensors.rst+ delete mode 100644 Documentation/hwmon/asus_wmi_ec_sensors.rst+ create mode 100644 drivers/hwmon/asus-ec-sensors.c++diff --git a/Documentation/hwmon/asus_ec_sensors.rst b/Documentation/hwmon/asus_ec_sensors.rst+new file mode 100644+index 000000000000..b12ac7ebeb1a+--- /dev/null++++ b/Documentation/hwmon/asus_ec_sensors.rst+@@ -0,0 +1,52 @@++.. SPDX-License-Identifier: GPL-2.0-or-later++++Kernel driver asus_ec_sensors++=================================++++Supported boards:++ * PRIME X570-PRO,++ * Pro WS X570-ACE,++ * ROG CROSSHAIR VIII DARK HERO,++ * ROG CROSSHAIR VIII FORMULA,++ * ROG CROSSHAIR VIII HERO,++ * ROG CROSSHAIR VIII IMPACT,++ * ROG STRIX B550-E GAMING,++ * ROG STRIX B550-I GAMING,++ * ROG STRIX X570-E GAMING,++ * ROG STRIX X570-F GAMING,++ * ROG STRIX X570-I GAMING++++Authors:++ - Eugene Shalygin <[email protected]>++++Description:++------------++ASUS mainboards publish hardware monitoring information via Super I/O++chip and the ACPI embedded controller (EC) registers. Some of the sensors++are only available via the EC.++++The driver is aware of and reads the following sensors:++++1. Chipset (PCH) temperature++2. CPU package temperature++3. Motherboard temperature++4. Readings from the T_Sensor header++5. VRM temperature++6. CPU_Opt fan RPM++7. VRM heatsink fan RPM++8. Chipset fan RPM++9. Readings from the "Water flow meter" header (RPM)++10. Readings from the "Water In" and "Water Out" temperature headers++11. CPU current++++Sensor values are read from EC registers, and to avoid race with the board++firmware the driver acquires ACPI mutex, the one used by the WMI when its++methods access the EC.++++Module Parameters++-----------------++ * mutex_path: string++ The driver holds path to the ACPI mutex for each board (actually,++ the path is mostly identical for them). If ASUS changes this path++ in a future BIOS update, this parameter can be used to override++ the stored in the driver value until it gets updated.+diff --git a/Documentation/hwmon/asus_wmi_ec_sensors.rst b/Documentation/hwmon/asus_wmi_ec_sensors.rst+deleted file mode 100644+index 1b287f229e86..000000000000+--- a/Documentation/hwmon/asus_wmi_ec_sensors.rst++++ /dev/null+@@ -1,38 +0,0 @@+-.. SPDX-License-Identifier: GPL-2.0-or-later+-+-Kernel driver asus_wmi_ec_sensors+-=================================+-+-Supported boards:+- * PRIME X570-PRO,+- * Pro WS X570-ACE,+- * ROG CROSSHAIR VIII DARK HERO,+- * ROG CROSSHAIR VIII FORMULA,+- * ROG CROSSHAIR VIII HERO,+- * ROG STRIX B550-E GAMING,+- * ROG STRIX B550-I GAMING,+- * ROG STRIX X570-E GAMING.+-+-Authors:+- - Eugene Shalygin <[email protected]>+-+-Description:+-------------+-ASUS mainboards publish hardware monitoring information via Super I/O+-chip and the ACPI embedded controller (EC) registers. Some of the sensors+-are only available via the EC.+-+-ASUS WMI interface provides a method (BREC) to read data from EC registers,+-which is utilized by this driver to publish those sensor readings to the+-HWMON system. The driver is aware of and reads the following sensors:+-+-1. Chipset (PCH) temperature+-2. CPU package temperature+-3. Motherboard temperature+-4. Readings from the T_Sensor header+-5. VRM temperature+-6. CPU_Opt fan RPM+-7. Chipset fan RPM+-8. Readings from the "Water flow meter" header (RPM)+-9. Readings from the "Water In" and "Water Out" temperature headers+-10. CPU current+diff --git a/MAINTAINERS b/MAINTAINERS+index ea3e6c914384..aff67314e63e 100644+--- a/MAINTAINERS++++ b/MAINTAINERS+@@ -3049,6 +3049,12 @@ L: [email protected]+ S: Maintained+ F: drivers/hwmon/asus_wmi_ec_sensors.c+++ASUS EC HARDWARE MONITOR DRIVER++M: Eugene Shalygin <[email protected]>++L: [email protected]++S: Maintained++F: drivers/hwmon/asus-ec-sensors.c+++ ASUS WIRELESS RADIO CONTROL DRIVER+ M: João Paulo Rechi Vita <[email protected]>+ L: [email protected]+diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig+index 8df25f1079ba..2acfcbd114c2 100644+--- a/drivers/hwmon/Kconfig++++ b/drivers/hwmon/Kconfig+@@ -2252,16 +2252,30 @@ config SENSORS_ASUS_WMI++ config SENSORS_ASUS_WMI_EC+ tristate "ASUS WMI B550/X570"+- depends on ACPI_WMI++ depends on ACPI_WMI && SENSORS_ASUS_EC=n+ help+ If you say yes here you get support for the ACPI embedded controller+ hardware monitoring interface found in B550/X570 ASUS motherboards.+ This driver will provide readings of fans, voltages and temperatures+ through the system firmware.+++ This driver is deprecated in favor of the ASUS EC Sensors driver++ which provides fully compatible output.+++ This driver can also be built as a module. If so, the module+ will be called asus_wmi_sensors_ec.+++config SENSORS_ASUS_EC++ tristate "ASUS EC Sensors"++ help++ If you say yes here you get support for the ACPI embedded controller++ hardware monitoring interface found in ASUS motherboards. The driver++ currently supports B550/X570 boards, although other ASUS boards might++ provide this monitoring interface as well.++++ This driver can also be built as a module. If so, the module++ will be called asus_ec_sensors.+++ endif # ACPI++ endif # HWMON+diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile+index 185f946d698b..7bba3415ca3d 100644+--- a/drivers/hwmon/Makefile++++ b/drivers/hwmon/Makefile+@@ -9,6 +9,7 @@ obj-$(CONFIG_HWMON_VID) += hwmon-vid.o+ # APCI drivers+ obj-$(CONFIG_SENSORS_ACPI_POWER) += acpi_power_meter.o+ obj-$(CONFIG_SENSORS_ATK0110) += asus_atk0110.o++obj-$(CONFIG_SENSORS_ASUS_EC) += asus-ec-sensors.o+ obj-$(CONFIG_SENSORS_ASUS_WMI) += asus_wmi_sensors.o+ obj-$(CONFIG_SENSORS_ASUS_WMI_EC) += asus_wmi_ec_sensors.o++diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c+new file mode 100644+index 000000000000..906597d89842+--- /dev/null++++ b/drivers/hwmon/asus-ec-sensors.c+@@ -0,0 +1,694 @@++// SPDX-License-Identifier: GPL-2.0+++/*++ * HWMON driver for ASUS motherboards that publish some sensor values++ * via the embedded controller registers.++ *++ * Copyright (C) 2021 Eugene Shalygin <[email protected]>++++ * EC provides:++ * - Chipset temperature++ * - CPU temperature++ * - Motherboard temperature++ * - T_Sensor temperature++ * - VRM temperature++ * - Water In temperature++ * - Water Out temperature++ * - CPU Optional fan RPM++ * - Chipset fan RPM++ * - VRM Heat Sink fan RPM++ * - Water Flow fan RPM++ * - CPU current++ */++++#include <linux/acpi.h>++#include <linux/bitops.h>++#include <linux/dev_printk.h>++#include <linux/dmi.h>++#include <linux/hwmon.h>++#include <linux/init.h>++#include <linux/jiffies.h>++#include <linux/kernel.h>++#include <linux/module.h>++#include <linux/platform_device.h>++#include <linux/sort.h>++#include <linux/units.h>++++#include <asm/unaligned.h>++++static char *mutex_path_override;++++/* Writing to this EC register switches EC bank */++#define ASUS_EC_BANK_REGISTER 0xff++#define SENSOR_LABEL_LEN 16++++/*++ * Arbitrary set max. allowed bank number. Required for sorting banks and++ * currently is overkill with just 2 banks used at max, but for the sake++ * of alignment let's set it to a higher value.++ */++#define ASUS_EC_MAX_BANK 3++++#define ACPI_LOCK_DELAY_MS 500++++/* ACPI mutex for locking access to the EC for the firmware */++#define ASUS_HW_ACCESS_MUTEX_ASMX "\\AMW0.ASMX"++++/* There are two variants of the vendor spelling */++#define VENDOR_ASUS_UPPER_CASE "ASUSTeK COMPUTER INC."++++typedef union {++ u32 value;++ struct {++ u8 index;++ u8 bank;++ u8 size;++ u8 dummy;++ } components;++} sensor_address;++++#define MAKE_SENSOR_ADDRESS(size, bank, index) { \++ .value = (size << 16) + (bank << 8) + index \++ }++++static u32 hwmon_attributes[hwmon_max] = {++ [hwmon_chip] = HWMON_C_REGISTER_TZ,++ [hwmon_temp] = HWMON_T_INPUT | HWMON_T_LABEL,++ [hwmon_in] = HWMON_I_INPUT | HWMON_I_LABEL,++ [hwmon_curr] = HWMON_C_INPUT | HWMON_C_LABEL,++ [hwmon_fan] = HWMON_F_INPUT | HWMON_F_LABEL,++};++++struct ec_sensor_info {++ char label[SENSOR_LABEL_LEN];++ enum hwmon_sensor_types type;++ sensor_address addr;++};++++#define EC_SENSOR(sensor_label, sensor_type, size, bank, index) { \++ .label = sensor_label, .type = sensor_type, \++ .addr = MAKE_SENSOR_ADDRESS(size, bank, index), \++ }++++enum ec_sensors {++ /* chipset temperature [℃] */++ ec_sensor_temp_chipset,++ /* CPU temperature [℃] */++ ec_sensor_temp_cpu,++ /* motherboard temperature [℃] */++ ec_sensor_temp_mb,++ /* "T_Sensor" temperature sensor reading [℃] */++ ec_sensor_temp_t_sensor,++ /* VRM temperature [℃] */++ ec_sensor_temp_vrm,++ /* CPU_Opt fan [RPM] */++ ec_sensor_fan_cpu_opt,++ /* VRM heat sink fan [RPM] */++ ec_sensor_fan_vrm_hs,++ /* Chipset fan [RPM] */++ ec_sensor_fan_chipset,++ /* Water flow sensor reading [RPM] */++ ec_sensor_fan_water_flow,++ /* CPU current [A] */++ ec_sensor_curr_cpu,++ /* "Water_In" temperature sensor reading [℃] */++ ec_sensor_temp_water_in,++ /* "Water_Out" temperature sensor reading [℃] */++ ec_sensor_temp_water_out,++};++++#define SENSOR_TEMP_CHIPSET BIT(ec_sensor_temp_chipset)++#define SENSOR_TEMP_CPU BIT(ec_sensor_temp_cpu)++#define SENSOR_TEMP_MB BIT(ec_sensor_temp_mb)++#define SENSOR_TEMP_T_SENSOR BIT(ec_sensor_temp_t_sensor)++#define SENSOR_TEMP_VRM BIT(ec_sensor_temp_vrm)++#define SENSOR_FAN_CPU_OPT BIT(ec_sensor_fan_cpu_opt)++#define SENSOR_FAN_VRM_HS BIT(ec_sensor_fan_vrm_hs)++#define SENSOR_FAN_CHIPSET BIT(ec_sensor_fan_chipset)++#define SENSOR_FAN_WATER_FLOW BIT(ec_sensor_fan_water_flow)++#define SENSOR_CURR_CPU BIT(ec_sensor_curr_cpu)++#define SENSOR_TEMP_WATER_IN BIT(ec_sensor_temp_water_in)++#define SENSOR_TEMP_WATER_OUT BIT(ec_sensor_temp_water_out)++++/* All the known sensors for ASUS EC controllers */++static const struct ec_sensor_info known_ec_sensors[] = {++ [ec_sensor_temp_chipset] =++ EC_SENSOR("Chipset", hwmon_temp, 1, 0x00, 0x3a),++ [ec_sensor_temp_cpu] = EC_SENSOR("CPU", hwmon_temp, 1, 0x00, 0x3b),++ [ec_sensor_temp_mb] =++ EC_SENSOR("Motherboard", hwmon_temp, 1, 0x00, 0x3c),++ [ec_sensor_temp_t_sensor] =++ EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x3d),++ [ec_sensor_temp_vrm] = EC_SENSOR("VRM", hwmon_temp, 1, 0x00, 0x3e),++ [ec_sensor_fan_cpu_opt] =++ EC_SENSOR("CPU_Opt", hwmon_fan, 2, 0x00, 0xb0),++ [ec_sensor_fan_vrm_hs] = EC_SENSOR("VRM HS", hwmon_fan, 2, 0x00, 0xb2),++ [ec_sensor_fan_chipset] =++ EC_SENSOR("Chipset", hwmon_fan, 2, 0x00, 0xb4),++ [ec_sensor_fan_water_flow] =++ EC_SENSOR("Water_Flow", hwmon_fan, 2, 0x00, 0xbc),++ [ec_sensor_curr_cpu] = EC_SENSOR("CPU", hwmon_curr, 1, 0x00, 0xf4),++ [ec_sensor_temp_water_in] =++ EC_SENSOR("Water_In", hwmon_temp, 1, 0x01, 0x00),++ [ec_sensor_temp_water_out] =++ EC_SENSOR("Water_Out", hwmon_temp, 1, 0x01, 0x01),++};++++/* Shortcuts for common combinations */++#define SENSOR_SET_TEMP_CHIPSET_CPU_MB \++ (SENSOR_TEMP_CHIPSET | SENSOR_TEMP_CPU | SENSOR_TEMP_MB)++#define SENSOR_SET_TEMP_WATER (SENSOR_TEMP_WATER_IN | SENSOR_TEMP_WATER_OUT)++++#define DMI_EXACT_MATCH_BOARD(vendor, name, sensors) { \++ .matches = { \++ DMI_EXACT_MATCH(DMI_BOARD_VENDOR, vendor), \++ DMI_EXACT_MATCH(DMI_BOARD_NAME, name), \++ }, \++ .driver_data = (void *)(sensors), \++}++++static const struct dmi_system_id asus_ec_dmi_table[] __initconst = {++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "PRIME X570-PRO",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM |++ SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "Pro WS X570-ACE",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM |++ SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE,++ "ROG CROSSHAIR VIII DARK HERO",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR |++ SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER |++ SENSOR_FAN_CPU_OPT | SENSOR_FAN_WATER_FLOW | SENSOR_CURR_CPU),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE,++ "ROG CROSSHAIR VIII FORMULA",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR |++ SENSOR_TEMP_VRM | SENSOR_FAN_CPU_OPT | SENSOR_FAN_CHIPSET |++ SENSOR_CURR_CPU),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG CROSSHAIR VIII HERO",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR |++ SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER |++ SENSOR_FAN_CPU_OPT | SENSOR_FAN_CHIPSET |++ SENSOR_FAN_WATER_FLOW | SENSOR_CURR_CPU),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE,++ "ROG CROSSHAIR VIII IMPACT",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR |++ SENSOR_TEMP_VRM | SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX B550-E GAMING",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB |++ SENSOR_TEMP_T_SENSOR |++ SENSOR_TEMP_VRM | SENSOR_FAN_CPU_OPT),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX B550-I GAMING",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB |++ SENSOR_TEMP_T_SENSOR |++ SENSOR_TEMP_VRM | SENSOR_FAN_VRM_HS | SENSOR_CURR_CPU),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX X570-E GAMING",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB |++ SENSOR_TEMP_T_SENSOR |++ SENSOR_TEMP_VRM | SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX X570-F GAMING",++ SENSOR_SET_TEMP_CHIPSET_CPU_MB |++ SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET++ ),++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX X570-I GAMING",++ SENSOR_TEMP_T_SENSOR | SENSOR_FAN_VRM_HS |++ SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU),++ {}++};++++struct ec_sensor {++ unsigned int info_index;++ u32 cached_value;++};++++struct ec_sensors_data {++ unsigned long board_sensors;++ struct ec_sensor *sensors;++ /** EC registers to read from */++ u16 *registers;++ u8 *read_buffer;++ /** sorted list of unique register banks */++ u8 banks[ASUS_EC_MAX_BANK + 1];++ /** in jiffies */++ unsigned long last_updated;++ acpi_handle aml_mutex;++ /** number of board EC sensors */++ u8 nr_sensors;++ /** number of EC registers to read (sensor might span more than 1 register) */++ u8 nr_registers;++ /** number of unique register banks */++ u8 nr_banks;++};++++static u8 register_bank(u16 reg)++{++ return reg >> 8;++}++++static u8 register_index(u16 reg)++{++ return reg & 0x00ff;++}++++static const struct ec_sensor_info *++get_sensor_info(const struct ec_sensors_data *state, int index)++{++ return &known_ec_sensors[state->sensors[index].info_index];++}++++static int find_ec_sensor_index(const struct ec_sensors_data *ec,++ enum hwmon_sensor_types type, int channel)++{++ unsigned int i;++++ for (i = 0; i < ec->nr_sensors; i++) {++ if (get_sensor_info(ec, i)->type == type) {++ if (channel == 0)++ return i;++ channel--;++ }++ }++ return -ENOENT;++}++++static int __init bank_compare(const void *a, const void *b)++{++ return *((const s8 *)a) - *((const s8 *)b);++}++++static int __init board_sensors_count(unsigned long sensors)++{++ return hweight_long(sensors);++}++++static void __init setup_sensor_data(struct ec_sensors_data *ec)++{++ struct ec_sensor *s = ec->sensors;++ bool bank_found;++ int i, j;++ u8 bank;++++ ec->nr_banks = 0;++ ec->nr_registers = 0;++++ for_each_set_bit(i, &ec->board_sensors,++ BITS_PER_TYPE(ec->board_sensors)) {++ s->info_index = i;++ s->cached_value = 0;++ ec->nr_registers +=++ known_ec_sensors[s->info_index].addr.components.size;++ bank_found = false;++ bank = known_ec_sensors[s->info_index].addr.components.bank;++ for (j = 0; j < ec->nr_banks; j++) {++ if (ec->banks[j] == bank) {++ bank_found = true;++ break;++ }++ }++ if (!bank_found) {++ ec->banks[ec->nr_banks++] = bank;++ }++ s++;++ }++ sort(ec->banks, ec->nr_banks, 1, bank_compare, NULL);++}++++static void __init fill_ec_registers(struct ec_sensors_data *ec)++{++ const struct ec_sensor_info *si;++ unsigned int i, j, register_idx = 0;++++ for (i = 0; i < ec->nr_sensors; ++i) {++ si = get_sensor_info(ec, i);++ for (j = 0; j < si->addr.components.size; ++j, ++register_idx) {++ ec->registers[register_idx] =++ (si->addr.components.bank << 8) +++ si->addr.components.index + j;++ }++ }++}++++static acpi_handle __init asus_hw_access_mutex(struct device *dev)++{++ const char *mutex_path;++ acpi_handle res;++ int status;++++ mutex_path = mutex_path_override ?++ mutex_path_override : ASUS_HW_ACCESS_MUTEX_ASMX;++++ status = acpi_get_handle(NULL, (acpi_string)mutex_path, &res);++ if (ACPI_FAILURE(status)) {++ dev_err(dev,++ "Could not get hardware access guard mutex '%s': error %d",++ mutex_path, status);++ return NULL;++ }++ return res;++}++++static int asus_ec_bank_switch(u8 bank, u8 *old)++{++ int status = 0;++++ if (old) {++ status = ec_read(ASUS_EC_BANK_REGISTER, old);++ }++ if (status || (old && (*old == bank)))++ return status;++ return ec_write(ASUS_EC_BANK_REGISTER, bank);++}++++static int asus_ec_block_read(const struct device *dev,++ struct ec_sensors_data *ec)++{++ int ireg, ibank, status;++ u8 bank, reg_bank, prev_bank;++++ bank = 0;++ status = asus_ec_bank_switch(bank, &prev_bank);++ if (status) {++ dev_warn(dev, "EC bank switch failed");++ return status;++ }++++ if (prev_bank) {++ /* oops... somebody else is working with the EC too */++ dev_warn(dev,++ "Concurrent access to the ACPI EC detected.\nRace condition possible.");++ }++++ /*++ * read registers minimizing bank switches.++ */++ for (ibank = 0; ibank < ec->nr_banks; ibank++) {++ if (bank != ec->banks[ibank]) {++ bank = ec->banks[ibank];++ if (asus_ec_bank_switch(bank, NULL)) {++ dev_warn(dev, "EC bank switch to %d failed",++ bank);++ break;++ }++ }++ for (ireg = 0; ireg < ec->nr_registers; ireg++) {++ reg_bank = register_bank(ec->registers[ireg]);++ if (reg_bank < bank) {++ continue;++ }++ ec_read(register_index(ec->registers[ireg]),++ ec->read_buffer + ireg);++ }++ }++++ status = asus_ec_bank_switch(prev_bank, NULL);++ return status;++}++++static inline u32 get_sensor_value(const struct ec_sensor_info *si, u8 *data)++{++ switch (si->addr.components.size) {++ case 1:++ return *data;++ case 2:++ return get_unaligned_be16(data);++ case 4:++ return get_unaligned_be32(data);++ default:++ return 0;++ }++}++++static void update_sensor_values(struct ec_sensors_data *ec, u8 *data)++{++ const struct ec_sensor_info *si;++ struct ec_sensor *s;++++ for (s = ec->sensors; s != ec->sensors + ec->nr_sensors; s++) {++ si = &known_ec_sensors[s->info_index];++ s->cached_value = get_sensor_value(si, data);++ data += si->addr.components.size;++ }++}++++static int update_ec_sensors(const struct device *dev,++ struct ec_sensors_data *ec)++{++ int status;++++ /*++ * ASUS DSDT does not specify that access to the EC has to be guarded,++ * but firmware does access it via ACPI++ */++ if (ACPI_FAILURE(acpi_acquire_mutex(ec->aml_mutex, NULL,++ ACPI_LOCK_DELAY_MS))) {++ dev_err(dev, "Failed to acquire AML mutex");++ status = -EBUSY;++ goto cleanup;++ }++++ status = asus_ec_block_read(dev, ec);++++ if (!status) {++ update_sensor_values(ec, ec->read_buffer);++ }++ if (ACPI_FAILURE(acpi_release_mutex(ec->aml_mutex, NULL))) {++ dev_err(dev, "Failed to release AML mutex");++ }++cleanup:++ return status;++}++++static int scale_sensor_value(u32 value, int data_type)++{++ switch (data_type) {++ case hwmon_curr:++ case hwmon_temp:++ case hwmon_in:++ return value * MILLI;++ default:++ return value;++ }++}++++static int get_cached_value_or_update(const struct device *dev,++ int sensor_index,++ struct ec_sensors_data *state, u32 *value)++{++ if (time_after(jiffies, state->last_updated + HZ)) {++ if (update_ec_sensors(dev, state)) {++ dev_err(dev, "update_ec_sensors() failure\n");++ return -EIO;++ }++++ state->last_updated = jiffies;++ }++++ *value = state->sensors[sensor_index].cached_value;++ return 0;++}++++/*++ * Now follow the functions that implement the hwmon interface++ */++++static int asus_ec_hwmon_read(struct device *dev, enum hwmon_sensor_types type,++ u32 attr, int channel, long *val)++{++ int ret;++ u32 value = 0;++++ struct ec_sensors_data *state = dev_get_drvdata(dev);++ int sidx = find_ec_sensor_index(state, type, channel);++++ if (sidx < 0) {++ return sidx;++ }++++ ret = get_cached_value_or_update(dev, sidx, state, &value);++ if (!ret) {++ *val = scale_sensor_value(value,++ get_sensor_info(state, sidx)->type);++ }++++ return ret;++}++++static int asus_ec_hwmon_read_string(struct device *dev,++ enum hwmon_sensor_types type, u32 attr,++ int channel, const char **str)++{++ struct ec_sensors_data *state = dev_get_drvdata(dev);++ int sensor_index = find_ec_sensor_index(state, type, channel);++ *str = get_sensor_info(state, sensor_index)->label;++++ return 0;++}++++static umode_t asus_ec_hwmon_is_visible(const void *drvdata,++ enum hwmon_sensor_types type, u32 attr,++ int channel)++{++ const struct ec_sensors_data *state = drvdata;++++ return find_ec_sensor_index(state, type, channel) >= 0 ? S_IRUGO : 0;++}++++static int __init++asus_ec_hwmon_add_chan_info(struct hwmon_channel_info *asus_ec_hwmon_chan,++ struct device *dev, int num,++ enum hwmon_sensor_types type, u32 config)++{++ int i;++ u32 *cfg = devm_kcalloc(dev, num + 1, sizeof(*cfg), GFP_KERNEL);++++ if (!cfg)++ return -ENOMEM;++++ asus_ec_hwmon_chan->type = type;++ asus_ec_hwmon_chan->config = cfg;++ for (i = 0; i < num; i++, cfg++)++ *cfg = config;++++ return 0;++}++++static const struct hwmon_ops asus_ec_hwmon_ops = {++ .is_visible = asus_ec_hwmon_is_visible,++ .read = asus_ec_hwmon_read,++ .read_string = asus_ec_hwmon_read_string,++};++++static struct hwmon_chip_info asus_ec_chip_info = {++ .ops = &asus_ec_hwmon_ops,++};++++static unsigned long __init++get_board_sensors(const struct device *dev)++{++ const struct dmi_system_id *dmi_entry;++++ dmi_entry = dmi_first_match(asus_ec_dmi_table);++ if (!dmi_entry) {++ dev_info(dev, "Unsupported board");++ return 0;++ }++++ return (unsigned long)dmi_entry->driver_data;++}++++static int __init configure_sensor_setup(struct device *dev)++{++ struct ec_sensors_data *ec_data = dev_get_drvdata(dev);++ int nr_count[hwmon_max] = { 0 }, nr_types = 0;++ struct device *hwdev;++ struct hwmon_channel_info *asus_ec_hwmon_chan;++ const struct hwmon_channel_info **ptr_asus_ec_ci;++ const struct hwmon_chip_info *chip_info;++ const struct ec_sensor_info *si;++ enum hwmon_sensor_types type;++ unsigned int i;++++ ec_data->board_sensors = get_board_sensors(dev);++ if (!ec_data->board_sensors) {++ return -ENODEV;++ }++++ ec_data->nr_sensors = board_sensors_count(ec_data->board_sensors);++ ec_data->sensors = devm_kcalloc(dev, ec_data->nr_sensors,++ sizeof(struct ec_sensor), GFP_KERNEL);++++ setup_sensor_data(ec_data);++ ec_data->registers = devm_kcalloc(dev, ec_data->nr_registers,++ sizeof(u16), GFP_KERNEL);++ ec_data->read_buffer = devm_kcalloc(dev, ec_data->nr_registers,++ sizeof(u8), GFP_KERNEL);++++ if (!ec_data->registers || !ec_data->read_buffer) {++ return -ENOMEM;++ }++++ fill_ec_registers(ec_data);++++ ec_data->aml_mutex = asus_hw_access_mutex(dev);++++ for (i = 0; i < ec_data->nr_sensors; ++i) {++ si = get_sensor_info(ec_data, i);++ if (!nr_count[si->type])++ ++nr_types;++ ++nr_count[si->type];++ }++++ if (nr_count[hwmon_temp])++ nr_count[hwmon_chip]++, nr_types++;++++ asus_ec_hwmon_chan = devm_kcalloc(++ dev, nr_types, sizeof(*asus_ec_hwmon_chan), GFP_KERNEL);++ if (!asus_ec_hwmon_chan)++ return -ENOMEM;++++ ptr_asus_ec_ci = devm_kcalloc(dev, nr_types + 1,++ sizeof(*ptr_asus_ec_ci), GFP_KERNEL);++ if (!ptr_asus_ec_ci)++ return -ENOMEM;++++ asus_ec_chip_info.info = ptr_asus_ec_ci;++ chip_info = &asus_ec_chip_info;++++ for (type = 0; type < hwmon_max; ++type) {++ if (!nr_count[type])++ continue;++++ asus_ec_hwmon_add_chan_info(asus_ec_hwmon_chan, dev,++ nr_count[type], type,++ hwmon_attributes[type]);++ *ptr_asus_ec_ci++ = asus_ec_hwmon_chan++;++ }++++ dev_info(dev, "board has %d EC sensors that span %d registers",++ ec_data->nr_sensors, ec_data->nr_registers);++++ hwdev = devm_hwmon_device_register_with_info(dev, "asus_ec_sensors",++ ec_data, chip_info, NULL);++++ return PTR_ERR_OR_ZERO(hwdev);++}++++static int __init asus_ec_probe(struct platform_device *pdev)++{++ struct asus_ec_sensors *state;++ int status = 0;++++ state = devm_kzalloc(&pdev->dev, sizeof(struct ec_sensors_data),++ GFP_KERNEL);++++ if (!state) {++ return -ENOMEM;++ }++++ dev_set_drvdata(&pdev->dev, state);++ status = configure_sensor_setup(&pdev->dev);++ return status;++}++++static const struct acpi_device_id acpi_ec_ids[] = {++ /* Embedded Controller Device */++ { "PNP0C09", 0 },++ {}++};++++static struct platform_driver asus_ec_sensors_platform_driver = {++ .driver = {++ .name = "asus-ec-sensors",++ .acpi_match_table = acpi_ec_ids,++ },++};++++MODULE_DEVICE_TABLE(dmi, asus_ec_dmi_table);++module_platform_driver_probe(asus_ec_sensors_platform_driver, asus_ec_probe);++++module_param_named(mutex_path, mutex_path_override, charp, 0);++MODULE_PARM_DESC(mutex_path,++ "Override ACPI mutex path used to guard access to hardware");++++MODULE_AUTHOR("Eugene Shalygin <[email protected]>");++MODULE_DESCRIPTION(++ "HWMON driver for sensors accessible via ACPI EC in ASUS motherboards");++MODULE_LICENSE("GPL");+--+2.35.0++From ea45e4a77e054ab10a669845700399f15213628e Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sun, 23 Jan 2022 16:50:19 +0000+Subject: [PATCH] bbr2++---+ include/linux/tcp.h | 3 +-+ include/net/inet_connection_sock.h | 3 +-+ include/net/tcp.h | 41 +-+ include/uapi/linux/inet_diag.h | 33 ++ net/ipv4/Kconfig | 22 ++ net/ipv4/Makefile | 1 ++ net/ipv4/bpf_tcp_ca.c | 2 +-+ net/ipv4/tcp.c | 1 ++ net/ipv4/tcp_bbr.c | 38 +-+ net/ipv4/tcp_bbr2.c | 2674 +++++++++++++++++++++++++++++ net/ipv4/tcp_cong.c | 1 ++ net/ipv4/tcp_input.c | 27 +-+ net/ipv4/tcp_output.c | 26 +-+ net/ipv4/tcp_rate.c | 30 +-+ net/ipv4/tcp_timer.c | 1 ++ 15 files changed, 2868 insertions(+), 35 deletions(-)+ create mode 100644 net/ipv4/tcp_bbr2.c++diff --git a/include/linux/tcp.h b/include/linux/tcp.h+index 78b91bb92f0d..618568b01a80 100644+--- a/include/linux/tcp.h++++ b/include/linux/tcp.h+@@ -225,7 +225,8 @@ struct tcp_sock {+ u8 compressed_ack;+ u8 dup_ack_counter:2,+ tlp_retrans:1, /* TLP is a retransmission */+- unused:5;++ fast_ack_mode:2, /* which fast ack mode ? */++ unused:3;+ u32 chrono_start; /* Start time in jiffies of a TCP chrono */+ u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */+ u8 chrono_type:2, /* current chronograph type */+diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h+index 4ad47d9f9d27..7623bb1f89c0 100644+--- a/include/net/inet_connection_sock.h++++ b/include/net/inet_connection_sock.h+@@ -134,7 +134,8 @@ struct inet_connection_sock {+ u32 icsk_probes_tstamp;+ u32 icsk_user_timeout;++- u64 icsk_ca_priv[104 / sizeof(u64)];++/* XXX inflated by temporary internal debugging info */++ u64 icsk_ca_priv[216 / sizeof(u64)];+ #define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv)+ };++diff --git a/include/net/tcp.h b/include/net/tcp.h+index 44e442bf23f9..b49fe895506e 100644+--- a/include/net/tcp.h++++ b/include/net/tcp.h+@@ -370,6 +370,7 @@ static inline void tcp_dec_quickack_mode(struct sock *sk,+ #define TCP_ECN_QUEUE_CWR 2+ #define TCP_ECN_DEMAND_CWR 4+ #define TCP_ECN_SEEN 8++#define TCP_ECN_ECT_PERMANENT 16++ enum tcp_tw_status {+ TCP_TW_SUCCESS = 0,+@@ -808,6 +809,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)+ return max_t(s64, t1 - t0, 0);+ }+++static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0)++{++ return max_t(s32, t1 - t0, 0);++}+++ static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)+ {+ return tcp_ns_to_ts(skb->skb_mstamp_ns);+@@ -883,9 +889,14 @@ struct tcp_skb_cb {+ /* pkts S/ACKed so far upon tx of skb, incl retrans: */+ __u32 delivered;+ /* start of send pipeline phase */+- u64 first_tx_mstamp;++ u32 first_tx_mstamp;+ /* when we reached the "delivered" count */+- u64 delivered_mstamp;++ u32 delivered_mstamp;++#define TCPCB_IN_FLIGHT_BITS 20++#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1)++ u32 in_flight:20, /* packets in flight at transmit */++ unused2:12;++ u32 lost; /* packets lost so far upon tx of skb */+ } tx; /* only used for outgoing skbs */+ union {+ struct inet_skb_parm h4;+@@ -1011,7 +1022,11 @@ enum tcp_ca_ack_event_flags {+ #define TCP_CONG_NON_RESTRICTED 0x1+ /* Requires ECN/ECT set on all packets */+ #define TCP_CONG_NEEDS_ECN 0x2+-#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)++/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */++#define TCP_CONG_WANTS_CE_EVENTS 0x4++#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \++ TCP_CONG_NEEDS_ECN | \++ TCP_CONG_WANTS_CE_EVENTS)++ union tcp_cc_info;++@@ -1031,8 +1046,11 @@ struct ack_sample {+ */+ struct rate_sample {+ u64 prior_mstamp; /* starting timestamp for interval */++ u32 prior_lost; /* tp->lost at "prior_mstamp" */+ u32 prior_delivered; /* tp->delivered at "prior_mstamp" */+ u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */++ u32 tx_in_flight; /* packets in flight at starting timestamp */++ s32 lost; /* number of packets lost over interval */+ s32 delivered; /* number of packets delivered over interval */+ s32 delivered_ce; /* number of packets delivered w/ CE marks*/+ long interval_us; /* time for tp->delivered to incr "delivered" */+@@ -1045,6 +1063,7 @@ struct rate_sample {+ bool is_app_limited; /* is sample from packet with bubble in pipe? */+ bool is_retrans; /* is sample from retransmission? */+ bool is_ack_delayed; /* is this (likely) a delayed ACK? */++ bool is_ece; /* did this ACK have ECN marked? */+ };++ struct tcp_congestion_ops {+@@ -1068,8 +1087,11 @@ struct tcp_congestion_ops {+ /* hook for packet ack accounting (optional) */+ void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);++- /* override sysctl_tcp_min_tso_segs */+- u32 (*min_tso_segs)(struct sock *sk);++ /* pick target number of segments per TSO/GSO skb (optional): */++ u32 (*tso_segs)(struct sock *sk, unsigned int mss_now);++++ /* react to a specific lost skb (optional) */++ void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb);++ /* call when packets are delivered to update cwnd and pacing rate,+ * after all the ca_state processing. (optional)+@@ -1132,6 +1154,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)+ }+ #endif+++static inline bool tcp_ca_wants_ce_events(const struct sock *sk)++{++ const struct inet_connection_sock *icsk = inet_csk(sk);++++ return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN |++ TCP_CONG_WANTS_CE_EVENTS);++}+++ static inline bool tcp_ca_needs_ecn(const struct sock *sk)+ {+ const struct inet_connection_sock *icsk = inet_csk(sk);+@@ -1157,6 +1187,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)+ }++ /* From tcp_rate.c */++void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb);+ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);+ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,+ struct rate_sample *rs);+diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h+index 20ee93f0f876..96d52dd9c48a 100644+--- a/include/uapi/linux/inet_diag.h++++ b/include/uapi/linux/inet_diag.h+@@ -231,9 +231,42 @@ struct tcp_bbr_info {+ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */+ };+++/* Phase as reported in netlink/ss stats. */++enum tcp_bbr2_phase {++ BBR2_PHASE_INVALID = 0,++ BBR2_PHASE_STARTUP = 1,++ BBR2_PHASE_DRAIN = 2,++ BBR2_PHASE_PROBE_RTT = 3,++ BBR2_PHASE_PROBE_BW_UP = 4,++ BBR2_PHASE_PROBE_BW_DOWN = 5,++ BBR2_PHASE_PROBE_BW_CRUISE = 6,++ BBR2_PHASE_PROBE_BW_REFILL = 7++};++++struct tcp_bbr2_info {++ /* u64 bw: bandwidth (app throughput) estimate in Byte per sec: */++ __u32 bbr_bw_lsb; /* lower 32 bits of bw */++ __u32 bbr_bw_msb; /* upper 32 bits of bw */++ __u32 bbr_min_rtt; /* min-filtered RTT in uSec */++ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */++ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */++ __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */++ __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */++ __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */++ __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */++ __u8 bbr_mode; /* current bbr_mode in state machine */++ __u8 bbr_phase; /* current state machine phase */++ __u8 unused1; /* alignment padding; not used yet */++ __u8 bbr_version; /* MUST be at this offset in struct */++ __u32 bbr_inflight_lo; /* lower/short-term data volume bound */++ __u32 bbr_inflight_hi; /* higher/long-term data volume bound */++ __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */++};+++ union tcp_cc_info {+ struct tcpvegas_info vegas;+ struct tcp_dctcp_info dctcp;+ struct tcp_bbr_info bbr;++ struct tcp_bbr2_info bbr2;+ };+ #endif /* _UAPI_INET_DIAG_H_ */+diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig+index 87983e70f03f..a833a7a67ce7 100644+--- a/net/ipv4/Kconfig++++ b/net/ipv4/Kconfig+@@ -669,6 +669,24 @@ config TCP_CONG_BBR+ AQM schemes that do not provide a delay signal. It requires the fq+ ("Fair Queue") pacing packet scheduler.+++config TCP_CONG_BBR2++ tristate "BBR2 TCP"++ default n++ help++++ BBR2 TCP congestion control is a model-based congestion control++ algorithm that aims to maximize network utilization, keep queues and++ retransmit rates low, and to be able to coexist with Reno/CUBIC in++ common scenarios. It builds an explicit model of the network path. It++ tolerates a targeted degree of random packet loss and delay that are++ unrelated to congestion. It can operate over LAN, WAN, cellular, wifi,++ or cable modem links, and can use DCTCP-L4S-style ECN signals. It can++ coexist with flows that use loss-based congestion control, and can++ operate with shallow buffers, deep buffers, bufferbloat, policers, or++ AQM schemes that do not provide a delay signal. It requires pacing,++ using either TCP internal pacing or the fq ("Fair Queue") pacing packet++ scheduler.+++ choice+ prompt "Default TCP congestion control"+ default DEFAULT_CUBIC+@@ -706,6 +724,9 @@ choice+ config DEFAULT_BBR+ bool "BBR" if TCP_CONG_BBR=y+++ config DEFAULT_BBR2++ bool "BBR2" if TCP_CONG_BBR2=y+++ config DEFAULT_RENO+ bool "Reno"+ endchoice+@@ -730,6 +751,7 @@ config DEFAULT_TCP_CONG+ default "dctcp" if DEFAULT_DCTCP+ default "cdg" if DEFAULT_CDG+ default "bbr" if DEFAULT_BBR++ default "bbr2" if DEFAULT_BBR2+ default "cubic"++ config TCP_MD5SIG+diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile+index bbdd9c44f14e..8dee1547d820 100644+--- a/net/ipv4/Makefile++++ b/net/ipv4/Makefile+@@ -46,6 +46,7 @@ obj-$(CONFIG_INET_TCP_DIAG) += tcp_diag.o+ obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o+ obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o+ obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o++obj-$(CONFIG_TCP_CONG_BBR2) += tcp_bbr2.o+ obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o+ obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o+ obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o+diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c+index de610cb83694..afda34c26c62 100644+--- a/net/ipv4/bpf_tcp_ca.c++++ b/net/ipv4/bpf_tcp_ca.c+@@ -20,7 +20,7 @@ static u32 optional_ops[] = {+ offsetof(struct tcp_congestion_ops, cwnd_event),+ offsetof(struct tcp_congestion_ops, in_ack_event),+ offsetof(struct tcp_congestion_ops, pkts_acked),+- offsetof(struct tcp_congestion_ops, min_tso_segs),++ offsetof(struct tcp_congestion_ops, tso_segs),+ offsetof(struct tcp_congestion_ops, sndbuf_expand),+ offsetof(struct tcp_congestion_ops, cong_control),+ };+diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c+index 3b75836db19b..1f1ff1e18e69 100644+--- a/net/ipv4/tcp.c++++ b/net/ipv4/tcp.c+@@ -3074,6 +3074,7 @@ int tcp_disconnect(struct sock *sk, int flags)+ tp->rx_opt.dsack = 0;+ tp->rx_opt.num_sacks = 0;+ tp->rcv_ooopack = 0;++ tp->fast_ack_mode = 0;+++ /* Clean up fastopen related fields */+diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c+index ec5550089b4d..56e719166d4c 100644+--- a/net/ipv4/tcp_bbr.c++++ b/net/ipv4/tcp_bbr.c+@@ -294,26 +294,40 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain)+ sk->sk_pacing_rate = rate;+ }++-/* override sysctl_tcp_min_tso_segs */+ static u32 bbr_min_tso_segs(struct sock *sk)+ {+ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2;+ }+++/* Return the number of segments BBR would like in a TSO/GSO skb, given++ * a particular max gso size as a constraint.++ */++static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now,++ u32 gso_max_size)++{++ u32 segs;++ u64 bytes;++++ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */++ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift;++++ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER);++ segs = max_t(u32, bytes / mss_now, bbr_min_tso_segs(sk));++ return segs;++}++++/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */++static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now)++{++ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size);++}++++/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */+ static u32 bbr_tso_segs_goal(struct sock *sk)+ {+ struct tcp_sock *tp = tcp_sk(sk);+- u32 segs, bytes;+-+- /* Sort of tcp_tso_autosize() but ignoring+- * driver provided sk_gso_max_size.+- */+- bytes = min_t(unsigned long,+- sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift),+- GSO_MAX_SIZE - 1 - MAX_TCP_HEADER);+- segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk));++- return min(segs, 0x7FU);++ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE);+ }++ /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */+@@ -1149,7 +1163,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = {+ .undo_cwnd = bbr_undo_cwnd,+ .cwnd_event = bbr_cwnd_event,+ .ssthresh = bbr_ssthresh,+- .min_tso_segs = bbr_min_tso_segs,++ .tso_segs = bbr_tso_segs,+ .get_info = bbr_get_info,+ .set_state = bbr_set_state,+ };+diff --git a/net/ipv4/tcp_bbr2.c b/net/ipv4/tcp_bbr2.c+new file mode 100644+index 000000000000..1bef2336b3ff+--- /dev/null++++ b/net/ipv4/tcp_bbr2.c+@@ -0,0 +1,2674 @@++/* BBR (Bottleneck Bandwidth and RTT) congestion control, v2++ *++ * BBRv2 is a model-based congestion control algorithm that aims for low++ * queues, low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model++ * of the network path, it uses measurements of bandwidth and RTT, as well as++ * (if they occur) packet loss and/or DCTCP/L4S-style ECN signals. Note that++ * although it can use ECN or loss signals explicitly, it does not require++ * either; it can bound its in-flight data based on its estimate of the BDP.++ *++ * The model has both higher and lower bounds for the operating range:++ * lo: bw_lo, inflight_lo: conservative short-term lower bound++ * hi: bw_hi, inflight_hi: robust long-term upper bound++ * The bandwidth-probing time scale is (a) extended dynamically based on++ * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by++ * an interactive wall-clock time-scale to be more scalable and responsive++ * than Reno and CUBIC.++ *++ * Here is a state transition diagram for BBR:++ *++ * |++ * V++ * +---> STARTUP ----+++ * | | |++ * | V |++ * | DRAIN ----+++ * | | |++ * | V |++ * +---> PROBE_BW ----+++ * | ^ | |++ * | | | |++ * | +----+ |++ * | |++ * +---- PROBE_RTT <--+++ *++ * A BBR flow starts in STARTUP, and ramps up its sending rate quickly.++ * When it estimates the pipe is full, it enters DRAIN to drain the queue.++ * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT.++ * A long-lived BBR flow spends the vast majority of its time remaining++ * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth++ * in a fair manner, with a small, bounded queue. *If* a flow has been++ * continuously sending for the entire min_rtt window, and hasn't seen an RTT++ * sample that matches or decreases its min_rtt estimate for 10 seconds, then++ * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe++ * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if++ * we estimated that we reached the full bw of the pipe then we enter PROBE_BW;++ * otherwise we enter STARTUP to try to fill the pipe.++ *++ * BBR is described in detail in:++ * "BBR: Congestion-Based Congestion Control",++ * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh,++ * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016.++ *++ * There is a public e-mail list for discussing BBR development and testing:++ * https://groups.google.com/forum/#!forum/bbr-dev++ *++ * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled,++ * otherwise TCP stack falls back to an internal pacing using one high++ * resolution timer per TCP socket and may use more resources.++ */++#include <linux/module.h>++#include <net/tcp.h>++#include <linux/inet_diag.h>++#include <linux/inet.h>++#include <linux/random.h>++++#include "tcp_dctcp.h"++++/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth++ * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps.++ * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32.++ * Since the minimum window is >=4 packets, the lower bound isn't++ * an issue. The upper bound isn't an issue with existing technologies.++ */++#define BW_SCALE 24++#define BW_UNIT (1 << BW_SCALE)++++#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */++#define BBR_UNIT (1 << BBR_SCALE)++++#define FLAG_DEBUG_VERBOSE 0x1 /* Verbose debugging messages */++#define FLAG_DEBUG_LOOPBACK 0x2 /* Do NOT skip loopback addr */++++#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */++++/* BBR has the following modes for deciding how fast to send: */++enum bbr_mode {++ BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */++ BBR_DRAIN, /* drain any queue created during startup */++ BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */++ BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */++};++++/* How does the incoming ACK stream relate to our bandwidth probing? */++enum bbr_ack_phase {++ BBR_ACKS_INIT, /* not probing; not getting probe feedback */++ BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */++ BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */++ BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */++ BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */++};++++/* BBR congestion control block */++struct bbr {++ u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */++ u32 min_rtt_stamp; /* timestamp of min_rtt_us */++ u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */++ u32 probe_rtt_min_us; /* min RTT in bbr_probe_rtt_win_ms window */++ u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/++ u32 next_rtt_delivered; /* scb->tx.delivered at end of round */++ u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */++ u64 cycle_mstamp; /* time of this cycle phase start */++ u32 mode:3, /* current bbr_mode in state machine */++ prev_ca_state:3, /* CA state on previous ACK */++ packet_conservation:1, /* use packet conservation? */++ round_start:1, /* start of packet-timed tx->ack round? */++ ce_state:1, /* If most recent data has CE bit set */++ bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */++ try_fast_path:1, /* can we take fast path? */++ unused2:11,++ idle_restart:1, /* restarting after idle? */++ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */++ cycle_idx:3, /* current index in pacing_gain cycle array */++ has_seen_rtt:1; /* have we seen an RTT sample yet? */++ u32 pacing_gain:11, /* current gain for setting pacing rate */++ cwnd_gain:11, /* current gain for setting cwnd */++ full_bw_reached:1, /* reached full bw in Startup? */++ full_bw_cnt:2, /* number of rounds without large bw gains */++ init_cwnd:7; /* initial cwnd */++ u32 prior_cwnd; /* prior cwnd upon entering loss recovery */++ u32 full_bw; /* recent bw, to estimate if pipe is full */++++ /* For tracking ACK aggregation: */++ u64 ack_epoch_mstamp; /* start of ACK sampling epoch */++ u16 extra_acked[2]; /* max excess data ACKed in epoch */++ u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */++ extra_acked_win_rtts:5, /* age of extra_acked, in round trips */++ extra_acked_win_idx:1, /* current index in extra_acked array */++ /* BBR v2 state: */++ unused1:2,++ startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */++ loss_in_cycle:1, /* packet loss in this cycle? */++ ecn_in_cycle:1; /* ECN in this cycle? */++ u32 loss_round_delivered; /* scb->tx.delivered ending loss round */++ u32 undo_bw_lo; /* bw_lo before latest losses */++ u32 undo_inflight_lo; /* inflight_lo before latest losses */++ u32 undo_inflight_hi; /* inflight_hi before latest losses */++ u32 bw_latest; /* max delivered bw in last round trip */++ u32 bw_lo; /* lower bound on sending bandwidth */++ u32 bw_hi[2]; /* upper bound of sending bandwidth range*/++ u32 inflight_latest; /* max delivered data in last round trip */++ u32 inflight_lo; /* lower bound of inflight data range */++ u32 inflight_hi; /* upper bound of inflight data range */++ u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */++ u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */++ u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */++ u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */++ ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */++ bw_probe_samples:1, /* rate samples reflect bw probing? */++ prev_probe_too_high:1, /* did last PROBE_UP go too high? */++ stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */++ rounds_since_probe:8, /* packet-timed rounds since probed bw */++ loss_round_start:1, /* loss_round_delivered round trip? */++ loss_in_round:1, /* loss marked in this round trip? */++ ecn_in_round:1, /* ECN marked in this round trip? */++ ack_phase:3, /* bbr_ack_phase: meaning of ACKs */++ loss_events_in_round:4,/* losses in STARTUP round */++ initialized:1; /* has bbr_init() been called? */++ u32 alpha_last_delivered; /* tp->delivered at alpha update */++ u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */++++ /* Params configurable using setsockopt. Refer to correspoding++ * module param for detailed description of params.++ */++ struct bbr_params {++ u32 high_gain:11, /* max allowed value: 2047 */++ drain_gain:10, /* max allowed value: 1023 */++ cwnd_gain:11; /* max allowed value: 2047 */++ u32 cwnd_min_target:4, /* max allowed value: 15 */++ min_rtt_win_sec:5, /* max allowed value: 31 */++ probe_rtt_mode_ms:9, /* max allowed value: 511 */++ full_bw_cnt:3, /* max allowed value: 7 */++ cwnd_tso_budget:1, /* allowed values: {0, 1} */++ unused3:6,++ drain_to_target:1, /* boolean */++ precise_ece_ack:1, /* boolean */++ extra_acked_in_startup:1, /* allowed values: {0, 1} */++ fast_path:1; /* boolean */++ u32 full_bw_thresh:10, /* max allowed value: 1023 */++ startup_cwnd_gain:11, /* max allowed value: 2047 */++ bw_probe_pif_gain:9, /* max allowed value: 511 */++ usage_based_cwnd:1, /* boolean */++ unused2:1;++ u16 probe_rtt_win_ms:14, /* max allowed value: 16383 */++ refill_add_inc:2; /* max allowed value: 3 */++ u16 extra_acked_gain:11, /* max allowed value: 2047 */++ extra_acked_win_rtts:5; /* max allowed value: 31*/++ u16 pacing_gain[CYCLE_LEN]; /* max allowed value: 1023 */++ /* Mostly BBR v2 parameters below here: */++ u32 ecn_alpha_gain:8, /* max allowed value: 255 */++ ecn_factor:8, /* max allowed value: 255 */++ ecn_thresh:8, /* max allowed value: 255 */++ beta:8; /* max allowed value: 255 */++ u32 ecn_max_rtt_us:19, /* max allowed value: 524287 */++ bw_probe_reno_gain:9, /* max allowed value: 511 */++ full_loss_cnt:4; /* max allowed value: 15 */++ u32 probe_rtt_cwnd_gain:8, /* max allowed value: 255 */++ inflight_headroom:8, /* max allowed value: 255 */++ loss_thresh:8, /* max allowed value: 255 */++ bw_probe_max_rounds:8; /* max allowed value: 255 */++ u32 bw_probe_rand_rounds:4, /* max allowed value: 15 */++ bw_probe_base_us:26, /* usecs: 0..2^26-1 (67 secs) */++ full_ecn_cnt:2; /* max allowed value: 3 */++ u32 bw_probe_rand_us:26, /* usecs: 0..2^26-1 (67 secs) */++ undo:1, /* boolean */++ tso_rtt_shift:4, /* max allowed value: 15 */++ unused5:1;++ u32 ecn_reprobe_gain:9, /* max allowed value: 511 */++ unused1:14,++ ecn_alpha_init:9; /* max allowed value: 256 */++ } params;++++ struct {++ u32 snd_isn; /* Initial sequence number */++ u32 rs_bw; /* last valid rate sample bw */++ u32 target_cwnd; /* target cwnd, based on BDP */++ u8 undo:1, /* Undo even happened but not yet logged */++ unused:7;++ char event; /* single-letter event debug codes */++ u16 unused2;++ } debug;++};++++struct bbr_context {++ u32 sample_bw;++ u32 target_cwnd;++ u32 log:1;++};++++/* Window length of min_rtt filter (in sec). Max allowed value is 31 (0x1F) */++static u32 bbr_min_rtt_win_sec = 10;++/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode.++ * Max allowed value is 511 (0x1FF).++ */++static u32 bbr_probe_rtt_mode_ms = 200;++/* Window length of probe_rtt_min_us filter (in ms), and consequently the++ * typical interval between PROBE_RTT mode entries.++ * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC++ */++static u32 bbr_probe_rtt_win_ms = 5000;++/* Skip TSO below the following bandwidth (bits/sec): */++static int bbr_min_tso_rate = 1200000;++++/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting++ * in bigger TSO bursts. By default we cut the RTT-based allowance in half++ * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance++ * is below 1500 bytes after 6 * ~500 usec = 3ms.++ */++static u32 bbr_tso_rtt_shift = 9; /* halve allowance per 2^9 usecs, 512us */++++/* Select cwnd TSO budget approach:++ * 0: padding++ * 1: flooring++ */++static uint bbr_cwnd_tso_budget = 1;++++/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck.++ * In order to help drive the network toward lower queues and low latency while++ * maintaining high utilization, the average pacing rate aims to be slightly++ * lower than the estimated bandwidth. This is an important aspect of the++ * design.++ */++static const int bbr_pacing_margin_percent = 1;++++/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain++ * that will allow a smoothly increasing pacing rate that will double each RTT++ * and send the same number of packets per RTT that an un-paced, slow-starting++ * Reno or CUBIC flow would. Max allowed value is 2047 (0x7FF).++ */++static int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1;++/* The gain for deriving startup cwnd. Max allowed value is 2047 (0x7FF). */++static int bbr_startup_cwnd_gain = BBR_UNIT * 2885 / 1000 + 1;++/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain++ * the queue created in BBR_STARTUP in a single round. Max allowed value++ * is 1023 (0x3FF).++ */++static int bbr_drain_gain = BBR_UNIT * 1000 / 2885;++/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs.++ * Max allowed value is 2047 (0x7FF).++ */++static int bbr_cwnd_gain = BBR_UNIT * 2;++/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw.++ * Max allowed value for each element is 1023 (0x3FF).++ */++enum bbr_pacing_gain_phase {++ BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */++ BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */++ BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */++ BBR_BW_PROBE_REFILL = 3, /* v2: refill the pipe again to 100% */++};++static int bbr_pacing_gain[] = {++ BBR_UNIT * 5 / 4, /* probe for more available bw */++ BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */++ BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */++ BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */++};++++/* Try to keep at least this many packets in flight, if things go smoothly. For++ * smooth functioning, a sliding window protocol ACKing every other packet++ * needs at least 4 packets in flight. Max allowed value is 15 (0xF).++ */++static u32 bbr_cwnd_min_target = 4;++++/* Cwnd to BDP proportion in PROBE_RTT mode scaled by BBR_UNIT. Default: 50%.++ * Use 0 to disable. Max allowed value is 255.++ */++static u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2;++++/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */++/* If bw has increased significantly (1.25x), there may be more bw available.++ * Max allowed value is 1023 (0x3FF).++ */++static u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4;++/* But after 3 rounds w/o significant bw growth, estimate pipe is full.++ * Max allowed value is 7 (0x7).++ */++static u32 bbr_full_bw_cnt = 3;++++static u32 bbr_flags; /* Debugging related stuff */++++/* Whether to debug using printk.++ */++static bool bbr_debug_with_printk;++++/* Whether to debug using ftrace event tcp:tcp_bbr_event.++ * Ignored when bbr_debug_with_printk is set.++ */++static bool bbr_debug_ftrace;++++/* Experiment: each cycle, try to hold sub-unity gain until inflight <= BDP. */++static bool bbr_drain_to_target = true; /* default: enabled */++++/* Experiment: Flags to control BBR with ECN behavior.++ */++static bool bbr_precise_ece_ack = true; /* default: enabled */++++/* The max rwin scaling shift factor is 14 (RFC 1323), so the max sane rwin is++ * (2^(16+14) B)/(1024 B/packet) = 1M packets.++ */++static u32 bbr_cwnd_warn_val = 1U << 20;++++static u16 bbr_debug_port_mask;++++/* BBR module parameters. These are module parameters only in Google prod.++ * Upstream these are intentionally not module parameters.++ */++static int bbr_pacing_gain_size = CYCLE_LEN;++++/* Gain factor for adding extra_acked to target cwnd: */++static int bbr_extra_acked_gain = 256;++++/* Window length of extra_acked window. Max allowed val is 31. */++static u32 bbr_extra_acked_win_rtts = 5;++++/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */++static u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20;++++/* Time period for clamping cwnd increment due to ack aggregation */++static u32 bbr_extra_acked_max_us = 100 * 1000;++++/* Use extra acked in startup ?++ * 0: disabled++ * 1: use latest extra_acked value from 1-2 rtt in startup++ */++static int bbr_extra_acked_in_startup = 1; /* default: enabled */++++/* Experiment: don't grow cwnd beyond twice of what we just probed. */++static bool bbr_usage_based_cwnd; /* default: disabled */++++/* For lab testing, researchers can enable BBRv2 ECN support with this flag,++ * when they know that any ECN marks that the connections experience will be++ * DCTCP/L4S-style ECN marks, rather than RFC3168 ECN marks.++ * TODO(ncardwell): Production use of the BBRv2 ECN functionality depends on++ * negotiation or configuration that is outside the scope of the BBRv2++ * alpha release.++ */++static bool bbr_ecn_enable = false;++++module_param_named(min_tso_rate, bbr_min_tso_rate, int, 0644);++module_param_named(tso_rtt_shift, bbr_tso_rtt_shift, int, 0644);++module_param_named(high_gain, bbr_high_gain, int, 0644);++module_param_named(drain_gain, bbr_drain_gain, int, 0644);++module_param_named(startup_cwnd_gain, bbr_startup_cwnd_gain, int, 0644);++module_param_named(cwnd_gain, bbr_cwnd_gain, int, 0644);++module_param_array_named(pacing_gain, bbr_pacing_gain, int,++ &bbr_pacing_gain_size, 0644);++module_param_named(cwnd_min_target, bbr_cwnd_min_target, uint, 0644);++module_param_named(probe_rtt_cwnd_gain,++ bbr_probe_rtt_cwnd_gain, uint, 0664);++module_param_named(cwnd_warn_val, bbr_cwnd_warn_val, uint, 0664);++module_param_named(debug_port_mask, bbr_debug_port_mask, ushort, 0644);++module_param_named(flags, bbr_flags, uint, 0644);++module_param_named(debug_ftrace, bbr_debug_ftrace, bool, 0644);++module_param_named(debug_with_printk, bbr_debug_with_printk, bool, 0644);++module_param_named(min_rtt_win_sec, bbr_min_rtt_win_sec, uint, 0644);++module_param_named(probe_rtt_mode_ms, bbr_probe_rtt_mode_ms, uint, 0644);++module_param_named(probe_rtt_win_ms, bbr_probe_rtt_win_ms, uint, 0644);++module_param_named(full_bw_thresh, bbr_full_bw_thresh, uint, 0644);++module_param_named(full_bw_cnt, bbr_full_bw_cnt, uint, 0644);++module_param_named(cwnd_tso_bduget, bbr_cwnd_tso_budget, uint, 0664);++module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664);++module_param_named(extra_acked_win_rtts,++ bbr_extra_acked_win_rtts, uint, 0664);++module_param_named(extra_acked_max_us,++ bbr_extra_acked_max_us, uint, 0664);++module_param_named(ack_epoch_acked_reset_thresh,++ bbr_ack_epoch_acked_reset_thresh, uint, 0664);++module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664);++module_param_named(precise_ece_ack, bbr_precise_ece_ack, bool, 0664);++module_param_named(extra_acked_in_startup,++ bbr_extra_acked_in_startup, int, 0664);++module_param_named(usage_based_cwnd, bbr_usage_based_cwnd, bool, 0664);++module_param_named(ecn_enable, bbr_ecn_enable, bool, 0664);++++static void bbr2_exit_probe_rtt(struct sock *sk);++static void bbr2_reset_congestion_signals(struct sock *sk);++++static void bbr_check_probe_rtt_done(struct sock *sk);++++/* Do we estimate that STARTUP filled the pipe? */++static bool bbr_full_bw_reached(const struct sock *sk)++{++ const struct bbr *bbr = inet_csk_ca(sk);++++ return bbr->full_bw_reached;++}++++/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */++static u32 bbr_max_bw(const struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ return max(bbr->bw_hi[0], bbr->bw_hi[1]);++}++++/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */++static u32 bbr_bw(const struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ return min(bbr_max_bw(sk), bbr->bw_lo);++}++++/* Return maximum extra acked in past k-2k round trips,++ * where k = bbr_extra_acked_win_rtts.++ */++static u16 bbr_extra_acked(const struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ return max(bbr->extra_acked[0], bbr->extra_acked[1]);++}++++/* Return rate in bytes per second, optionally with a gain.++ * The order here is chosen carefully to avoid overflow of u64. This should++ * work for input rates of up to 2.9Tbit/sec and gain of 2.89x.++ */++static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain,++ int margin)++{++ unsigned int mss = tcp_sk(sk)->mss_cache;++++ rate *= mss;++ rate *= gain;++ rate >>= BBR_SCALE;++ rate *= USEC_PER_SEC / 100 * (100 - margin);++ rate >>= BW_SCALE;++ rate = max(rate, 1ULL);++ return rate;++}++++static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate)++{++ return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0);++}++++static u64 bbr_rate_kbps(struct sock *sk, u64 rate)++{++ rate = bbr_bw_bytes_per_sec(sk, rate);++ rate *= 8;++ do_div(rate, 1000);++ return rate;++}++++static u32 bbr_tso_segs_goal(struct sock *sk);++static void bbr_debug(struct sock *sk, u32 acked,++ const struct rate_sample *rs, struct bbr_context *ctx)++{++ static const char ca_states[] = {++ [TCP_CA_Open] = 'O',++ [TCP_CA_Disorder] = 'D',++ [TCP_CA_CWR] = 'C',++ [TCP_CA_Recovery] = 'R',++ [TCP_CA_Loss] = 'L',++ };++ static const char mode[] = {++ 'G', /* Growing - BBR_STARTUP */++ 'D', /* Drain - BBR_DRAIN */++ 'W', /* Window - BBR_PROBE_BW */++ 'M', /* Min RTT - BBR_PROBE_RTT */++ };++ static const char ack_phase[] = { /* bbr_ack_phase strings */++ 'I', /* BBR_ACKS_INIT - 'Init' */++ 'R', /* BBR_ACKS_REFILLING - 'Refilling' */++ 'B', /* BBR_ACKS_PROBE_STARTING - 'Before' */++ 'F', /* BBR_ACKS_PROBE_FEEDBACK - 'Feedback' */++ 'A', /* BBR_ACKS_PROBE_STOPPING - 'After' */++ };++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ const u32 una = tp->snd_una - bbr->debug.snd_isn;++ const u32 fack = tcp_highest_sack_seq(tp);++ const u16 dport = ntohs(inet_sk(sk)->inet_dport);++ bool is_port_match = (bbr_debug_port_mask &&++ ((dport & bbr_debug_port_mask) == 0));++ char debugmsg[320];++++ if (sk->sk_state == TCP_SYN_SENT)++ return; /* no bbr_init() yet if SYN retransmit -> CA_Loss */++++ if (!tp->snd_cwnd || tp->snd_cwnd > bbr_cwnd_warn_val) {++ char addr[INET6_ADDRSTRLEN + 10] = { 0 };++++ if (sk->sk_family == AF_INET)++ snprintf(addr, sizeof(addr), "%pI4:%u",++ &inet_sk(sk)->inet_daddr, dport);++ else if (sk->sk_family == AF_INET6)++ snprintf(addr, sizeof(addr), "%pI6:%u",++ &sk->sk_daddr, dport);++++ WARN_ONCE(1,++ "BBR %s cwnd alert: %u "++ "snd_una: %u ca: %d pacing_gain: %u cwnd_gain: %u "++ "bw: %u rtt: %u min_rtt: %u "++ "acked: %u tso_segs: %u "++ "bw: %d %ld %d pif: %u\n",++ addr, tp->snd_cwnd,++ una, inet_csk(sk)->icsk_ca_state,++ bbr->pacing_gain, bbr->cwnd_gain,++ bbr_max_bw(sk), (tp->srtt_us >> 3), bbr->min_rtt_us,++ acked, bbr_tso_segs_goal(sk),++ rs->delivered, rs->interval_us, rs->is_retrans,++ tcp_packets_in_flight(tp));++ }++++ if (likely(!bbr_debug_with_printk && !bbr_debug_ftrace))++ return;++++ if (!sock_flag(sk, SOCK_DBG) && !is_port_match)++ return;++++ if (!ctx->log && !tp->app_limited && !(bbr_flags & FLAG_DEBUG_VERBOSE))++ return;++++ if (ipv4_is_loopback(inet_sk(sk)->inet_daddr) &&++ !(bbr_flags & FLAG_DEBUG_LOOPBACK))++ return;++++ snprintf(debugmsg, sizeof(debugmsg) - 1,++ "BBR %pI4:%-5u %5u,%03u:%-7u %c "++ "%c %2u br %2u cr %2d rtt %5ld d %2d i %5ld mrtt %d %cbw %llu "++ "bw %llu lb %llu ib %llu qb %llu "++ "a %u if %2u %c %c dl %u l %u al %u # %u t %u %c %c "++ "lr %d er %d ea %d bwl %lld il %d ih %d c %d "++ "v %d %c %u %c %s\n",++ &inet_sk(sk)->inet_daddr, dport,++ una / 1000, una % 1000, fack - tp->snd_una,++ ca_states[inet_csk(sk)->icsk_ca_state],++ bbr->debug.undo ? '@' : mode[bbr->mode],++ tp->snd_cwnd,++ bbr_extra_acked(sk), /* br (legacy): extra_acked */++ rs->tx_in_flight, /* cr (legacy): tx_inflight */++ rs->rtt_us,++ rs->delivered,++ rs->interval_us,++ bbr->min_rtt_us,++ rs->is_app_limited ? '_' : 'l',++ bbr_rate_kbps(sk, ctx->sample_bw), /* lbw: latest sample bw */++ bbr_rate_kbps(sk, bbr_max_bw(sk)), /* bw: max bw */++ 0ULL, /* lb: [obsolete] */++ 0ULL, /* ib: [obsolete] */++ (u64)sk->sk_pacing_rate * 8 / 1000,++ acked,++ tcp_packets_in_flight(tp),++ rs->is_ack_delayed ? 'd' : '.',++ bbr->round_start ? '*' : '.',++ tp->delivered, tp->lost,++ tp->app_limited,++ 0, /* #: [obsolete] */++ ctx->target_cwnd,++ tp->reord_seen ? 'r' : '.', /* r: reordering seen? */++ ca_states[bbr->prev_ca_state],++ (rs->lost + rs->delivered) > 0 ?++ (1000 * rs->lost /++ (rs->lost + rs->delivered)) : 0, /* lr: loss rate x1000 */++ (rs->delivered) > 0 ?++ (1000 * rs->delivered_ce /++ (rs->delivered)) : 0, /* er: ECN rate x1000 */++ 1000 * bbr->ecn_alpha >> BBR_SCALE, /* ea: ECN alpha x1000 */++ bbr->bw_lo == ~0U ?++ -1 : (s64)bbr_rate_kbps(sk, bbr->bw_lo), /* bwl */++ bbr->inflight_lo, /* il */++ bbr->inflight_hi, /* ih */++ bbr->bw_probe_up_cnt, /* c */++ 2, /* v: version */++ bbr->debug.event,++ bbr->cycle_idx,++ ack_phase[bbr->ack_phase],++ bbr->bw_probe_samples ? "Y" : "N");++ debugmsg[sizeof(debugmsg) - 1] = 0;++++ /* printk takes a higher precedence. */++ if (bbr_debug_with_printk)++ printk(KERN_DEBUG "%s", debugmsg);++++ if (unlikely(bbr->debug.undo))++ bbr->debug.undo = 0;++}++++/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */++static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain)++{++ u64 rate = bw;++++ rate = bbr_rate_bytes_per_sec(sk, rate, gain,++ bbr_pacing_margin_percent);++ rate = min_t(u64, rate, sk->sk_max_pacing_rate);++ return rate;++}++++/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */++static void bbr_init_pacing_rate_from_rtt(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ u64 bw;++ u32 rtt_us;++++ if (tp->srtt_us) { /* any RTT sample yet? */++ rtt_us = max(tp->srtt_us >> 3, 1U);++ bbr->has_seen_rtt = 1;++ } else { /* no RTT sample yet */++ rtt_us = USEC_PER_MSEC; /* use nominal default RTT */++ }++ bw = (u64)tp->snd_cwnd * BW_UNIT;++ do_div(bw, rtt_us);++ sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr->params.high_gain);++}++++/* Pace using current bw estimate and a gain factor. */++static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain);++++ if (unlikely(!bbr->has_seen_rtt && tp->srtt_us))++ bbr_init_pacing_rate_from_rtt(sk);++ if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate)++ sk->sk_pacing_rate = rate;++}++++static u32 bbr_min_tso_segs(struct sock *sk)++{++ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2;++}++++/* Return the number of segments BBR would like in a TSO/GSO skb, given++ * a particular max gso size as a constraint.++ */++static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now,++ u32 gso_max_size)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u32 segs, r;++ u64 bytes;++++ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */++ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift;++++ /* Budget a TSO/GSO burst size allowance based on min_rtt. For every++ * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst.++ * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K)++ */++ if (bbr->params.tso_rtt_shift) {++ r = bbr->min_rtt_us >> bbr->params.tso_rtt_shift;++ if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */++ bytes += GSO_MAX_SIZE >> r;++ }++++ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER);++ segs = max_t(u32, bytes / mss_now, bbr_min_tso_segs(sk));++ return segs;++}++++/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */++static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now)++{++ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size);++}++++/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */++static u32 bbr_tso_segs_goal(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++++ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE);++}++++/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */++static void bbr_save_cwnd(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT)++ bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */++ else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */++ bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd);++}++++static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ if (event == CA_EVENT_TX_START && tp->app_limited) {++ bbr->idle_restart = 1;++ bbr->ack_epoch_mstamp = tp->tcp_mstamp;++ bbr->ack_epoch_acked = 0;++ /* Avoid pointless buffer overflows: pace at est. bw if we don't++ * need more speed (we're restarting from idle and app-limited).++ */++ if (bbr->mode == BBR_PROBE_BW)++ bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT);++ else if (bbr->mode == BBR_PROBE_RTT)++ bbr_check_probe_rtt_done(sk);++ } else if ((event == CA_EVENT_ECN_IS_CE ||++ event == CA_EVENT_ECN_NO_CE) &&++ bbr_ecn_enable &&++ bbr->params.precise_ece_ack) {++ u32 state = bbr->ce_state;++ dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state);++ bbr->ce_state = state;++ if (tp->fast_ack_mode == 2 && event == CA_EVENT_ECN_IS_CE)++ tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);++ }++}++++/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth:++ *++ * bdp = ceil(bw * min_rtt * gain)++ *++ * The key factor, gain, controls the amount of queue. While a small gain++ * builds a smaller queue, it becomes more vulnerable to noise in RTT++ * measurements (e.g., delayed ACKs or other ACK compression effects). This++ * noise may cause BBR to under-estimate the rate.++ */++static u32 bbr_bdp(struct sock *sk, u32 bw, int gain)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u32 bdp;++ u64 w;++++ /* If we've never had a valid RTT sample, cap cwnd at the initial++ * default. This should only happen when the connection is not using TCP++ * timestamps and has retransmitted all of the SYN/SYNACK/data packets++ * ACKed so far. In this case, an RTO can cut cwnd to 1, in which++ * case we need to slow-start up toward something safe: initial cwnd.++ */++ if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */++ return bbr->init_cwnd; /* be safe: cap at initial cwnd */++++ w = (u64)bw * bbr->min_rtt_us;++++ /* Apply a gain to the given value, remove the BW_SCALE shift, and++ * round the value up to avoid a negative feedback loop.++ */++ bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT;++++ return bdp;++}++++/* To achieve full performance in high-speed paths, we budget enough cwnd to++ * fit full-sized skbs in-flight on both end hosts to fully utilize the path:++ * - one skb in sending host Qdisc,++ * - one skb in sending host TSO/GSO engine++ * - one skb being received by receiver host LRO/GRO/delayed-ACK engine++ * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because++ * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets,++ * which allows 2 outstanding 2-packet sequences, to try to keep pipe++ * full even with ACK-every-other-packet delayed ACKs.++ */++static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u32 tso_segs_goal;++++ tso_segs_goal = 3 * bbr_tso_segs_goal(sk);++++ /* Allow enough full-sized skbs in flight to utilize end systems. */++ if (bbr->params.cwnd_tso_budget == 1) {++ cwnd = max_t(u32, cwnd, tso_segs_goal);++ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target);++ } else {++ cwnd += tso_segs_goal;++ cwnd = (cwnd + 1) & ~1U;++ }++ /* Ensure gain cycling gets inflight above BDP even for small BDPs. */++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP)++ cwnd += 2;++++ return cwnd;++}++++/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */++static u32 bbr_inflight(struct sock *sk, u32 bw, int gain)++{++ u32 inflight;++++ inflight = bbr_bdp(sk, bw, gain);++ inflight = bbr_quantization_budget(sk, inflight);++++ return inflight;++}++++/* With pacing at lower layers, there's often less data "in the network" than++ * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq),++ * we often have several skbs queued in the pacing layer with a pre-scheduled++ * earliest departure time (EDT). BBR adapts its pacing rate based on the++ * inflight level that it estimates has already been "baked in" by previous++ * departure time decisions. We calculate a rough estimate of the number of our++ * packets that might be in the network at the earliest departure time for the++ * next skb scheduled:++ * in_network_at_edt = inflight_at_edt - (EDT - now) * bw++ * If we're increasing inflight, then we want to know if the transmit of the++ * EDT skb will push inflight above the target, so inflight_at_edt includes++ * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight,++ * then estimate if inflight will sink too low just before the EDT transmit.++ */++static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ u64 now_ns, edt_ns, interval_us;++ u32 interval_delivered, inflight_at_edt;++++ now_ns = tp->tcp_clock_cache;++ edt_ns = max(tp->tcp_wstamp_ns, now_ns);++ interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC);++ interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE;++ inflight_at_edt = inflight_now;++ if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */++ inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */++ if (interval_delivered >= inflight_at_edt)++ return 0;++ return inflight_at_edt - interval_delivered;++}++++/* Find the cwnd increment based on estimate of ack aggregation */++static u32 bbr_ack_aggregation_cwnd(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u32 max_aggr_cwnd, aggr_cwnd = 0;++++ if (bbr->params.extra_acked_gain &&++ (bbr_full_bw_reached(sk) || bbr->params.extra_acked_in_startup)) {++ max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us)++ / BW_UNIT;++ aggr_cwnd = (bbr->params.extra_acked_gain * bbr_extra_acked(sk))++ >> BBR_SCALE;++ aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd);++ }++++ return aggr_cwnd;++}++++/* Returns the cwnd for PROBE_RTT mode. */++static u32 bbr_probe_rtt_cwnd(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ if (bbr->params.probe_rtt_cwnd_gain == 0)++ return bbr->params.cwnd_min_target;++ return max_t(u32, bbr->params.cwnd_min_target,++ bbr_bdp(sk, bbr_bw(sk), bbr->params.probe_rtt_cwnd_gain));++}++++/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss++ * has drawn us down below target), or snap down to target if we're above it.++ */++static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs,++ u32 acked, u32 bw, int gain, u32 cwnd,++ struct bbr_context *ctx)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ u32 target_cwnd = 0, prev_cwnd = tp->snd_cwnd, max_probe;++++ if (!acked)++ goto done; /* no packet fully ACKed; just apply caps */++++ target_cwnd = bbr_bdp(sk, bw, gain);++++ /* Increment the cwnd to account for excess ACKed data that seems++ * due to aggregation (of data and/or ACKs) visible in the ACK stream.++ */++ target_cwnd += bbr_ack_aggregation_cwnd(sk);++ target_cwnd = bbr_quantization_budget(sk, target_cwnd);++++ /* If we're below target cwnd, slow start cwnd toward target cwnd. */++ bbr->debug.target_cwnd = target_cwnd;++++ /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */++ bbr->try_fast_path = 0;++ if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */++ cwnd += acked;++ if (cwnd >= target_cwnd) {++ cwnd = target_cwnd;++ bbr->try_fast_path = 1;++ }++ } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) {++ cwnd += acked;++ } else {++ bbr->try_fast_path = 1;++ }++++ /* When growing cwnd, don't grow beyond twice what we just probed. */++ if (bbr->params.usage_based_cwnd) {++ max_probe = max(2 * tp->max_packets_out, tp->snd_cwnd);++ cwnd = min(cwnd, max_probe);++ }++++ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target);++done:++ tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */++ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */++ tp->snd_cwnd = min_t(u32, tp->snd_cwnd, bbr_probe_rtt_cwnd(sk));++++ ctx->target_cwnd = target_cwnd;++ ctx->log = (tp->snd_cwnd != prev_cwnd);++}++++/* See if we have reached next round trip */++static void bbr_update_round_start(struct sock *sk,++ const struct rate_sample *rs, struct bbr_context *ctx)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr->round_start = 0;++++ /* See if we've reached the next RTT */++ if (rs->interval_us > 0 &&++ !before(rs->prior_delivered, bbr->next_rtt_delivered)) {++ bbr->next_rtt_delivered = tp->delivered;++ bbr->round_start = 1;++ }++}++++/* Calculate the bandwidth based on how fast packets are delivered */++static void bbr_calculate_bw_sample(struct sock *sk,++ const struct rate_sample *rs, struct bbr_context *ctx)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u64 bw = 0;++++ /* Divide delivered by the interval to find a (lower bound) bottleneck++ * bandwidth sample. Delivered is in packets and interval_us in uS and++ * ratio will be <<1 for most connections. So delivered is first scaled.++ * Round up to allow growth at low rates, even with integer division.++ */++ if (rs->interval_us > 0) {++ if (WARN_ONCE(rs->delivered < 0,++ "negative delivered: %d interval_us: %ld\n",++ rs->delivered, rs->interval_us))++ return;++++ bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us);++ }++++ ctx->sample_bw = bw;++ bbr->debug.rs_bw = bw;++}++++/* Estimates the windowed max degree of ack aggregation.++ * This is used to provision extra in-flight data to keep sending during++ * inter-ACK silences.++ *++ * Degree of ack aggregation is estimated as extra data acked beyond expected.++ *++ * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval"++ * cwnd += max_extra_acked++ *++ * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms).++ * Max filter is an approximate sliding window of 5-10 (packet timed) round++ * trips for non-startup phase, and 1-2 round trips for startup.++ */++static void bbr_update_ack_aggregation(struct sock *sk,++ const struct rate_sample *rs)++{++ u32 epoch_us, expected_acked, extra_acked;++ struct bbr *bbr = inet_csk_ca(sk);++ struct tcp_sock *tp = tcp_sk(sk);++ u32 extra_acked_win_rtts_thresh = bbr->params.extra_acked_win_rtts;++++ if (!bbr->params.extra_acked_gain || rs->acked_sacked <= 0 ||++ rs->delivered < 0 || rs->interval_us <= 0)++ return;++++ if (bbr->round_start) {++ bbr->extra_acked_win_rtts = min(0x1F,++ bbr->extra_acked_win_rtts + 1);++ if (bbr->params.extra_acked_in_startup &&++ !bbr_full_bw_reached(sk))++ extra_acked_win_rtts_thresh = 1;++ if (bbr->extra_acked_win_rtts >=++ extra_acked_win_rtts_thresh) {++ bbr->extra_acked_win_rtts = 0;++ bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ?++ 0 : 1;++ bbr->extra_acked[bbr->extra_acked_win_idx] = 0;++ }++ }++++ /* Compute how many packets we expected to be delivered over epoch. */++ epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp,++ bbr->ack_epoch_mstamp);++ expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT;++++ /* Reset the aggregation epoch if ACK rate is below expected rate or++ * significantly large no. of ack received since epoch (potentially++ * quite old epoch).++ */++ if (bbr->ack_epoch_acked <= expected_acked ||++ (bbr->ack_epoch_acked + rs->acked_sacked >=++ bbr_ack_epoch_acked_reset_thresh)) {++ bbr->ack_epoch_acked = 0;++ bbr->ack_epoch_mstamp = tp->delivered_mstamp;++ expected_acked = 0;++ }++++ /* Compute excess data delivered, beyond what was expected. */++ bbr->ack_epoch_acked = min_t(u32, 0xFFFFF,++ bbr->ack_epoch_acked + rs->acked_sacked);++ extra_acked = bbr->ack_epoch_acked - expected_acked;++ extra_acked = min(extra_acked, tp->snd_cwnd);++ if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx])++ bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked;++}++++/* Estimate when the pipe is full, using the change in delivery rate: BBR++ * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by++ * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited++ * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the++ * higher rwin, 3: we get higher delivery rate samples. Or transient++ * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar++ * design goal, but uses delay and inter-ACK spacing instead of bandwidth.++ */++static void bbr_check_full_bw_reached(struct sock *sk,++ const struct rate_sample *rs)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u32 bw_thresh;++++ if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited)++ return;++++ bw_thresh = (u64)bbr->full_bw * bbr->params.full_bw_thresh >> BBR_SCALE;++ if (bbr_max_bw(sk) >= bw_thresh) {++ bbr->full_bw = bbr_max_bw(sk);++ bbr->full_bw_cnt = 0;++ return;++ }++ ++bbr->full_bw_cnt;++ bbr->full_bw_reached = bbr->full_bw_cnt >= bbr->params.full_bw_cnt;++}++++/* If pipe is probably full, drain the queue and then enter steady-state. */++static bool bbr_check_drain(struct sock *sk, const struct rate_sample *rs,++ struct bbr_context *ctx)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) {++ bbr->mode = BBR_DRAIN; /* drain queue we created */++ tcp_sk(sk)->snd_ssthresh =++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT);++ bbr2_reset_congestion_signals(sk);++ } /* fall through to check if in-flight is already small: */++ if (bbr->mode == BBR_DRAIN &&++ bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <=++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT))++ return true; /* exiting DRAIN now */++ return false;++}++++static void bbr_check_probe_rtt_done(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ if (!(bbr->probe_rtt_done_stamp &&++ after(tcp_jiffies32, bbr->probe_rtt_done_stamp)))++ return;++++ bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd);++ bbr2_exit_probe_rtt(sk);++}++++/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and++ * periodically drain the bottleneck queue, to converge to measure the true++ * min_rtt (unloaded propagation delay). This allows the flows to keep queues++ * small (reducing queuing delay and packet loss) and achieve fairness among++ * BBR flows.++ *++ * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires,++ * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets.++ * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed++ * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and++ * re-enter the previous mode. BBR uses 200ms to approximately bound the++ * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s).++ *++ * Note that flows need only pay 2% if they are busy sending over the last 10++ * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have++ * natural silences or low-rate periods within 10 seconds where the rate is low++ * enough for long enough to drain its queue in the bottleneck. We pick up++ * these min RTT measurements opportunistically with our min_rtt filter. :-)++ */++static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ bool probe_rtt_expired, min_rtt_expired;++ u32 expire;++++ /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */++ expire = bbr->probe_rtt_min_stamp +++ msecs_to_jiffies(bbr->params.probe_rtt_win_ms);++ probe_rtt_expired = after(tcp_jiffies32, expire);++ if (rs->rtt_us >= 0 &&++ (rs->rtt_us <= bbr->probe_rtt_min_us ||++ (probe_rtt_expired && !rs->is_ack_delayed))) {++ bbr->probe_rtt_min_us = rs->rtt_us;++ bbr->probe_rtt_min_stamp = tcp_jiffies32;++ }++ /* Track min RTT seen in the min_rtt_win_sec filter window: */++ expire = bbr->min_rtt_stamp + bbr->params.min_rtt_win_sec * HZ;++ min_rtt_expired = after(tcp_jiffies32, expire);++ if (bbr->probe_rtt_min_us <= bbr->min_rtt_us ||++ min_rtt_expired) {++ bbr->min_rtt_us = bbr->probe_rtt_min_us;++ bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp;++ }++++ if (bbr->params.probe_rtt_mode_ms > 0 && probe_rtt_expired &&++ !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) {++ bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */++ bbr_save_cwnd(sk); /* note cwnd so we can restore it */++ bbr->probe_rtt_done_stamp = 0;++ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING;++ bbr->next_rtt_delivered = tp->delivered;++ }++++ if (bbr->mode == BBR_PROBE_RTT) {++ /* Ignore low rate samples during this mode. */++ tp->app_limited =++ (tp->delivered + tcp_packets_in_flight(tp)) ? : 1;++ /* Maintain min packets in flight for max(200 ms, 1 round). */++ if (!bbr->probe_rtt_done_stamp &&++ tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) {++ bbr->probe_rtt_done_stamp = tcp_jiffies32 +++ msecs_to_jiffies(bbr->params.probe_rtt_mode_ms);++ bbr->probe_rtt_round_done = 0;++ bbr->next_rtt_delivered = tp->delivered;++ } else if (bbr->probe_rtt_done_stamp) {++ if (bbr->round_start)++ bbr->probe_rtt_round_done = 1;++ if (bbr->probe_rtt_round_done)++ bbr_check_probe_rtt_done(sk);++ }++ }++ /* Restart after idle ends only once we process a new S/ACK for data */++ if (rs->delivered > 0)++ bbr->idle_restart = 0;++}++++static void bbr_update_gains(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ switch (bbr->mode) {++ case BBR_STARTUP:++ bbr->pacing_gain = bbr->params.high_gain;++ bbr->cwnd_gain = bbr->params.startup_cwnd_gain;++ break;++ case BBR_DRAIN:++ bbr->pacing_gain = bbr->params.drain_gain; /* slow, to drain */++ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; /* keep cwnd */++ break;++ case BBR_PROBE_BW:++ bbr->pacing_gain = bbr->params.pacing_gain[bbr->cycle_idx];++ bbr->cwnd_gain = bbr->params.cwnd_gain;++ break;++ case BBR_PROBE_RTT:++ bbr->pacing_gain = BBR_UNIT;++ bbr->cwnd_gain = BBR_UNIT;++ break;++ default:++ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode);++ break;++ }++}++++static void bbr_init(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ int i;++++ WARN_ON_ONCE(tp->snd_cwnd >= bbr_cwnd_warn_val);++++ bbr->initialized = 1;++ bbr->params.high_gain = min(0x7FF, bbr_high_gain);++ bbr->params.drain_gain = min(0x3FF, bbr_drain_gain);++ bbr->params.startup_cwnd_gain = min(0x7FF, bbr_startup_cwnd_gain);++ bbr->params.cwnd_gain = min(0x7FF, bbr_cwnd_gain);++ bbr->params.cwnd_tso_budget = min(0x1U, bbr_cwnd_tso_budget);++ bbr->params.cwnd_min_target = min(0xFU, bbr_cwnd_min_target);++ bbr->params.min_rtt_win_sec = min(0x1FU, bbr_min_rtt_win_sec);++ bbr->params.probe_rtt_mode_ms = min(0x1FFU, bbr_probe_rtt_mode_ms);++ bbr->params.full_bw_cnt = min(0x7U, bbr_full_bw_cnt);++ bbr->params.full_bw_thresh = min(0x3FFU, bbr_full_bw_thresh);++ bbr->params.extra_acked_gain = min(0x7FF, bbr_extra_acked_gain);++ bbr->params.extra_acked_win_rtts = min(0x1FU, bbr_extra_acked_win_rtts);++ bbr->params.drain_to_target = bbr_drain_to_target ? 1 : 0;++ bbr->params.precise_ece_ack = bbr_precise_ece_ack ? 1 : 0;++ bbr->params.extra_acked_in_startup = bbr_extra_acked_in_startup ? 1 : 0;++ bbr->params.probe_rtt_cwnd_gain = min(0xFFU, bbr_probe_rtt_cwnd_gain);++ bbr->params.probe_rtt_win_ms =++ min(0x3FFFU,++ min_t(u32, bbr_probe_rtt_win_ms,++ bbr->params.min_rtt_win_sec * MSEC_PER_SEC));++ for (i = 0; i < CYCLE_LEN; i++)++ bbr->params.pacing_gain[i] = min(0x3FF, bbr_pacing_gain[i]);++ bbr->params.usage_based_cwnd = bbr_usage_based_cwnd ? 1 : 0;++ bbr->params.tso_rtt_shift = min(0xFU, bbr_tso_rtt_shift);++++ bbr->debug.snd_isn = tp->snd_una;++ bbr->debug.target_cwnd = 0;++ bbr->debug.undo = 0;++++ bbr->init_cwnd = min(0x7FU, tp->snd_cwnd);++ bbr->prior_cwnd = tp->prior_cwnd;++ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;++ bbr->next_rtt_delivered = 0;++ bbr->prev_ca_state = TCP_CA_Open;++ bbr->packet_conservation = 0;++++ bbr->probe_rtt_done_stamp = 0;++ bbr->probe_rtt_round_done = 0;++ bbr->probe_rtt_min_us = tcp_min_rtt(tp);++ bbr->probe_rtt_min_stamp = tcp_jiffies32;++ bbr->min_rtt_us = tcp_min_rtt(tp);++ bbr->min_rtt_stamp = tcp_jiffies32;++++ bbr->has_seen_rtt = 0;++ bbr_init_pacing_rate_from_rtt(sk);++++ bbr->round_start = 0;++ bbr->idle_restart = 0;++ bbr->full_bw_reached = 0;++ bbr->full_bw = 0;++ bbr->full_bw_cnt = 0;++ bbr->cycle_mstamp = 0;++ bbr->cycle_idx = 0;++ bbr->mode = BBR_STARTUP;++ bbr->debug.rs_bw = 0;++++ bbr->ack_epoch_mstamp = tp->tcp_mstamp;++ bbr->ack_epoch_acked = 0;++ bbr->extra_acked_win_rtts = 0;++ bbr->extra_acked_win_idx = 0;++ bbr->extra_acked[0] = 0;++ bbr->extra_acked[1] = 0;++++ bbr->ce_state = 0;++ bbr->prior_rcv_nxt = tp->rcv_nxt;++ bbr->try_fast_path = 0;++++ cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED);++}++++static u32 bbr_sndbuf_expand(struct sock *sk)++{++ /* Provision 3 * cwnd since BBR may slow-start even during recovery. */++ return 3;++}++++/* __________________________________________________________________________++ *++ * Functions new to BBR v2 ("bbr") congestion control are below here.++ * __________________________________________________________________________++ */++++/* Incorporate a new bw sample into the current window of our max filter. */++static void bbr2_take_bw_hi_sample(struct sock *sk, u32 bw)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]);++}++++/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */++static void bbr2_advance_bw_hi_filter(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ if (!bbr->bw_hi[1])++ return; /* no samples in this window; remember old window */++ bbr->bw_hi[0] = bbr->bw_hi[1];++ bbr->bw_hi[1] = 0;++}++++/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */++static u32 bbr2_target_inflight(struct sock *sk)++{++ u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT);++++ return min(bdp, tcp_sk(sk)->snd_cwnd);++}++++static bool bbr2_is_probing_bandwidth(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ return (bbr->mode == BBR_STARTUP) ||++ (bbr->mode == BBR_PROBE_BW &&++ (bbr->cycle_idx == BBR_BW_PROBE_REFILL ||++ bbr->cycle_idx == BBR_BW_PROBE_UP));++}++++/* Has the given amount of time elapsed since we marked the phase start? */++static bool bbr2_has_elapsed_in_phase(const struct sock *sk, u32 interval_us)++{++ const struct tcp_sock *tp = tcp_sk(sk);++ const struct bbr *bbr = inet_csk_ca(sk);++++ return tcp_stamp_us_delta(tp->tcp_mstamp,++ bbr->cycle_mstamp + interval_us) > 0;++}++++static void bbr2_handle_queue_too_high_in_startup(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr->full_bw_reached = 1;++ bbr->inflight_hi = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT);++}++++/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */++static void bbr2_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible ||++ !bbr->params.full_ecn_cnt || !bbr->params.ecn_thresh)++ return;++++ if (ce_ratio >= bbr->params.ecn_thresh)++ bbr->startup_ecn_rounds++;++ else++ bbr->startup_ecn_rounds = 0;++++ if (bbr->startup_ecn_rounds >= bbr->params.full_ecn_cnt) {++ bbr->debug.event = 'E'; /* ECN caused STARTUP exit */++ bbr2_handle_queue_too_high_in_startup(sk);++ return;++ }++}++++static void bbr2_update_ecn_alpha(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ s32 delivered, delivered_ce;++ u64 alpha, ce_ratio;++ u32 gain;++++ if (bbr->params.ecn_factor == 0)++ return;++++ delivered = tp->delivered - bbr->alpha_last_delivered;++ delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce;++++ if (delivered == 0 || /* avoid divide by zero */++ WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */++ return;++++ /* See if we should use ECN sender logic for this connection. */++ if (!bbr->ecn_eligible && bbr_ecn_enable &&++ (bbr->min_rtt_us <= bbr->params.ecn_max_rtt_us ||++ !bbr->params.ecn_max_rtt_us))++ bbr->ecn_eligible = 1;++++ ce_ratio = (u64)delivered_ce << BBR_SCALE;++ do_div(ce_ratio, delivered);++ gain = bbr->params.ecn_alpha_gain;++ alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE;++ alpha += (gain * ce_ratio) >> BBR_SCALE;++ bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT);++++ bbr->alpha_last_delivered = tp->delivered;++ bbr->alpha_last_delivered_ce = tp->delivered_ce;++++ bbr2_check_ecn_too_high_in_startup(sk, ce_ratio);++}++++/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */++static void bbr2_raise_inflight_hi_slope(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ u32 growth_this_round, cnt;++++ /* Calculate "slope": packets S/Acked per inflight_hi increment. */++ growth_this_round = 1 << bbr->bw_probe_up_rounds;++ bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30);++ cnt = tp->snd_cwnd / growth_this_round;++ cnt = max(cnt, 1U);++ bbr->bw_probe_up_cnt = cnt;++ bbr->debug.event = 'G'; /* Grow inflight_hi slope */++}++++/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */++static void bbr2_probe_inflight_hi_upward(struct sock *sk,++ const struct rate_sample *rs)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ u32 delta;++++ if (!tp->is_cwnd_limited || tp->snd_cwnd < bbr->inflight_hi) {++ bbr->bw_probe_up_acks = 0; /* don't accmulate unused credits */++ return; /* not fully using inflight_hi, so don't grow it */++ }++++ /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */++ bbr->bw_probe_up_acks += rs->acked_sacked;++ if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) {++ delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt;++ bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt;++ bbr->inflight_hi += delta;++ bbr->debug.event = 'I'; /* Increment inflight_hi */++ }++++ if (bbr->round_start)++ bbr2_raise_inflight_hi_slope(sk);++}++++/* Does loss/ECN rate for this sample say inflight is "too high"?++ * This is used by both the bbr_check_loss_too_high_in_startup() function,++ * which can be used in either v1 or v2, and the PROBE_UP phase of v2, which++ * uses it to notice when loss/ECN rates suggest inflight is too high.++ */++static bool bbr2_is_inflight_too_high(const struct sock *sk,++ const struct rate_sample *rs)++{++ const struct bbr *bbr = inet_csk_ca(sk);++ u32 loss_thresh, ecn_thresh;++++ if (rs->lost > 0 && rs->tx_in_flight) {++ loss_thresh = (u64)rs->tx_in_flight * bbr->params.loss_thresh >>++ BBR_SCALE;++ if (rs->lost > loss_thresh)++ return true;++ }++++ if (rs->delivered_ce > 0 && rs->delivered > 0 &&++ bbr->ecn_eligible && bbr->params.ecn_thresh) {++ ecn_thresh = (u64)rs->delivered * bbr->params.ecn_thresh >>++ BBR_SCALE;++ if (rs->delivered_ce >= ecn_thresh)++ return true;++ }++++ return false;++}++++/* Calculate the tx_in_flight level that corresponded to excessive loss.++ * We find "lost_prefix" segs of the skb where loss rate went too high,++ * by solving for "lost_prefix" in the following equation:++ * lost / inflight >= loss_thresh++ * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh++ * Then we take that equation, convert it to fixed point, and++ * round up to the nearest packet.++ */++static u32 bbr2_inflight_hi_from_lost_skb(const struct sock *sk,++ const struct rate_sample *rs,++ const struct sk_buff *skb)++{++ const struct bbr *bbr = inet_csk_ca(sk);++ u32 loss_thresh = bbr->params.loss_thresh;++ u32 pcount, divisor, inflight_hi;++ s32 inflight_prev, lost_prev;++ u64 loss_budget, lost_prefix;++++ pcount = tcp_skb_pcount(skb);++++ /* How much data was in flight before this skb? */++ inflight_prev = rs->tx_in_flight - pcount;++ if (WARN_ONCE(inflight_prev < 0,++ "tx_in_flight: %u pcount: %u reneg: %u",++ rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg))++ return ~0U;++++ /* How much inflight data was marked lost before this skb? */++ lost_prev = rs->lost - pcount;++ if (WARN_ON_ONCE(lost_prev < 0))++ return ~0U;++++ /* At what prefix of this lost skb did losss rate exceed loss_thresh? */++ loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1;++ loss_budget >>= BBR_SCALE;++ if (lost_prev >= loss_budget) {++ lost_prefix = 0; /* previous losses crossed loss_thresh */++ } else {++ lost_prefix = loss_budget - lost_prev;++ lost_prefix <<= BBR_SCALE;++ divisor = BBR_UNIT - loss_thresh;++ if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */++ return ~0U;++ do_div(lost_prefix, divisor);++ }++++ inflight_hi = inflight_prev + lost_prefix;++ return inflight_hi;++}++++/* If loss/ECN rates during probing indicated we may have overfilled a++ * buffer, return an operating point that tries to leave unutilized headroom in++ * the path for other flows, for fairness convergence and lower RTTs and loss.++ */++static u32 bbr2_inflight_with_headroom(const struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u32 headroom, headroom_fraction;++++ if (bbr->inflight_hi == ~0U)++ return ~0U;++++ headroom_fraction = bbr->params.inflight_headroom;++ headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE;++ headroom = max(headroom, 1U);++ return max_t(s32, bbr->inflight_hi - headroom,++ bbr->params.cwnd_min_target);++}++++/* Bound cwnd to a sensible level, based on our current probing state++ * machine phase and model of a good inflight level (inflight_lo, inflight_hi).++ */++static void bbr2_bound_cwnd_for_inflight_model(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ u32 cap;++++ /* tcp_rcv_synsent_state_process() currently calls tcp_ack()++ * and thus cong_control() without first initializing us(!).++ */++ if (!bbr->initialized)++ return;++++ cap = ~0U;++ if (bbr->mode == BBR_PROBE_BW &&++ bbr->cycle_idx != BBR_BW_PROBE_CRUISE) {++ /* Probe to see if more packets fit in the path. */++ cap = bbr->inflight_hi;++ } else {++ if (bbr->mode == BBR_PROBE_RTT ||++ (bbr->mode == BBR_PROBE_BW &&++ bbr->cycle_idx == BBR_BW_PROBE_CRUISE))++ cap = bbr2_inflight_with_headroom(sk);++ }++ /* Adapt to any loss/ECN since our last bw probe. */++ cap = min(cap, bbr->inflight_lo);++++ cap = max_t(u32, cap, bbr->params.cwnd_min_target);++ tp->snd_cwnd = min(cap, tp->snd_cwnd);++}++++/* Estimate a short-term lower bound on the capacity available now, based++ * on measurements of the current delivery process and recent history. When we++ * are seeing loss/ECN at times when we are not probing bw, then conservatively++ * move toward flow balance by multiplicatively cutting our short-term++ * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a++ * multiplicative decrease in order to converge to a lower capacity in time++ * logarithmic in the magnitude of the decrease.++ *++ * However, we do not cut our short-term estimates lower than the current rate++ * and volume of delivered data from this round trip, since from the current++ * delivery process we can estimate the measured capacity available now.++ *++ * Anything faster than that approach would knowingly risk high loss, which can++ * cause low bw for Reno/CUBIC and high loss recovery latency for++ * request/response flows using any congestion control.++ */++static void bbr2_adapt_lower_bounds(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ u32 ecn_cut, ecn_inflight_lo, beta;++++ /* We only use lower-bound estimates when not probing bw.++ * When probing we need to push inflight higher to probe bw.++ */++ if (bbr2_is_probing_bandwidth(sk))++ return;++++ /* ECN response. */++ if (bbr->ecn_in_round && bbr->ecn_eligible && bbr->params.ecn_factor) {++ /* Reduce inflight to (1 - alpha*ecn_factor). */++ ecn_cut = (BBR_UNIT -++ ((bbr->ecn_alpha * bbr->params.ecn_factor) >>++ BBR_SCALE));++ if (bbr->inflight_lo == ~0U)++ bbr->inflight_lo = tp->snd_cwnd;++ ecn_inflight_lo = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE;++ } else {++ ecn_inflight_lo = ~0U;++ }++++ /* Loss response. */++ if (bbr->loss_in_round) {++ /* Reduce bw and inflight to (1 - beta). */++ if (bbr->bw_lo == ~0U)++ bbr->bw_lo = bbr_max_bw(sk);++ if (bbr->inflight_lo == ~0U)++ bbr->inflight_lo = tp->snd_cwnd;++ beta = bbr->params.beta;++ bbr->bw_lo =++ max_t(u32, bbr->bw_latest,++ (u64)bbr->bw_lo *++ (BBR_UNIT - beta) >> BBR_SCALE);++ bbr->inflight_lo =++ max_t(u32, bbr->inflight_latest,++ (u64)bbr->inflight_lo *++ (BBR_UNIT - beta) >> BBR_SCALE);++ }++++ /* Adjust to the lower of the levels implied by loss or ECN. */++ bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo);++}++++/* Reset any short-term lower-bound adaptation to congestion, so that we can++ * push our inflight up.++ */++static void bbr2_reset_lower_bounds(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr->bw_lo = ~0U;++ bbr->inflight_lo = ~0U;++}++++/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state++ * machine phase where we adapt our lower bound based on congestion signals.++ */++static void bbr2_reset_congestion_signals(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr->loss_in_round = 0;++ bbr->ecn_in_round = 0;++ bbr->loss_in_cycle = 0;++ bbr->ecn_in_cycle = 0;++ bbr->bw_latest = 0;++ bbr->inflight_latest = 0;++}++++/* Update (most of) our congestion signals: track the recent rate and volume of++ * delivered data, presence of loss, and EWMA degree of ECN marking.++ */++static void bbr2_update_congestion_signals(++ struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ u64 bw;++++ bbr->loss_round_start = 0;++ if (rs->interval_us <= 0 || !rs->acked_sacked)++ return; /* Not a valid observation */++ bw = ctx->sample_bw;++++ if (!rs->is_app_limited || bw >= bbr_max_bw(sk))++ bbr2_take_bw_hi_sample(sk, bw);++++ bbr->loss_in_round |= (rs->losses > 0);++++ /* Update rate and volume of delivered data from latest round trip: */++ bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw);++ bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered);++++ if (before(rs->prior_delivered, bbr->loss_round_delivered))++ return; /* skip the per-round-trip updates */++ /* Now do per-round-trip updates. */++ bbr->loss_round_delivered = tp->delivered; /* mark round trip */++ bbr->loss_round_start = 1;++ bbr2_adapt_lower_bounds(sk);++++ /* Update windowed "latest" (single-round-trip) filters. */++ bbr->loss_in_round = 0;++ bbr->ecn_in_round = 0;++ bbr->bw_latest = ctx->sample_bw;++ bbr->inflight_latest = rs->delivered;++}++++/* Bandwidth probing can cause loss. To help coexistence with loss-based++ * congestion control we spread out our probing in a Reno-conscious way. Due to++ * the shape of the Reno sawtooth, the time required between loss epochs for an++ * idealized Reno flow is a number of round trips that is the BDP of that++ * flow. We count packet-timed round trips directly, since measured RTT can++ * vary widely, and Reno is driven by packet-timed round trips.++ */++static bool bbr2_is_reno_coexistence_probe_time(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u32 inflight, rounds, reno_gain, reno_rounds;++++ /* Random loss can shave some small percentage off of our inflight++ * in each round. To survive this, flows need robust periodic probes.++ */++ rounds = bbr->params.bw_probe_max_rounds;++++ reno_gain = bbr->params.bw_probe_reno_gain;++ if (reno_gain) {++ inflight = bbr2_target_inflight(sk);++ reno_rounds = ((u64)inflight * reno_gain) >> BBR_SCALE;++ rounds = min(rounds, reno_rounds);++ }++ return bbr->rounds_since_probe >= rounds;++}++++/* How long do we want to wait before probing for bandwidth (and risking++ * loss)? We randomize the wait, for better mixing and fairness convergence.++ *++ * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips.++ * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow,++ * (eg 4K video to a broadband user):++ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets++ *++ * We bound the BBR-native inter-bw-probe wall clock time to be:++ * (a) higher than 2 sec: to try to avoid causing loss for a long enough time++ * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must++ * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs++ * (b) lower than 3 sec: to ensure flows can start probing in a reasonable++ * amount of time to discover unutilized bw on human-scale interactive++ * time-scales (e.g. perhaps traffic from a web page download that we++ * were competing with is now complete).++ */++static void bbr2_pick_probe_wait(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ /* Decide the random round-trip bound for wait until probe: */++ bbr->rounds_since_probe =++ prandom_u32_max(bbr->params.bw_probe_rand_rounds);++ /* Decide the random wall clock bound for wait until probe: */++ bbr->probe_wait_us = bbr->params.bw_probe_base_us +++ prandom_u32_max(bbr->params.bw_probe_rand_us);++}++++static void bbr2_set_cycle_idx(struct sock *sk, int cycle_idx)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr->cycle_idx = cycle_idx;++ /* New phase, so need to update cwnd and pacing rate. */++ bbr->try_fast_path = 0;++}++++/* Send at estimated bw to fill the pipe, but not queue. We need this phase++ * before PROBE_UP, because as soon as we send faster than the available bw++ * we will start building a queue, and if the buffer is shallow we can cause++ * loss. If we do not fill the pipe before we cause this loss, our bw_hi and++ * inflight_hi estimates will underestimate.++ */++static void bbr2_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr2_reset_lower_bounds(sk);++ if (bbr->inflight_hi != ~0U)++ bbr->inflight_hi += bbr->params.refill_add_inc;++ bbr->bw_probe_up_rounds = bw_probe_up_rounds;++ bbr->bw_probe_up_acks = 0;++ bbr->stopped_risky_probe = 0;++ bbr->ack_phase = BBR_ACKS_REFILLING;++ bbr->next_rtt_delivered = tp->delivered;++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_REFILL);++}++++/* Now probe max deliverable data rate and volume. */++static void bbr2_start_bw_probe_up(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr->ack_phase = BBR_ACKS_PROBE_STARTING;++ bbr->next_rtt_delivered = tp->delivered;++ bbr->cycle_mstamp = tp->tcp_mstamp;++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_UP);++ bbr2_raise_inflight_hi_slope(sk);++}++++/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall++ * clock time at which to probe beyond an inflight that we think to be++ * safe. This will knowingly risk packet loss, so we want to do this rarely, to++ * keep packet loss rates low. Also start a round-trip counter, to probe faster++ * if we estimate a Reno flow at our BDP would probe faster.++ */++static void bbr2_start_bw_probe_down(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr2_reset_congestion_signals(sk);++ bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */++ bbr2_pick_probe_wait(sk);++ bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */++ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING;++ bbr->next_rtt_delivered = tp->delivered;++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_DOWN);++}++++/* Cruise: maintain what we estimate to be a neutral, conservative++ * operating point, without attempting to probe up for bandwidth or down for++ * RTT, and only reducing inflight in response to loss/ECN signals.++ */++static void bbr2_start_bw_probe_cruise(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ if (bbr->inflight_lo != ~0U)++ bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi);++++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE);++}++++/* Loss and/or ECN rate is too high while probing.++ * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle.++ */++static void bbr2_handle_inflight_too_high(struct sock *sk,++ const struct rate_sample *rs)++{++ struct bbr *bbr = inet_csk_ca(sk);++ const u32 beta = bbr->params.beta;++++ bbr->prev_probe_too_high = 1;++ bbr->bw_probe_samples = 0; /* only react once per probe */++ bbr->debug.event = 'L'; /* Loss/ECN too high */++ /* If we are app-limited then we are not robustly++ * probing the max volume of inflight data we think++ * might be safe (analogous to how app-limited bw++ * samples are not known to be robustly probing bw).++ */++ if (!rs->is_app_limited)++ bbr->inflight_hi = max_t(u32, rs->tx_in_flight,++ (u64)bbr2_target_inflight(sk) *++ (BBR_UNIT - beta) >> BBR_SCALE);++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP)++ bbr2_start_bw_probe_down(sk);++}++++/* If we're seeing bw and loss samples reflecting our bw probing, adapt++ * using the signals we see. If loss or ECN mark rate gets too high, then adapt++ * inflight_hi downward. If we're able to push inflight higher without such++ * signals, push higher: adapt inflight_hi upward.++ */++static bool bbr2_adapt_upper_bounds(struct sock *sk,++ const struct rate_sample *rs)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ /* Track when we'll see bw/loss samples resulting from our bw probes. */++ if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start)++ bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK;++ if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) {++ /* End of samples from bw probing phase. */++ bbr->bw_probe_samples = 0;++ bbr->ack_phase = BBR_ACKS_INIT;++ /* At this point in the cycle, our current bw sample is also++ * our best recent chance at finding the highest available bw++ * for this flow. So now is the best time to forget the bw++ * samples from the previous cycle, by advancing the window.++ */++ if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited)++ bbr2_advance_bw_hi_filter(sk);++ /* If we had an inflight_hi, then probed and pushed inflight all++ * the way up to hit that inflight_hi without seeing any++ * high loss/ECN in all the resulting ACKs from that probing,++ * then probe up again, this time letting inflight persist at++ * inflight_hi for a round trip, then accelerating beyond.++ */++ if (bbr->mode == BBR_PROBE_BW &&++ bbr->stopped_risky_probe && !bbr->prev_probe_too_high) {++ bbr->debug.event = 'R'; /* reprobe */++ bbr2_start_bw_probe_refill(sk, 0);++ return true; /* yes, decided state transition */++ }++ }++++ if (bbr2_is_inflight_too_high(sk, rs)) {++ if (bbr->bw_probe_samples) /* sample is from bw probing? */++ bbr2_handle_inflight_too_high(sk, rs);++ } else {++ /* Loss/ECN rate is declared safe. Adjust upper bound upward. */++ if (bbr->inflight_hi == ~0U) /* no excess queue signals yet? */++ return false;++++ /* To be resilient to random loss, we must raise inflight_hi++ * if we observe in any phase that a higher level is safe.++ */++ if (rs->tx_in_flight > bbr->inflight_hi) {++ bbr->inflight_hi = rs->tx_in_flight;++ bbr->debug.event = 'U'; /* raise up inflight_hi */++ }++++ if (bbr->mode == BBR_PROBE_BW &&++ bbr->cycle_idx == BBR_BW_PROBE_UP)++ bbr2_probe_inflight_hi_upward(sk, rs);++ }++++ return false;++}++++/* Check if it's time to probe for bandwidth now, and if so, kick it off. */++static bool bbr2_check_time_to_probe_bw(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u32 n;++++ /* If we seem to be at an operating point where we are not seeing loss++ * but we are seeing ECN marks, then when the ECN marks cease we reprobe++ * quickly (in case a burst of cross-traffic has ceased and freed up bw,++ * or in case we are sharing with multiplicatively probing traffic).++ */++ if (bbr->params.ecn_reprobe_gain && bbr->ecn_eligible &&++ bbr->ecn_in_cycle && !bbr->loss_in_cycle &&++ inet_csk(sk)->icsk_ca_state == TCP_CA_Open) {++ bbr->debug.event = 'A'; /* *A*ll clear to probe *A*gain */++ /* Calculate n so that when bbr2_raise_inflight_hi_slope()++ * computes growth_this_round as 2^n it will be roughly the++ * desired volume of data (inflight_hi*ecn_reprobe_gain).++ */++ n = ilog2((((u64)bbr->inflight_hi *++ bbr->params.ecn_reprobe_gain) >> BBR_SCALE));++ bbr2_start_bw_probe_refill(sk, n);++ return true;++ }++++ if (bbr2_has_elapsed_in_phase(sk, bbr->probe_wait_us) ||++ bbr2_is_reno_coexistence_probe_time(sk)) {++ bbr2_start_bw_probe_refill(sk, 0);++ return true;++ }++ return false;++}++++/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */++static bool bbr2_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw)++{++ struct bbr *bbr = inet_csk_ca(sk);++ bool is_under_bdp, is_long_enough;++++ /* Always need to pull inflight down to leave headroom in queue. */++ if (inflight > bbr2_inflight_with_headroom(sk))++ return false;++++ is_under_bdp = inflight <= bbr_inflight(sk, bw, BBR_UNIT);++ if (bbr->params.drain_to_target)++ return is_under_bdp;++++ is_long_enough = bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us);++ return is_under_bdp || is_long_enough;++}++++/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */++static void bbr2_update_cycle_phase(struct sock *sk,++ const struct rate_sample *rs)++{++ struct bbr *bbr = inet_csk_ca(sk);++ bool is_risky = false, is_queuing = false;++ u32 inflight, bw;++++ if (!bbr_full_bw_reached(sk))++ return;++++ /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */++ if (bbr2_adapt_upper_bounds(sk, rs))++ return; /* already decided state transition */++++ if (bbr->mode != BBR_PROBE_BW)++ return;++++ inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight);++ bw = bbr_max_bw(sk);++++ switch (bbr->cycle_idx) {++ /* First we spend most of our time cruising with a pacing_gain of 1.0,++ * which paces at the estimated bw, to try to fully use the pipe++ * without building queue. If we encounter loss/ECN marks, we adapt++ * by slowing down.++ */++ case BBR_BW_PROBE_CRUISE:++ if (bbr2_check_time_to_probe_bw(sk))++ return; /* already decided state transition */++ break;++++ /* After cruising, when it's time to probe, we first "refill": we send++ * at the estimated bw to fill the pipe, before probing higher and++ * knowingly risking overflowing the bottleneck buffer (causing loss).++ */++ case BBR_BW_PROBE_REFILL:++ if (bbr->round_start) {++ /* After one full round trip of sending in REFILL, we++ * start to see bw samples reflecting our REFILL, which++ * may be putting too much data in flight.++ */++ bbr->bw_probe_samples = 1;++ bbr2_start_bw_probe_up(sk);++ }++ break;++++ /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to++ * probe for bw. If we have not seen loss/ECN, we try to raise inflight++ * to at least pacing_gain*BDP; note that this may take more than++ * min_rtt if min_rtt is small (e.g. on a LAN).++ *++ * We terminate PROBE_UP bandwidth probing upon any of the following:++ *++ * (1) We've pushed inflight up to hit the inflight_hi target set in the++ * most recent previous bw probe phase. Thus we want to start++ * draining the queue immediately because it's very likely the most++ * recently sent packets will fill the queue and cause drops.++ * (checked here)++ * (2) We have probed for at least 1*min_rtt_us, and the++ * estimated queue is high enough (inflight > 1.25 * estimated_bdp).++ * (checked here)++ * (3) Loss filter says loss rate is "too high".++ * (checked in bbr_is_inflight_too_high())++ * (4) ECN filter says ECN mark rate is "too high".++ * (checked in bbr_is_inflight_too_high())++ */++ case BBR_BW_PROBE_UP:++ if (bbr->prev_probe_too_high &&++ inflight >= bbr->inflight_hi) {++ bbr->stopped_risky_probe = 1;++ is_risky = true;++ bbr->debug.event = 'D'; /* D for danger */++ } else if (bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us) &&++ inflight >=++ bbr_inflight(sk, bw,++ bbr->params.bw_probe_pif_gain)) {++ is_queuing = true;++ bbr->debug.event = 'Q'; /* building Queue */++ }++ if (is_risky || is_queuing) {++ bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */++ bbr2_start_bw_probe_down(sk); /* restart w/ down */++ }++ break;++++ /* After probing in PROBE_UP, we have usually accumulated some data in++ * the bottleneck buffer (if bw probing didn't find more bw). We next++ * enter PROBE_DOWN to try to drain any excess data from the queue. To++ * do this, we use a pacing_gain < 1.0. We hold this pacing gain until++ * our inflight is less then that target cruising point, which is the++ * minimum of (a) the amount needed to leave headroom, and (b) the++ * estimated BDP. Once inflight falls to match the target, we estimate++ * the queue is drained; persisting would underutilize the pipe.++ */++ case BBR_BW_PROBE_DOWN:++ if (bbr2_check_time_to_probe_bw(sk))++ return; /* already decided state transition */++ if (bbr2_check_time_to_cruise(sk, inflight, bw))++ bbr2_start_bw_probe_cruise(sk);++ break;++++ default:++ WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx);++ }++}++++/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */++static void bbr2_exit_probe_rtt(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr2_reset_lower_bounds(sk);++ if (bbr_full_bw_reached(sk)) {++ bbr->mode = BBR_PROBE_BW;++ /* Raising inflight after PROBE_RTT may cause loss, so reset++ * the PROBE_BW clock and schedule the next bandwidth probe for++ * a friendly and randomized future point in time.++ */++ bbr2_start_bw_probe_down(sk);++ /* Since we are exiting PROBE_RTT, we know inflight is++ * below our estimated BDP, so it is reasonable to cruise.++ */++ bbr2_start_bw_probe_cruise(sk);++ } else {++ bbr->mode = BBR_STARTUP;++ }++}++++/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until++ * the end of the round in recovery to get a good estimate of how many packets++ * have been lost, and how many we need to drain with a low pacing rate.++ */++static void bbr2_check_loss_too_high_in_startup(struct sock *sk,++ const struct rate_sample *rs)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ if (bbr_full_bw_reached(sk))++ return;++++ /* For STARTUP exit, check the loss rate at the end of each round trip++ * of Recovery episodes in STARTUP. We check the loss rate at the end++ * of the round trip to filter out noisy/low loss and have a better++ * sense of inflight (extent of loss), so we can drain more accurately.++ */++ if (rs->losses && bbr->loss_events_in_round < 0xf)++ bbr->loss_events_in_round++; /* update saturating counter */++ if (bbr->params.full_loss_cnt && bbr->loss_round_start &&++ inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery &&++ bbr->loss_events_in_round >= bbr->params.full_loss_cnt &&++ bbr2_is_inflight_too_high(sk, rs)) {++ bbr->debug.event = 'P'; /* Packet loss caused STARTUP exit */++ bbr2_handle_queue_too_high_in_startup(sk);++ return;++ }++ if (bbr->loss_round_start)++ bbr->loss_events_in_round = 0;++}++++/* If we are done draining, advance into steady state operation in PROBE_BW. */++static void bbr2_check_drain(struct sock *sk, const struct rate_sample *rs,++ struct bbr_context *ctx)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ if (bbr_check_drain(sk, rs, ctx)) {++ bbr->mode = BBR_PROBE_BW;++ bbr2_start_bw_probe_down(sk);++ }++}++++static void bbr2_update_model(struct sock *sk, const struct rate_sample *rs,++ struct bbr_context *ctx)++{++ bbr2_update_congestion_signals(sk, rs, ctx);++ bbr_update_ack_aggregation(sk, rs);++ bbr2_check_loss_too_high_in_startup(sk, rs);++ bbr_check_full_bw_reached(sk, rs);++ bbr2_check_drain(sk, rs, ctx);++ bbr2_update_cycle_phase(sk, rs);++ bbr_update_min_rtt(sk, rs);++}++++/* Fast path for app-limited case.++ *++ * On each ack, we execute bbr state machine, which primarily consists of:++ * 1) update model based on new rate sample, and++ * 2) update control based on updated model or state change.++ *++ * There are certain workload/scenarios, e.g. app-limited case, where++ * either we can skip updating model or we can skip update of both model++ * as well as control. This provides signifcant softirq cpu savings for++ * processing incoming acks.++ *++ * In case of app-limited, if there is no congestion (loss/ecn) and++ * if observed bw sample is less than current estimated bw, then we can++ * skip some of the computation in bbr state processing:++ *++ * - if there is no rtt/mode/phase change: In this case, since all the++ * parameters of the network model are constant, we can skip model++ * as well control update.++ *++ * - else we can skip rest of the model update. But we still need to++ * update the control to account for the new rtt/mode/phase.++ *++ * Returns whether we can take fast path or not.++ */++static bool bbr2_fast_path(struct sock *sk, bool *update_model,++ const struct rate_sample *rs, struct bbr_context *ctx)++{++ struct bbr *bbr = inet_csk_ca(sk);++ u32 prev_min_rtt_us, prev_mode;++++ if (bbr->params.fast_path && bbr->try_fast_path &&++ rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) &&++ !bbr->loss_in_round && !bbr->ecn_in_round) {++ prev_mode = bbr->mode;++ prev_min_rtt_us = bbr->min_rtt_us;++ bbr2_check_drain(sk, rs, ctx);++ bbr2_update_cycle_phase(sk, rs);++ bbr_update_min_rtt(sk, rs);++++ if (bbr->mode == prev_mode &&++ bbr->min_rtt_us == prev_min_rtt_us &&++ bbr->try_fast_path)++ return true;++++ /* Skip model update, but control still needs to be updated */++ *update_model = false;++ }++ return false;++}++++static void bbr2_main(struct sock *sk, const struct rate_sample *rs)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ struct bbr_context ctx = { 0 };++ bool update_model = true;++ u32 bw;++++ bbr->debug.event = '.'; /* init to default NOP (no event yet) */++++ bbr_update_round_start(sk, rs, &ctx);++ if (bbr->round_start) {++ bbr->rounds_since_probe =++ min_t(s32, bbr->rounds_since_probe + 1, 0xFF);++ bbr2_update_ecn_alpha(sk);++ }++++ bbr->ecn_in_round |= rs->is_ece;++ bbr_calculate_bw_sample(sk, rs, &ctx);++++ if (bbr2_fast_path(sk, &update_model, rs, &ctx))++ goto out;++++ if (update_model)++ bbr2_update_model(sk, rs, &ctx);++++ bbr_update_gains(sk);++ bw = bbr_bw(sk);++ bbr_set_pacing_rate(sk, bw, bbr->pacing_gain);++ bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain,++ tp->snd_cwnd, &ctx);++ bbr2_bound_cwnd_for_inflight_model(sk);++++out:++ bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state;++ bbr->loss_in_cycle |= rs->lost > 0;++ bbr->ecn_in_cycle |= rs->delivered_ce > 0;++++ bbr_debug(sk, rs->acked_sacked, rs, &ctx);++}++++/* Module parameters that are settable by TCP_CONGESTION_PARAMS are declared++ * down here, so that the algorithm functions that use the parameters must use++ * the per-socket parameters; if they accidentally use the global version++ * then there will be a compile error.++ * TODO(ncardwell): move all per-socket parameters down to this section.++ */++++/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE.++ * No loss response when 0. Max allwed value is 255.++ */++static u32 bbr_beta = BBR_UNIT * 30 / 100;++++/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE.++ * Max allowed value is 255.++ */++static u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; /* 1/16 = 6.25% */++++/* The initial value for the ecn_alpha state variable. Default and max++ * BBR_UNIT (256), representing 1.0. This allows a flow to respond quickly++ * to congestion if the bottleneck is congested when the flow starts up.++ */++static u32 bbr_ecn_alpha_init = BBR_UNIT; /* 1.0, to respond quickly */++++/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE.++ * No ECN based bounding when 0. Max allwed value is 255.++ */++static u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */++++/* Estimate bw probing has gone too far if CE ratio exceeds this threshold.++ * Scaled by BBR_SCALE. Disabled when 0. Max allowed is 255.++ */++static u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */++++/* Max RTT (in usec) at which to use sender-side ECN logic.++ * Disabled when 0 (ECN allowed at any RTT).++ * Max allowed for the parameter is 524287 (0x7ffff) us, ~524 ms.++ */++static u32 bbr_ecn_max_rtt_us = 5000;++++/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN++ * clears then use a multiplicative increase to quickly reprobe bw by++ * starting inflight probing at the given multiple of inflight_hi.++ * Default for this experimental knob is 0 (disabled).++ * Planned value for experiments: BBR_UNIT * 1 / 2 = 128, representing 0.5.++ */++static u32 bbr_ecn_reprobe_gain;++++/* Estimate bw probing has gone too far if loss rate exceeds this level. */++static u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */++++/* Exit STARTUP if number of loss marking events in a Recovery round is >= N,++ * and loss rate is higher than bbr_loss_thresh.++ * Disabled if 0. Max allowed value is 15 (0xF).++ */++static u32 bbr_full_loss_cnt = 8;++++/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh++ * meets this count. Max allowed value is 3.++ */++static u32 bbr_full_ecn_cnt = 2;++++/* Fraction of unutilized headroom to try to leave in path upon high loss. */++static u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100;++++/* Multiplier to get target inflight (as multiple of BDP) for PROBE_UP phase.++ * Default is 1.25x, as in BBR v1. Max allowed is 511.++ */++static u32 bbr_bw_probe_pif_gain = BBR_UNIT * 5 / 4;++++/* Multiplier to get Reno-style probe epoch duration as: k * BDP round trips.++ * If zero, disables this BBR v2 Reno-style BDP-scaled coexistence mechanism.++ * Max allowed is 511.++ */++static u32 bbr_bw_probe_reno_gain = BBR_UNIT;++++/* Max number of packet-timed rounds to wait before probing for bandwidth. If++ * we want to tolerate 1% random loss per round, and not have this cut our++ * inflight too much, we must probe for bw periodically on roughly this scale.++ * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance.++ * We aim to be fair with Reno/CUBIC up to a BDP of at least:++ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets++ */++static u32 bbr_bw_probe_max_rounds = 63;++++/* Max amount of randomness to inject in round counting for Reno-coexistence.++ * Max value is 15.++ */++static u32 bbr_bw_probe_rand_rounds = 2;++++/* Use BBR-native probe time scale starting at this many usec.++ * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least:++ * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs++ */++static u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */++++/* Use BBR-native probes spread over this many usec: */++static u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */++++/* Undo the model changes made in loss recovery if recovery was spurious? */++static bool bbr_undo = true;++++/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */++static bool bbr_fast_path = true; /* default: enabled */++++/* Use fast ack mode ? */++static int bbr_fast_ack_mode = 1; /* default: rwnd check off */++++/* How much to additively increase inflight_hi when entering REFILL? */++static u32 bbr_refill_add_inc; /* default: disabled */++++module_param_named(beta, bbr_beta, uint, 0644);++module_param_named(ecn_alpha_gain, bbr_ecn_alpha_gain, uint, 0644);++module_param_named(ecn_alpha_init, bbr_ecn_alpha_init, uint, 0644);++module_param_named(ecn_factor, bbr_ecn_factor, uint, 0644);++module_param_named(ecn_thresh, bbr_ecn_thresh, uint, 0644);++module_param_named(ecn_max_rtt_us, bbr_ecn_max_rtt_us, uint, 0644);++module_param_named(ecn_reprobe_gain, bbr_ecn_reprobe_gain, uint, 0644);++module_param_named(loss_thresh, bbr_loss_thresh, uint, 0664);++module_param_named(full_loss_cnt, bbr_full_loss_cnt, uint, 0664);++module_param_named(full_ecn_cnt, bbr_full_ecn_cnt, uint, 0664);++module_param_named(inflight_headroom, bbr_inflight_headroom, uint, 0664);++module_param_named(bw_probe_pif_gain, bbr_bw_probe_pif_gain, uint, 0664);++module_param_named(bw_probe_reno_gain, bbr_bw_probe_reno_gain, uint, 0664);++module_param_named(bw_probe_max_rounds, bbr_bw_probe_max_rounds, uint, 0664);++module_param_named(bw_probe_rand_rounds, bbr_bw_probe_rand_rounds, uint, 0664);++module_param_named(bw_probe_base_us, bbr_bw_probe_base_us, uint, 0664);++module_param_named(bw_probe_rand_us, bbr_bw_probe_rand_us, uint, 0664);++module_param_named(undo, bbr_undo, bool, 0664);++module_param_named(fast_path, bbr_fast_path, bool, 0664);++module_param_named(fast_ack_mode, bbr_fast_ack_mode, uint, 0664);++module_param_named(refill_add_inc, bbr_refill_add_inc, uint, 0664);++++static void bbr2_init(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr_init(sk); /* run shared init code for v1 and v2 */++++ /* BBR v2 parameters: */++ bbr->params.beta = min_t(u32, 0xFFU, bbr_beta);++ bbr->params.ecn_alpha_gain = min_t(u32, 0xFFU, bbr_ecn_alpha_gain);++ bbr->params.ecn_alpha_init = min_t(u32, BBR_UNIT, bbr_ecn_alpha_init);++ bbr->params.ecn_factor = min_t(u32, 0xFFU, bbr_ecn_factor);++ bbr->params.ecn_thresh = min_t(u32, 0xFFU, bbr_ecn_thresh);++ bbr->params.ecn_max_rtt_us = min_t(u32, 0x7ffffU, bbr_ecn_max_rtt_us);++ bbr->params.ecn_reprobe_gain = min_t(u32, 0x1FF, bbr_ecn_reprobe_gain);++ bbr->params.loss_thresh = min_t(u32, 0xFFU, bbr_loss_thresh);++ bbr->params.full_loss_cnt = min_t(u32, 0xFU, bbr_full_loss_cnt);++ bbr->params.full_ecn_cnt = min_t(u32, 0x3U, bbr_full_ecn_cnt);++ bbr->params.inflight_headroom =++ min_t(u32, 0xFFU, bbr_inflight_headroom);++ bbr->params.bw_probe_pif_gain =++ min_t(u32, 0x1FFU, bbr_bw_probe_pif_gain);++ bbr->params.bw_probe_reno_gain =++ min_t(u32, 0x1FFU, bbr_bw_probe_reno_gain);++ bbr->params.bw_probe_max_rounds =++ min_t(u32, 0xFFU, bbr_bw_probe_max_rounds);++ bbr->params.bw_probe_rand_rounds =++ min_t(u32, 0xFU, bbr_bw_probe_rand_rounds);++ bbr->params.bw_probe_base_us =++ min_t(u32, (1 << 26) - 1, bbr_bw_probe_base_us);++ bbr->params.bw_probe_rand_us =++ min_t(u32, (1 << 26) - 1, bbr_bw_probe_rand_us);++ bbr->params.undo = bbr_undo;++ bbr->params.fast_path = bbr_fast_path ? 1 : 0;++ bbr->params.refill_add_inc = min_t(u32, 0x3U, bbr_refill_add_inc);++++ /* BBR v2 state: */++ bbr->initialized = 1;++ /* Start sampling ECN mark rate after first full flight is ACKed: */++ bbr->loss_round_delivered = tp->delivered + 1;++ bbr->loss_round_start = 0;++ bbr->undo_bw_lo = 0;++ bbr->undo_inflight_lo = 0;++ bbr->undo_inflight_hi = 0;++ bbr->loss_events_in_round = 0;++ bbr->startup_ecn_rounds = 0;++ bbr2_reset_congestion_signals(sk);++ bbr->bw_lo = ~0U;++ bbr->bw_hi[0] = 0;++ bbr->bw_hi[1] = 0;++ bbr->inflight_lo = ~0U;++ bbr->inflight_hi = ~0U;++ bbr->bw_probe_up_cnt = ~0U;++ bbr->bw_probe_up_acks = 0;++ bbr->bw_probe_up_rounds = 0;++ bbr->probe_wait_us = 0;++ bbr->stopped_risky_probe = 0;++ bbr->ack_phase = BBR_ACKS_INIT;++ bbr->rounds_since_probe = 0;++ bbr->bw_probe_samples = 0;++ bbr->prev_probe_too_high = 0;++ bbr->ecn_eligible = 0;++ bbr->ecn_alpha = bbr->params.ecn_alpha_init;++ bbr->alpha_last_delivered = 0;++ bbr->alpha_last_delivered_ce = 0;++++ tp->fast_ack_mode = min_t(u32, 0x2U, bbr_fast_ack_mode);++++ if ((tp->ecn_flags & TCP_ECN_OK) && bbr_ecn_enable)++ tp->ecn_flags |= TCP_ECN_ECT_PERMANENT;++}++++/* Core TCP stack informs us that the given skb was just marked lost. */++static void bbr2_skb_marked_lost(struct sock *sk, const struct sk_buff *skb)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++ struct tcp_skb_cb *scb = TCP_SKB_CB(skb);++ struct rate_sample rs;++++ /* Capture "current" data over the full round trip of loss,++ * to have a better chance to see the full capacity of the path.++ */++ if (!bbr->loss_in_round) /* first loss in this round trip? */++ bbr->loss_round_delivered = tp->delivered; /* set round trip */++ bbr->loss_in_round = 1;++ bbr->loss_in_cycle = 1;++++ if (!bbr->bw_probe_samples)++ return; /* not an skb sent while probing for bandwidth */++ if (unlikely(!scb->tx.delivered_mstamp))++ return; /* skb was SACKed, reneged, marked lost; ignore it */++ /* We are probing for bandwidth. Construct a rate sample that++ * estimates what happened in the flight leading up to this lost skb,++ * then see if the loss rate went too high, and if so at which packet.++ */++ memset(&rs, 0, sizeof(rs));++ rs.tx_in_flight = scb->tx.in_flight;++ rs.lost = tp->lost - scb->tx.lost;++ rs.is_app_limited = scb->tx.is_app_limited;++ if (bbr2_is_inflight_too_high(sk, &rs)) {++ rs.tx_in_flight = bbr2_inflight_hi_from_lost_skb(sk, &rs, skb);++ bbr2_handle_inflight_too_high(sk, &rs);++ }++}++++/* Revert short-term model if current loss recovery event was spurious. */++static u32 bbr2_undo_cwnd(struct sock *sk)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr->debug.undo = 1;++ bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */++ bbr->full_bw_cnt = 0;++ bbr->loss_in_round = 0;++++ if (!bbr->params.undo)++ return tp->snd_cwnd;++++ /* Revert to cwnd and other state saved before loss episode. */++ bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo);++ bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo);++ bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi);++ return bbr->prior_cwnd;++}++++/* Entering loss recovery, so save state for when we undo recovery. */++static u32 bbr2_ssthresh(struct sock *sk)++{++ struct bbr *bbr = inet_csk_ca(sk);++++ bbr_save_cwnd(sk);++ /* For undo, save state that adapts based on loss signal. */++ bbr->undo_bw_lo = bbr->bw_lo;++ bbr->undo_inflight_lo = bbr->inflight_lo;++ bbr->undo_inflight_hi = bbr->inflight_hi;++ return tcp_sk(sk)->snd_ssthresh;++}++++static enum tcp_bbr2_phase bbr2_get_phase(struct bbr *bbr)++{++ switch (bbr->mode) {++ case BBR_STARTUP:++ return BBR2_PHASE_STARTUP;++ case BBR_DRAIN:++ return BBR2_PHASE_DRAIN;++ case BBR_PROBE_BW:++ break;++ case BBR_PROBE_RTT:++ return BBR2_PHASE_PROBE_RTT;++ default:++ return BBR2_PHASE_INVALID;++ }++ switch (bbr->cycle_idx) {++ case BBR_BW_PROBE_UP:++ return BBR2_PHASE_PROBE_BW_UP;++ case BBR_BW_PROBE_DOWN:++ return BBR2_PHASE_PROBE_BW_DOWN;++ case BBR_BW_PROBE_CRUISE:++ return BBR2_PHASE_PROBE_BW_CRUISE;++ case BBR_BW_PROBE_REFILL:++ return BBR2_PHASE_PROBE_BW_REFILL;++ default:++ return BBR2_PHASE_INVALID;++ }++}++++static size_t bbr2_get_info(struct sock *sk, u32 ext, int *attr,++ union tcp_cc_info *info)++{++ if (ext & (1 << (INET_DIAG_BBRINFO - 1)) ||++ ext & (1 << (INET_DIAG_VEGASINFO - 1))) {++ struct bbr *bbr = inet_csk_ca(sk);++ u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk));++ u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk));++ u64 bw_lo = bbr->bw_lo == ~0U ?++ ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo);++++ memset(&info->bbr2, 0, sizeof(info->bbr2));++ info->bbr2.bbr_bw_lsb = (u32)bw;++ info->bbr2.bbr_bw_msb = (u32)(bw >> 32);++ info->bbr2.bbr_min_rtt = bbr->min_rtt_us;++ info->bbr2.bbr_pacing_gain = bbr->pacing_gain;++ info->bbr2.bbr_cwnd_gain = bbr->cwnd_gain;++ info->bbr2.bbr_bw_hi_lsb = (u32)bw_hi;++ info->bbr2.bbr_bw_hi_msb = (u32)(bw_hi >> 32);++ info->bbr2.bbr_bw_lo_lsb = (u32)bw_lo;++ info->bbr2.bbr_bw_lo_msb = (u32)(bw_lo >> 32);++ info->bbr2.bbr_mode = bbr->mode;++ info->bbr2.bbr_phase = (__u8)bbr2_get_phase(bbr);++ info->bbr2.bbr_version = (__u8)2;++ info->bbr2.bbr_inflight_lo = bbr->inflight_lo;++ info->bbr2.bbr_inflight_hi = bbr->inflight_hi;++ info->bbr2.bbr_extra_acked = bbr_extra_acked(sk);++ *attr = INET_DIAG_BBRINFO;++ return sizeof(info->bbr2);++ }++ return 0;++}++++static void bbr2_set_state(struct sock *sk, u8 new_state)++{++ struct tcp_sock *tp = tcp_sk(sk);++ struct bbr *bbr = inet_csk_ca(sk);++++ if (new_state == TCP_CA_Loss) {++ struct rate_sample rs = { .losses = 1 };++ struct bbr_context ctx = { 0 };++++ bbr->prev_ca_state = TCP_CA_Loss;++ bbr->full_bw = 0;++ if (!bbr2_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) {++ /* bbr_adapt_lower_bounds() needs cwnd before++ * we suffered an RTO, to update inflight_lo:++ */++ bbr->inflight_lo =++ max(tp->snd_cwnd, bbr->prior_cwnd);++ }++ bbr_debug(sk, 0, &rs, &ctx);++ } else if (bbr->prev_ca_state == TCP_CA_Loss &&++ new_state != TCP_CA_Loss) {++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd);++ bbr->try_fast_path = 0; /* bound cwnd using latest model */++ }++}++++static struct tcp_congestion_ops tcp_bbr2_cong_ops __read_mostly = {++ .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS,++ .name = "bbr2",++ .owner = THIS_MODULE,++ .init = bbr2_init,++ .cong_control = bbr2_main,++ .sndbuf_expand = bbr_sndbuf_expand,++ .skb_marked_lost = bbr2_skb_marked_lost,++ .undo_cwnd = bbr2_undo_cwnd,++ .cwnd_event = bbr_cwnd_event,++ .ssthresh = bbr2_ssthresh,++ .tso_segs = bbr_tso_segs,++ .get_info = bbr2_get_info,++ .set_state = bbr2_set_state,++};++++static int __init bbr_register(void)++{++ BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE);++ return tcp_register_congestion_control(&tcp_bbr2_cong_ops);++}++++static void __exit bbr_unregister(void)++{++ tcp_unregister_congestion_control(&tcp_bbr2_cong_ops);++}++++module_init(bbr_register);++module_exit(bbr_unregister);++++MODULE_AUTHOR("Van Jacobson <[email protected]>");++MODULE_AUTHOR("Neal Cardwell <[email protected]>");++MODULE_AUTHOR("Yuchung Cheng <[email protected]>");++MODULE_AUTHOR("Soheil Hassas Yeganeh <[email protected]>");++MODULE_AUTHOR("Priyaranjan Jha <[email protected]>");++MODULE_AUTHOR("Yousuk Seung <[email protected]>");++MODULE_AUTHOR("Kevin Yang <[email protected]>");++MODULE_AUTHOR("Arjun Roy <[email protected]>");++++MODULE_LICENSE("Dual BSD/GPL");++MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)");+diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c+index db5831e6c136..153ed9010c0c 100644+--- a/net/ipv4/tcp_cong.c++++ b/net/ipv4/tcp_cong.c+@@ -179,6 +179,7 @@ void tcp_init_congestion_control(struct sock *sk)+ struct inet_connection_sock *icsk = inet_csk(sk);++ tcp_sk(sk)->prior_ssthresh = 0;++ tcp_sk(sk)->fast_ack_mode = 0;+ if (icsk->icsk_ca_ops->init)+ icsk->icsk_ca_ops->init(sk);+ if (tcp_ca_needs_ecn(sk))+diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c+index dc49a3d551eb..a403e21c702e 100644+--- a/net/ipv4/tcp_input.c++++ b/net/ipv4/tcp_input.c+@@ -349,7 +349,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)+ tcp_enter_quickack_mode(sk, 2);+ break;+ case INET_ECN_CE:+- if (tcp_ca_needs_ecn(sk))++ if (tcp_ca_wants_ce_events(sk))+ tcp_ca_event(sk, CA_EVENT_ECN_IS_CE);++ if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) {+@@ -360,7 +360,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)+ tp->ecn_flags |= TCP_ECN_SEEN;+ break;+ default:+- if (tcp_ca_needs_ecn(sk))++ if (tcp_ca_wants_ce_events(sk))+ tcp_ca_event(sk, CA_EVENT_ECN_NO_CE);+ tp->ecn_flags |= TCP_ECN_SEEN;+ break;+@@ -1079,7 +1079,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb)+ */+ static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb)+ {++ struct sock *sk = (struct sock *)tp;++ const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;+++ tp->lost += tcp_skb_pcount(skb);++ if (ca_ops->skb_marked_lost)++ ca_ops->skb_marked_lost(sk, skb);+ }++ void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb)+@@ -1460,6 +1465,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev,+ WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount);+ tcp_skb_pcount_add(skb, -pcount);+++ /* Adjust tx.in_flight as pcount is shifted from skb to prev. */++ if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount,++ "prev in_flight: %u skb in_flight: %u pcount: %u",++ TCP_SKB_CB(prev)->tx.in_flight,++ TCP_SKB_CB(skb)->tx.in_flight,++ pcount))++ TCP_SKB_CB(skb)->tx.in_flight = 0;++ else++ TCP_SKB_CB(skb)->tx.in_flight -= pcount;++ TCP_SKB_CB(prev)->tx.in_flight += pcount;+++ /* When we're adding to gso_segs == 1, gso_size will be zero,+ * in theory this shouldn't be necessary but as long as DSACK+ * code can come after this skb later on it's better to keep+@@ -3788,6 +3804,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)++ prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una;+ rs.prior_in_flight = tcp_packets_in_flight(tp);++ tcp_rate_check_app_limited(sk);++ /* ts_recent update must be made after we are sure that the packet+ * is in window.+@@ -3885,6 +3902,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)+ delivered = tcp_newly_delivered(sk, delivered, flag);+ lost = tp->lost - lost; /* freshly marked lost */+ rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED);++ rs.is_ece = !!(flag & FLAG_ECE);+ tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate);+ tcp_cong_control(sk, ack, delivered, flag, sack_state.rate);+ tcp_xmit_recovery(sk, rexmit);+@@ -5463,13 +5481,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible)++ /* More than one full frame received... */+ if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss &&++ (tp->fast_ack_mode == 1 ||+ /* ... and right edge of window advances far enough.+ * (tcp_recvmsg() will send ACK otherwise).+ * If application uses SO_RCVLOWAT, we want send ack now if+ * we have not received enough bytes to satisfy the condition.+ */+- (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat ||+- __tcp_select_window(sk) >= tp->rcv_wnd)) ||++ (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat ||++ __tcp_select_window(sk) >= tp->rcv_wnd))) ||+ /* We ACK each frame or... */+ tcp_in_quickack_mode(sk) ||+ /* Protocol state mandates a one-time immediate ACK */+diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c+index 5079832af5c1..baa20918f11a 100644+--- a/net/ipv4/tcp_output.c++++ b/net/ipv4/tcp_output.c+@@ -377,7 +377,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,+ th->cwr = 1;+ skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;+ }+- } else if (!tcp_ca_needs_ecn(sk)) {++ } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) &&++ !tcp_ca_needs_ecn(sk)) {+ /* ACK or retransmitted segment: clear ECT|CE */+ INET_ECN_dontxmit(sk);+ }+@@ -1533,7 +1534,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,+ {+ struct tcp_sock *tp = tcp_sk(sk);+ struct sk_buff *buff;+- int nsize, old_factor;++ int nsize, old_factor, inflight_prev;+ long limit;+ int nlen;+ u8 flags;+@@ -1610,6 +1611,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,++ if (diff)+ tcp_adjust_pcount(sk, skb, diff);++++ /* Set buff tx.in_flight as if buff were sent by itself. */++ inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor;++ if (WARN_ONCE(inflight_prev < 0,++ "inconsistent: tx.in_flight: %u old_factor: %d",++ TCP_SKB_CB(skb)->tx.in_flight, old_factor))++ inflight_prev = 0;++ TCP_SKB_CB(buff)->tx.in_flight = inflight_prev +++ tcp_skb_pcount(buff);+ }++ /* Link BUFF into the send queue. */+@@ -1978,13 +1988,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,+ static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)+ {+ const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;+- u32 min_tso, tso_segs;+-+- min_tso = ca_ops->min_tso_segs ?+- ca_ops->min_tso_segs(sk) :+- sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs;++ u32 tso_segs;++- tso_segs = tcp_tso_autosize(sk, mss_now, min_tso);++ tso_segs = ca_ops->tso_segs ?++ ca_ops->tso_segs(sk, mss_now) :++ tcp_tso_autosize(sk, mss_now,++ sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs);+ return min_t(u32, tso_segs, sk->sk_gso_max_segs);+ }++@@ -2619,6 +2628,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,+ skb->skb_mstamp_ns = tp->tcp_wstamp_ns = tp->tcp_clock_cache;+ list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);+ tcp_init_tso_segs(skb, mss_now);++ tcp_set_tx_in_flight(sk, skb);+ goto repair; /* Skip network transmission */+ }++diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c+index fbab921670cc..796fa6e5310c 100644+--- a/net/ipv4/tcp_rate.c++++ b/net/ipv4/tcp_rate.c+@@ -34,6 +34,24 @@+ * ready to send in the write queue.+ */+++void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb)++{++ struct tcp_sock *tp = tcp_sk(sk);++ u32 in_flight;++++ /* Check, sanitize, and record packets in flight after skb was sent. */++ in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb);++ if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX,++ "insane in_flight %u cc %s mss %u "++ "cwnd %u pif %u %u %u %u\n",++ in_flight, inet_csk(sk)->icsk_ca_ops->name,++ tp->mss_cache, tp->snd_cwnd,++ tp->packets_out, tp->retrans_out,++ tp->sacked_out, tp->lost_out))++ in_flight = TCPCB_IN_FLIGHT_MAX;++ TCP_SKB_CB(skb)->tx.in_flight = in_flight;++}+++ /* Snapshot the current delivery information in the skb, to generate+ * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered().+ */+@@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb)+ TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp;+ TCP_SKB_CB(skb)->tx.delivered = tp->delivered;+ TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce;++ TCP_SKB_CB(skb)->tx.lost = tp->lost;+ TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0;++ tcp_set_tx_in_flight(sk, skb);+ }++ /* When an skb is sacked or acked, we fill in the rate sample with the (prior)+@@ -87,17 +107,20 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,++ if (!rs->prior_delivered ||+ after(scb->tx.delivered, rs->prior_delivered)) {++ rs->prior_lost = scb->tx.lost;+ rs->prior_delivered_ce = scb->tx.delivered_ce;+ rs->prior_delivered = scb->tx.delivered;+ rs->prior_mstamp = scb->tx.delivered_mstamp;+ rs->is_app_limited = scb->tx.is_app_limited;+ rs->is_retrans = scb->sacked & TCPCB_RETRANS;++ rs->tx_in_flight = scb->tx.in_flight;++ /* Record send time of most recently ACKed packet: */+ tp->first_tx_mstamp = tcp_skb_timestamp_us(skb);+ /* Find the duration of the "send phase" of this window: */+- rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp,+- scb->tx.first_tx_mstamp);++ rs->interval_us = tcp_stamp32_us_delta(++ tp->first_tx_mstamp,++ scb->tx.first_tx_mstamp);++ }+ /* Mark off the skb delivered once it's sacked to avoid being+@@ -139,6 +162,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,+ return;+ }+ rs->delivered = tp->delivered - rs->prior_delivered;++ rs->lost = tp->lost - rs->prior_lost;++ rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce;+ /* delivered_ce occupies less than 32 bits in the skb control block */+@@ -150,7 +174,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,+ * longer phase.+ */+ snd_us = rs->interval_us; /* send phase */+- ack_us = tcp_stamp_us_delta(tp->tcp_mstamp,++ ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp,+ rs->prior_mstamp); /* ack phase */+ rs->interval_us = max(snd_us, ack_us);++diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c+index 20cf4a98c69d..b5f7e49a003a 100644+--- a/net/ipv4/tcp_timer.c++++ b/net/ipv4/tcp_timer.c+@@ -607,6 +607,7 @@ void tcp_write_timer_handler(struct sock *sk)+ goto out;+ }+++ tcp_rate_check_app_limited(sk);+ tcp_mstamp_refresh(tcp_sk(sk));+ event = icsk->icsk_pending;++--+2.34.1++From f7f9bfc59cd69338d35b85f688cc94edc86bdba9 Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sat, 29 Jan 2022 07:49:56 +0000+Subject: [PATCH] cfi++---+ arch/x86/Kconfig | 1 ++ arch/x86/include/asm/ftrace.h | 2 +-+ arch/x86/include/asm/idtentry.h | 10 +++---+ arch/x86/include/asm/page_64.h | 7 ++--+ arch/x86/include/asm/paravirt_types.h | 3 +-+ arch/x86/include/asm/processor.h | 2 +-+ arch/x86/include/asm/proto.h | 25 ++++++-------+ arch/x86/include/asm/static_call.h | 1 -+ arch/x86/include/asm/uaccess_64.h | 9 ++---+ arch/x86/kernel/alternative.c | 2 +-+ arch/x86/kernel/ftrace.c | 2 +-+ arch/x86/kernel/kprobes/opt.c | 4 +--+ arch/x86/kernel/module.c | 4 ++++ arch/x86/kernel/paravirt.c | 4 +--+ arch/x86/kernel/static_call.c | 14 +++-----+ arch/x86/kvm/emulate.c | 4 +--+ arch/x86/kvm/kvm_emulate.h | 9 ++---+ arch/x86/power/Makefile | 2 +++ arch/x86/purgatory/Makefile | 4 ++++ arch/x86/tools/relocs.c | 7 +++++ arch/x86/xen/enlighten_pv.c | 6 ++--+ arch/x86/xen/xen-ops.h | 10 +++---+ drivers/misc/lkdtm/bugs.c | 2 +-+ drivers/misc/lkdtm/lkdtm.h | 2 +-+ drivers/misc/lkdtm/perms.c | 2 +-+ drivers/misc/lkdtm/rodata.c | 2 +-+ include/asm-generic/vmlinux.lds.h | 11 +++++++ include/linux/cfi.h | 13 ++++++++ include/linux/ftrace.h | 7 ++--+ include/linux/linkage.h | 13 ++++++++ include/linux/objtool.h | 6 +++++ kernel/cfi.c | 24 ++++++++++++-+ kernel/tracepoint.c | 6 ++--+ tools/include/linux/objtool.h | 6 +++++ tools/objtool/arch/x86/decode.c | 17 ++++++++++ tools/objtool/check.c | 3 --+ tools/objtool/elf.c | 51 ++++++++++++++++++++++++++++ tools/objtool/include/objtool/arch.h | 3 +++ tools/objtool/include/objtool/elf.h | 2 +-+ 39 files changed, 222 insertions(+), 80 deletions(-)++diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig+index ebe8fc76949a..fb7e51bb0210 100644+--- a/arch/x86/Kconfig++++ b/arch/x86/Kconfig+@@ -109,6 +109,7 @@ config X86+ select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096+ select ARCH_SUPPORTS_LTO_CLANG+ select ARCH_SUPPORTS_LTO_CLANG_THIN++ select ARCH_SUPPORTS_CFI_CLANG if X86_64 && CLANG_VERSION >= 130000+ select ARCH_USE_BUILTIN_BSWAP+ select ARCH_USE_MEMTEST+ select ARCH_USE_QUEUED_RWLOCKS+diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h+index 024d9797646e..6fa07b9015e5 100644+--- a/arch/x86/include/asm/ftrace.h++++ b/arch/x86/include/asm/ftrace.h+@@ -17,7 +17,7 @@++ #ifndef __ASSEMBLY__+ extern atomic_t modifying_ftrace_code;+-extern void __fentry__(void);++DECLARE_NOT_CALLED_FROM_C(__fentry__);++ static inline unsigned long ftrace_call_adjust(unsigned long addr)+ {+diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h+index 1345088e9902..6538bf5a47d6 100644+--- a/arch/x86/include/asm/idtentry.h++++ b/arch/x86/include/asm/idtentry.h+@@ -27,8 +27,8 @@+ * as well which is used to emit the entry stubs in entry_32/64.S.+ */+ #define DECLARE_IDTENTRY(vector, func) \+- asmlinkage void asm_##func(void); \+- asmlinkage void xen_asm_##func(void); \++ DECLARE_NOT_CALLED_FROM_C(asm_##func); \++ DECLARE_NOT_CALLED_FROM_C(xen_asm_##func); \+ __visible void func(struct pt_regs *regs)++ /**+@@ -78,8 +78,8 @@ static __always_inline void __##func(struct pt_regs *regs)+ * C-handler.+ */+ #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \+- asmlinkage void asm_##func(void); \+- asmlinkage void xen_asm_##func(void); \++ DECLARE_NOT_CALLED_FROM_C(asm_##func); \++ DECLARE_NOT_CALLED_FROM_C(xen_asm_##func); \+ __visible void func(struct pt_regs *regs, unsigned long error_code)++ /**+@@ -386,7 +386,7 @@ static __always_inline void __##func(struct pt_regs *regs)+ * - The C handler called from the C shim+ */+ #define DECLARE_IDTENTRY_DF(vector, func) \+- asmlinkage void asm_##func(void); \++ DECLARE_NOT_CALLED_FROM_C(asm_##func); \+ __visible void func(struct pt_regs *regs, \+ unsigned long error_code, \+ unsigned long address)+diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h+index e9c86299b835..7f314bcc78e8 100644+--- a/arch/x86/include/asm/page_64.h++++ b/arch/x86/include/asm/page_64.h+@@ -6,6 +6,7 @@++ #ifndef __ASSEMBLY__+ #include <asm/cpufeatures.h>++#include <linux/linkage.h>+ #include <asm/alternative.h>++ /* duplicated to the one in bootmem.h */+@@ -41,9 +42,9 @@ extern unsigned long __phys_addr_symbol(unsigned long);+ #define pfn_valid(pfn) ((pfn) < max_pfn)+ #endif++-void clear_page_orig(void *page);+-void clear_page_rep(void *page);+-void clear_page_erms(void *page);++DECLARE_NOT_CALLED_FROM_C(clear_page_orig);++DECLARE_NOT_CALLED_FROM_C(clear_page_rep);++DECLARE_NOT_CALLED_FROM_C(clear_page_erms);++ static inline void clear_page(void *page)+ {+diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h+index a69012e1903f..c3b00efd8658 100644+--- a/arch/x86/include/asm/paravirt_types.h++++ b/arch/x86/include/asm/paravirt_types.h+@@ -38,6 +38,7 @@+ #include <asm/desc_defs.h>+ #include <asm/pgtable_types.h>+ #include <asm/nospec-branch.h>++#include <asm/proto.h>++ struct page;+ struct thread_struct;+@@ -272,7 +273,7 @@ struct paravirt_patch_template {++ extern struct pv_info pv_info;+ extern struct paravirt_patch_template pv_ops;+-extern void (*paravirt_iret)(void);++extern asm_func_ptr paravirt_iret;++ #define PARAVIRT_PATCH(x) \+ (offsetof(struct paravirt_patch_template, x) / sizeof(void *))+diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h+index 2c5f12ae7d04..4fe53e114fe0 100644+--- a/arch/x86/include/asm/processor.h++++ b/arch/x86/include/asm/processor.h+@@ -450,7 +450,7 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu)++ DECLARE_PER_CPU(void *, hardirq_stack_ptr);+ DECLARE_PER_CPU(bool, hardirq_stack_inuse);+-extern asmlinkage void ignore_sysret(void);++DECLARE_NOT_CALLED_FROM_C(ignore_sysret);++ /* Save actual FS/GS selectors and bases to current->thread */+ void current_save_fsgs(void);+diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h+index feed36d44d04..31ea9aeaf68e 100644+--- a/arch/x86/include/asm/proto.h++++ b/arch/x86/include/asm/proto.h+@@ -2,6 +2,7 @@+ #ifndef _ASM_X86_PROTO_H+ #define _ASM_X86_PROTO_H+++#include <linux/linkage.h>+ #include <asm/ldt.h>++ struct task_struct;+@@ -11,26 +12,26 @@ struct task_struct;+ void syscall_init(void);++ #ifdef CONFIG_X86_64+-void entry_SYSCALL_64(void);+-void entry_SYSCALL_64_safe_stack(void);++DECLARE_NOT_CALLED_FROM_C(entry_SYSCALL_64);++DECLARE_NOT_CALLED_FROM_C(entry_SYSCALL_64_safe_stack);+ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2);+ #endif++ #ifdef CONFIG_X86_32+-void entry_INT80_32(void);+-void entry_SYSENTER_32(void);+-void __begin_SYSENTER_singlestep_region(void);+-void __end_SYSENTER_singlestep_region(void);++DECLARE_NOT_CALLED_FROM_C(entry_INT80_32);++DECLARE_NOT_CALLED_FROM_C(entry_SYSENTER_32);++DECLARE_NOT_CALLED_FROM_C(__begin_SYSENTER_singlestep_region);++DECLARE_NOT_CALLED_FROM_C(__end_SYSENTER_singlestep_region);+ #endif++ #ifdef CONFIG_IA32_EMULATION+-void entry_SYSENTER_compat(void);+-void __end_entry_SYSENTER_compat(void);+-void entry_SYSCALL_compat(void);+-void entry_SYSCALL_compat_safe_stack(void);+-void entry_INT80_compat(void);++DECLARE_NOT_CALLED_FROM_C(entry_SYSENTER_compat);++DECLARE_NOT_CALLED_FROM_C(__end_entry_SYSENTER_compat);++DECLARE_NOT_CALLED_FROM_C(entry_SYSCALL_compat);++DECLARE_NOT_CALLED_FROM_C(entry_SYSCALL_compat_safe_stack);++DECLARE_NOT_CALLED_FROM_C(entry_INT80_compat);+ #ifdef CONFIG_XEN_PV+-void xen_entry_INT80_compat(void);++DECLARE_NOT_CALLED_FROM_C(xen_entry_INT80_compat);+ #endif+ #endif++diff --git a/arch/x86/include/asm/static_call.h b/arch/x86/include/asm/static_call.h+index ed4f8bb6c2d9..343234569392 100644+--- a/arch/x86/include/asm/static_call.h++++ b/arch/x86/include/asm/static_call.h+@@ -27,7 +27,6 @@+ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \+ STATIC_CALL_TRAMP_STR(name) ": \n" \+ insns " \n" \+- ".byte 0x53, 0x43, 0x54 \n" \+ ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \+ ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \+ ".popsection \n")+diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h+index 45697e04d771..96cf72d6b75c 100644+--- a/arch/x86/include/asm/uaccess_64.h++++ b/arch/x86/include/asm/uaccess_64.h+@@ -17,12 +17,9 @@+ */++ /* Handles exceptions in both to and from, but doesn't do access_ok */+-__must_check unsigned long+-copy_user_enhanced_fast_string(void *to, const void *from, unsigned len);+-__must_check unsigned long+-copy_user_generic_string(void *to, const void *from, unsigned len);+-__must_check unsigned long+-copy_user_generic_unrolled(void *to, const void *from, unsigned len);++DECLARE_NOT_CALLED_FROM_C(copy_user_enhanced_fast_string);++DECLARE_NOT_CALLED_FROM_C(copy_user_generic_string);++DECLARE_NOT_CALLED_FROM_C(copy_user_generic_unrolled);++ static __always_inline __must_check unsigned long+ copy_user_generic(void *to, const void *from, unsigned len)+diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c+index 5007c3ffe96f..60347ce35871 100644+--- a/arch/x86/kernel/alternative.c++++ b/arch/x86/kernel/alternative.c+@@ -707,7 +707,7 @@ extern struct paravirt_patch_site __start_parainstructions[],+ * convention such that we can 'call' it from assembly.+ */++-extern void int3_magic(unsigned int *ptr); /* defined in asm */++DECLARE_NOT_CALLED_FROM_C(int3_magic);++ asm (+ " .pushsection .init.text, \"ax\", @progbits\n"+diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c+index 7cc540e6de0c..d5a5df914e16 100644+--- a/arch/x86/kernel/ftrace.c++++ b/arch/x86/kernel/ftrace.c+@@ -587,7 +587,7 @@ void arch_ftrace_trampoline_free(struct ftrace_ops *ops)+ #ifdef CONFIG_DYNAMIC_FTRACE++ #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS+-extern void ftrace_graph_call(void);++DECLARE_NOT_CALLED_FROM_C(ftrace_graph_call);+ static const char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)+ {+ return text_gen_insn(JMP32_INSN_OPCODE, (void *)ip, (void *)addr);+diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c+index b4a54a52aa59..48d302cd7448 100644+--- a/arch/x86/kernel/kprobes/opt.c++++ b/arch/x86/kernel/kprobes/opt.c+@@ -103,6 +103,7 @@ static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val)+ asm (+ ".pushsection .rodata\n"+ "optprobe_template_func:\n"++ ASM_STACK_FRAME_NON_STANDARD(optprobe_template_func)+ ".global optprobe_template_entry\n"+ "optprobe_template_entry:\n"+ #ifdef CONFIG_X86_64+@@ -154,9 +155,6 @@ asm (+ "optprobe_template_end:\n"+ ".popsection\n");++-void optprobe_template_func(void);+-STACK_FRAME_NON_STANDARD(optprobe_template_func);+-+ #define TMPL_CLAC_IDX \+ ((long)optprobe_template_clac - (long)optprobe_template_entry)+ #define TMPL_MOVE_IDX \+diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c+index 95fa745e310a..35baee327976 100644+--- a/arch/x86/kernel/module.c++++ b/arch/x86/kernel/module.c+@@ -198,6 +198,10 @@ static int __apply_relocate_add(Elf64_Shdr *sechdrs,+ val -= (u64)loc;+ write(loc, &val, 8);+ break;++ case R_X86_64_8:++ if (!strncmp(strtab + sym->st_name, "__typeid__", 10))++ break;++ fallthrough;+ default:+ pr_err("%s: Unknown rela relocation: %llu\n",+ me->name, ELF64_R_TYPE(rel[i].r_info));+diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c+index 4420499f7bb4..3dec0dd45d92 100644+--- a/arch/x86/kernel/paravirt.c++++ b/arch/x86/kernel/paravirt.c+@@ -149,7 +149,7 @@ void paravirt_set_sched_clock(u64 (*func)(void))+ }++ /* These are in entry.S */+-extern void native_iret(void);++DECLARE_NOT_CALLED_FROM_C(native_iret);++ static struct resource reserve_ioports = {+ .start = 0,+@@ -415,7 +415,7 @@ struct paravirt_patch_template pv_ops = {+ #ifdef CONFIG_PARAVIRT_XXL+ NOKPROBE_SYMBOL(native_load_idt);++-void (*paravirt_iret)(void) = native_iret;++asm_func_ptr paravirt_iret = native_iret;+ #endif++ EXPORT_SYMBOL(pv_ops);+diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c+index 531fb4cbb63f..81c9052033c7 100644+--- a/arch/x86/kernel/static_call.c++++ b/arch/x86/kernel/static_call.c+@@ -57,15 +57,10 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, void+ text_poke_bp(insn, code, size, emulate);+ }++-static void __static_call_validate(void *insn, bool tail, bool tramp)++static void __static_call_validate(void *insn, bool tail)+ {+ u8 opcode = *(u8 *)insn;++- if (tramp && memcmp(insn+5, "SCT", 3)) {+- pr_err("trampoline signature fail");+- BUG();+- }+-+ if (tail) {+ if (opcode == JMP32_INSN_OPCODE ||+ opcode == RET_INSN_OPCODE)+@@ -80,8 +75,7 @@ static void __static_call_validate(void *insn, bool tail, bool tramp)+ /*+ * If we ever trigger this, our text is corrupt, we'll probably not live long.+ */+- pr_err("unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn);+- BUG();++ WARN_ONCE(1, "unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn);+ }++ static inline enum insn_type __sc_insn(bool null, bool tail)+@@ -104,12 +98,12 @@ void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)+ mutex_lock(&text_mutex);++ if (tramp) {+- __static_call_validate(tramp, true, true);++ __static_call_validate(tramp, true);+ __static_call_transform(tramp, __sc_insn(!func, true), func);+ }++ if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site) {+- __static_call_validate(site, tail, false);++ __static_call_validate(site, tail);+ __static_call_transform(site, __sc_insn(!func, tail), func);+ }++diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c+index 5719d8cfdbd9..978e6fffcc69 100644+--- a/arch/x86/kvm/emulate.c++++ b/arch/x86/kvm/emulate.c+@@ -203,7 +203,7 @@ struct opcode {+ const struct escape *esc;+ const struct instr_dual *idual;+ const struct mode_dual *mdual;+- void (*fastop)(struct fastop *fake);++ fastop_t fastop;+ } u;+ int (*check_perm)(struct x86_emulate_ctxt *ctxt);+ };+@@ -324,7 +324,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);+ __FOP_RET(#name)++ #define FOP_START(op) \+- extern void em_##op(struct fastop *fake); \++ DECLARE_NOT_CALLED_FROM_C(em_##op); \+ asm(".pushsection .text, \"ax\" \n\t" \+ ".global em_" #op " \n\t" \+ ".align " __stringify(FASTOP_SIZE) " \n\t" \+diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h+index 39eded2426ff..49ca866e478d 100644+--- a/arch/x86/kvm/kvm_emulate.h++++ b/arch/x86/kvm/kvm_emulate.h+@@ -290,13 +290,8 @@ enum x86emul_mode {+ #define X86EMUL_SMM_MASK (1 << 6)+ #define X86EMUL_SMM_INSIDE_NMI_MASK (1 << 7)++-/*+- * fastop functions are declared as taking a never-defined fastop parameter,+- * so they can't be called from C directly.+- */+-struct fastop;+-+-typedef void (*fastop_t)(struct fastop *);++/* fastop functions cannot be called from C directly. */++typedef asm_func_ptr fastop_t;++ struct x86_emulate_ctxt {+ void *vcpu;+diff --git a/arch/x86/power/Makefile b/arch/x86/power/Makefile+index 379777572bc9..a0532851fed7 100644+--- a/arch/x86/power/Makefile++++ b/arch/x86/power/Makefile+@@ -4,9 +4,11 @@+ # itself be stack-protected+ CFLAGS_cpu.o := -fno-stack-protector+++ifndef CONFIG_CFI_CLANG+ # Clang may incorrectly inline functions with stack protector enabled into+ # __restore_processor_state(): https://bugs.llvm.org/show_bug.cgi?id=47479+ CFLAGS_REMOVE_cpu.o := $(CC_FLAGS_LTO)++endif++ obj-$(CONFIG_PM_SLEEP) += cpu.o+ obj-$(CONFIG_HIBERNATION) += hibernate_$(BITS).o hibernate_asm_$(BITS).o hibernate.o+diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile+index ae53d54d7959..b3fa947fa38b 100644+--- a/arch/x86/purgatory/Makefile++++ b/arch/x86/purgatory/Makefile+@@ -55,6 +55,10 @@ ifdef CONFIG_RETPOLINE+ PURGATORY_CFLAGS_REMOVE += $(RETPOLINE_CFLAGS)+ endif+++ifdef CONFIG_CFI_CLANG++PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_CFI)++endif+++ CFLAGS_REMOVE_purgatory.o += $(PURGATORY_CFLAGS_REMOVE)+ CFLAGS_purgatory.o += $(PURGATORY_CFLAGS)++diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c+index e2c5b296120d..e1d261ed4da8 100644+--- a/arch/x86/tools/relocs.c++++ b/arch/x86/tools/relocs.c+@@ -56,6 +56,7 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = {+ "^(xen_irq_disable_direct_reloc$|"+ "xen_save_fl_direct_reloc$|"+ "VDSO|"++ "__typeid__|"+ "__crc_)",++ /*+@@ -866,6 +867,12 @@ static int do_reloc64(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym,+ symname);+ break;+++ case R_X86_64_8:++ if (!shn_abs || !is_reloc(S_ABS, symname))++ die("Non-whitelisted %s relocation: %s\n",++ rel_type(r_type), symname);++ break;+++ case R_X86_64_32:+ case R_X86_64_32S:+ case R_X86_64_64:+diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c+index 5004feb16783..20c7fc1a660a 100644+--- a/arch/x86/xen/enlighten_pv.c++++ b/arch/x86/xen/enlighten_pv.c+@@ -583,8 +583,8 @@ DEFINE_IDTENTRY_RAW(xenpv_exc_machine_check)+ #endif++ struct trap_array_entry {+- void (*orig)(void);+- void (*xen)(void);++ asm_func_ptr orig;++ asm_func_ptr xen;+ bool ist_okay;+ };++@@ -643,7 +643,7 @@ static bool __ref get_trap_addr(void **addr, unsigned int ist)+ struct trap_array_entry *entry = trap_array + nr;++ if (*addr == entry->orig) {+- *addr = entry->xen;++ *addr = (void *)entry->xen;+ ist_okay = entry->ist_okay;+ found = true;+ break;+diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h+index fd0fec6e92f4..fc1afa5cd935 100644+--- a/arch/x86/xen/xen-ops.h++++ b/arch/x86/xen/xen-ops.h+@@ -8,12 +8,12 @@+ #include <xen/xen-ops.h>++ /* These are code, but not functions. Defined in entry.S */+-extern const char xen_failsafe_callback[];++DECLARE_NOT_CALLED_FROM_C(xen_failsafe_callback);++-void xen_sysenter_target(void);++DECLARE_NOT_CALLED_FROM_C(xen_sysenter_target);+ #ifdef CONFIG_X86_64+-void xen_syscall_target(void);+-void xen_syscall32_target(void);++DECLARE_NOT_CALLED_FROM_C(xen_syscall_target);++DECLARE_NOT_CALLED_FROM_C(xen_syscall32_target);+ #endif++ extern void *xen_initial_gdt;+@@ -137,7 +137,7 @@ __visible unsigned long xen_read_cr2(void);+ __visible unsigned long xen_read_cr2_direct(void);++ /* These are not functions, and cannot be called normally */+-__visible void xen_iret(void);++DECLARE_NOT_CALLED_FROM_C(xen_iret);++ extern int xen_panic_handler_init(void);++diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c+index f21854ac5cc2..7b4181720848 100644+--- a/drivers/misc/lkdtm/bugs.c++++ b/drivers/misc/lkdtm/bugs.c+@@ -446,7 +446,7 @@ void lkdtm_STACK_GUARD_PAGE_TRAILING(void)++ void lkdtm_UNSET_SMEP(void)+ {+-#if IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_UML)++#if IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_UML) && !IS_ENABLED(CONFIG_CFI_CLANG)+ #define MOV_CR4_DEPTH 64+ void (*direct_write_cr4)(unsigned long val);+ unsigned char *insn;+diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h+index d6137c70ebbe..867d2fcb1abe 100644+--- a/drivers/misc/lkdtm/lkdtm.h++++ b/drivers/misc/lkdtm/lkdtm.h+@@ -138,7 +138,7 @@ void lkdtm_REFCOUNT_TIMING(void);+ void lkdtm_ATOMIC_TIMING(void);++ /* rodata.c */+-void lkdtm_rodata_do_nothing(void);++DECLARE_NOT_CALLED_FROM_C(lkdtm_rodata_do_nothing);++ /* usercopy.c */+ void __init lkdtm_usercopy_init(void);+diff --git a/drivers/misc/lkdtm/perms.c b/drivers/misc/lkdtm/perms.c+index 2dede2ef658f..fa2bd90bd8ee 100644+--- a/drivers/misc/lkdtm/perms.c++++ b/drivers/misc/lkdtm/perms.c+@@ -151,7 +151,7 @@ void lkdtm_EXEC_VMALLOC(void)++ void lkdtm_EXEC_RODATA(void)+ {+- execute_location(lkdtm_rodata_do_nothing, CODE_AS_IS);++ execute_location((void *)lkdtm_rodata_do_nothing, CODE_AS_IS);+ }++ void lkdtm_EXEC_USERSPACE(void)+diff --git a/drivers/misc/lkdtm/rodata.c b/drivers/misc/lkdtm/rodata.c+index baacb876d1d9..17ed0ad4e6ae 100644+--- a/drivers/misc/lkdtm/rodata.c++++ b/drivers/misc/lkdtm/rodata.c+@@ -3,7 +3,7 @@+ * This includes functions that are meant to live entirely in .rodata+ * (via objcopy tricks), to validate the non-executability of .rodata.+ */+-#include "lkdtm.h"++void lkdtm_rodata_do_nothing(void);++ void noinstr lkdtm_rodata_do_nothing(void)+ {+diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h+index 42f3866bca69..b4d0be59c8d5 100644+--- a/include/asm-generic/vmlinux.lds.h++++ b/include/asm-generic/vmlinux.lds.h+@@ -413,6 +413,16 @@+ KEEP(*(.static_call_tramp_key)) \+ __stop_static_call_tramp_key = .;+++#ifdef CONFIG_CFI_CLANG++#define CFI_EXCLUDED_DATA \++ . = ALIGN(8); \++ __start_cfi_excluded = .; \++ KEEP(*(.cfi_excluded_stubs)) \++ __stop_cfi_excluded = .;++#else++#define CFI_EXCLUDED_DATA++#endif+++ /*+ * Allow architectures to handle ro_after_init data on their+ * own by defining an empty RO_AFTER_INIT_DATA.+@@ -436,6 +446,7 @@+ __start_rodata = .; \+ *(.rodata) *(.rodata.*) \+ SCHED_DATA \++ CFI_EXCLUDED_DATA \+ RO_AFTER_INIT_DATA /* Read only after init */ \+ . = ALIGN(8); \+ __start___tracepoints_ptrs = .; \+diff --git a/include/linux/cfi.h b/include/linux/cfi.h+index 879744aaa6e0..19f74af8eac2 100644+--- a/include/linux/cfi.h++++ b/include/linux/cfi.h+@@ -20,6 +20,17 @@ extern void __cfi_check(uint64_t id, void *ptr, void *diag);+ #define __CFI_ADDRESSABLE(fn, __attr) \+ const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn+++/*++ * Defines a stub function that returns immediately, and when defined and++ * referenced in the core kernel, always passes CFI checking. This should++ * be used only for stubs that cannot be called using the correct function++ * pointer type, which should be rare.++ */++#define DEFINE_CFI_IMMEDIATE_RETURN_STUB(fn) \++ void fn(void) { return; } \++ const void *__cfi_excl_ ## fn __visible \++ __section(".cfi_excluded_stubs") = (void *)&fn+++ #ifdef CONFIG_CFI_CLANG_SHADOW++ extern void cfi_module_add(struct module *mod, unsigned long base_addr);+@@ -35,6 +46,8 @@ static inline void cfi_module_remove(struct module *mod, unsigned long base_addr+ #else /* !CONFIG_CFI_CLANG */++ #define __CFI_ADDRESSABLE(fn, __attr)++#define DEFINE_CFI_IMMEDIATE_RETURN_STUB(fn) \++ void fn(void) { return; }++ #endif /* CONFIG_CFI_CLANG */++diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h+index 9999e29187de..d36aac8caeef 100644+--- a/include/linux/ftrace.h++++ b/include/linux/ftrace.h+@@ -603,9 +603,10 @@ extern void ftrace_replace_code(int enable);+ extern int ftrace_update_ftrace_func(ftrace_func_t func);+ extern void ftrace_caller(void);+ extern void ftrace_regs_caller(void);+-extern void ftrace_call(void);+-extern void ftrace_regs_call(void);+-extern void mcount_call(void);++++DECLARE_NOT_CALLED_FROM_C(ftrace_call);++DECLARE_NOT_CALLED_FROM_C(ftrace_regs_call);++DECLARE_NOT_CALLED_FROM_C(mcount_call);++ void ftrace_modify_all_code(int command);++diff --git a/include/linux/linkage.h b/include/linux/linkage.h+index dbf8506decca..f982d5f550ac 100644+--- a/include/linux/linkage.h++++ b/include/linux/linkage.h+@@ -48,6 +48,19 @@+ #define __PAGE_ALIGNED_DATA .section ".data..page_aligned", "aw"+ #define __PAGE_ALIGNED_BSS .section ".bss..page_aligned", "aw"+++/*++ * Declares a function not callable from C using an opaque type. Defined as++ * an array to allow the address of the symbol to be taken without '&'.++ */++#ifndef DECLARE_NOT_CALLED_FROM_C++#define DECLARE_NOT_CALLED_FROM_C(sym) \++ extern const u8 sym[]++#endif++++#ifndef __ASSEMBLY__++typedef const u8 *asm_func_ptr;++#endif+++ /*+ * This is used by architectures to keep arguments on the stack+ * untouched by the compiler by keeping them live until the end.+diff --git a/include/linux/objtool.h b/include/linux/objtool.h+index aca52db2f3f3..f3374fc3ff01 100644+--- a/include/linux/objtool.h++++ b/include/linux/objtool.h+@@ -66,6 +66,11 @@ struct unwind_hint {+ static void __used __section(".discard.func_stack_frame_non_standard") \+ *__func_stack_frame_non_standard_##func = func+++#define ASM_STACK_FRAME_NON_STANDARD(func) \++ ".pushsection .discard.func_stack_frame_non_standard, \"aw\"\n" \++ ".long " __stringify(func) " - .\n" \++ ".popsection\n"+++ /*+ * STACK_FRAME_NON_STANDARD_FP() is a frame-pointer-specific function ignore+ * for the case where a function is intentionally missing frame pointer setup,+@@ -138,6 +143,7 @@ struct unwind_hint {+ #define UNWIND_HINT(sp_reg, sp_offset, type, end) \+ "\n\t"+ #define STACK_FRAME_NON_STANDARD(func)++#define ASM_STACK_FRAME_NON_STANDARD(func)+ #define STACK_FRAME_NON_STANDARD_FP(func)+ #else+ #define ANNOTATE_INTRA_FUNCTION_CALL+diff --git a/kernel/cfi.c b/kernel/cfi.c+index 9594cfd1cf2c..8d931089141b 100644+--- a/kernel/cfi.c++++ b/kernel/cfi.c+@@ -278,12 +278,34 @@ static inline cfi_check_fn find_module_check_fn(unsigned long ptr)+ return fn;+ }+++extern unsigned long __start_cfi_excluded[];++extern unsigned long __stop_cfi_excluded[];++++static inline bool is_cfi_excluded(unsigned long ptr)++{++ unsigned long *p = __start_cfi_excluded;++++ for ( ; p < __stop_cfi_excluded; ++p)++ if (*p == ptr)++ return true;++++ return false;++}++++static void __cfi_pass(uint64_t id, void *ptr, void *diag)++{++}+++ static inline cfi_check_fn find_check_fn(unsigned long ptr)+ {+ cfi_check_fn fn = NULL;++- if (is_kernel_text(ptr))++ if (is_kernel_text(ptr)) {++ if (unlikely(is_cfi_excluded(ptr)))++ return __cfi_pass;+++ return __cfi_check;++ }++ /*+ * Indirect call checks can happen when RCU is not watching. Both+diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c+index 64ea283f2f86..8a0d463c8507 100644+--- a/kernel/tracepoint.c++++ b/kernel/tracepoint.c+@@ -9,6 +9,7 @@+ #include <linux/list.h>+ #include <linux/rcupdate.h>+ #include <linux/tracepoint.h>++#include <linux/cfi.h>+ #include <linux/err.h>+ #include <linux/slab.h>+ #include <linux/sched/signal.h>+@@ -99,10 +100,7 @@ struct tp_probes {+ };++ /* Called in removal of a func but failed to allocate a new tp_funcs */+-static void tp_stub_func(void)+-{+- return;+-}++static DEFINE_CFI_IMMEDIATE_RETURN_STUB(tp_stub_func);++ static inline void *allocate_probes(int count)+ {+diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h+index aca52db2f3f3..f3374fc3ff01 100644+--- a/tools/include/linux/objtool.h++++ b/tools/include/linux/objtool.h+@@ -66,6 +66,11 @@ struct unwind_hint {+ static void __used __section(".discard.func_stack_frame_non_standard") \+ *__func_stack_frame_non_standard_##func = func+++#define ASM_STACK_FRAME_NON_STANDARD(func) \++ ".pushsection .discard.func_stack_frame_non_standard, \"aw\"\n" \++ ".long " __stringify(func) " - .\n" \++ ".popsection\n"+++ /*+ * STACK_FRAME_NON_STANDARD_FP() is a frame-pointer-specific function ignore+ * for the case where a function is intentionally missing frame pointer setup,+@@ -138,6 +143,7 @@ struct unwind_hint {+ #define UNWIND_HINT(sp_reg, sp_offset, type, end) \+ "\n\t"+ #define STACK_FRAME_NON_STANDARD(func)++#define ASM_STACK_FRAME_NON_STANDARD(func)+ #define STACK_FRAME_NON_STANDARD_FP(func)+ #else+ #define ANNOTATE_INTRA_FUNCTION_CALL+diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c+index c10ef78df050..52436b0ae0d7 100644+--- a/tools/objtool/arch/x86/decode.c++++ b/tools/objtool/arch/x86/decode.c+@@ -63,6 +63,23 @@ bool arch_callee_saved_reg(unsigned char reg)+ }+ }+++unsigned long arch_cfi_section_reloc_offset(struct reloc *reloc)++{++ if (!reloc->addend)++ return 0;++++ if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)++ return reloc->addend + 4;++++ return reloc->addend;++}++++unsigned long arch_cfi_jump_reloc_offset(unsigned long offset)++{++ /* offset to the relocation in a jmp instruction */++ return offset + 1;++}+++ unsigned long arch_dest_reloc_offset(int addend)+ {+ return addend + 4;+diff --git a/tools/objtool/check.c b/tools/objtool/check.c+index c2d2ab9a2861..de98b9704ae4 100644+--- a/tools/objtool/check.c++++ b/tools/objtool/check.c+@@ -3351,9 +3351,6 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio+ if (!insn->func)+ return false;++- if (insn->func->static_call_tramp)+- return true;+-+ /*+ * CONFIG_UBSAN_TRAP inserts a UD2 when it sees+ * __builtin_unreachable(). The BUG() macro has an unreachable() after+diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c+index 4b384c907027..177f53007788 100644+--- a/tools/objtool/elf.c++++ b/tools/objtool/elf.c+@@ -18,6 +18,7 @@+ #include <errno.h>+ #include <objtool/builtin.h>+++#include <objtool/arch.h>+ #include <objtool/elf.h>+ #include <objtool/warn.h>++@@ -290,6 +291,10 @@ static int read_sections(struct elf *elf)+ if (sec->sh.sh_flags & SHF_EXECINSTR)+ elf->text_size += sec->sh.sh_size;+++ /* Detect -fsanitize=cfi jump table sections */++ if (!strncmp(sec->name, ".text..L.cfi.jumptable", 22))++ sec->cfi_jt = true;+++ list_add_tail(&sec->list, &elf->sections);+ elf_hash_add(section, &sec->hash, sec->idx);+ elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name));+@@ -577,6 +582,49 @@ static int read_rela_reloc(struct section *sec, int i, struct reloc *reloc, unsi+ return 0;+ }+++/*++ * CONFIG_CFI_CLANG replaces function relocations to refer to an intermediate++ * jump table. Undo the conversion so objtool can make sense of things.++ */++static int fix_cfi_relocs(const struct elf *elf)++{++ struct section *sec;++ struct reloc *reloc;++++ list_for_each_entry(sec, &elf->sections, list) {++ list_for_each_entry(reloc, &sec->reloc_list, list) {++ struct reloc *cfi_reloc;++ unsigned long offset;++++ if (!reloc->sym->sec->cfi_jt)++ continue;++++ if (reloc->sym->type == STT_SECTION)++ offset = arch_cfi_section_reloc_offset(reloc);++ else++ offset = reloc->sym->offset;++++ /*++ * The jump table immediately jumps to the actual function,++ * so look up the relocation there.++ */++ offset = arch_cfi_jump_reloc_offset(offset);++ cfi_reloc = find_reloc_by_dest(elf, reloc->sym->sec, offset);++++ if (!cfi_reloc || !cfi_reloc->sym) {++ WARN("can't find a CFI jump table relocation at %s+0x%lx",++ reloc->sym->sec->name, offset);++ return -1;++ }++++ reloc->sym = cfi_reloc->sym;++ reloc->addend = 0;++ }++ }++++ return 0;++}+++ static int read_relocs(struct elf *elf)+ {+ struct section *sec;+@@ -640,6 +688,9 @@ static int read_relocs(struct elf *elf)+ tot_reloc += nr_reloc;+ }+++ if (fix_cfi_relocs(elf))++ return -1;+++ if (stats) {+ printf("max_reloc: %lu\n", max_reloc);+ printf("tot_reloc: %lu\n", tot_reloc);+diff --git a/tools/objtool/include/objtool/arch.h b/tools/objtool/include/objtool/arch.h+index 76bae3078286..86fbf9d84033 100644+--- a/tools/objtool/include/objtool/arch.h++++ b/tools/objtool/include/objtool/arch.h+@@ -82,6 +82,9 @@ unsigned long arch_jump_destination(struct instruction *insn);++ unsigned long arch_dest_reloc_offset(int addend);+++unsigned long arch_cfi_section_reloc_offset(struct reloc *reloc);++unsigned long arch_cfi_jump_reloc_offset(unsigned long offset);+++ const char *arch_nop_insn(int len);+ const char *arch_ret_insn(int len);++diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h+index d22336781401..31c9be55b10e 100644+--- a/tools/objtool/include/objtool/elf.h++++ b/tools/objtool/include/objtool/elf.h+@@ -38,7 +38,7 @@ struct section {+ Elf_Data *data;+ char *name;+ int idx;+- bool changed, text, rodata, noinstr;++ bool changed, text, rodata, noinstr, cfi_jt;+ };++ struct symbol {+--+2.35.0++From 44aeb477991bdddef576e7eb93c5fb433b9ba48a Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sun, 23 Jan 2022 16:52:45 +0000+Subject: [PATCH] clearlinux++---+ arch/x86/include/asm/topology.h | 1 ++ arch/x86/include/asm/vdso/processor.h | 2 +-+ arch/x86/kernel/alternative.c | 2 ++ arch/x86/kernel/cpu/intel_epb.c | 4 +++ arch/x86/kernel/cpu/microcode/core.c | 9 +++-+ arch/x86/kernel/cpu/microcode/intel.c | 59 ++++++++++++++++++++++--+ arch/x86/kernel/itmt.c | 29 +++++++++++-+ arch/x86/kernel/tsc.c | 3 +++ arch/x86/mm/fault.c | 4 +-+ block/blk-core.c | 3 +++ drivers/Makefile | 15 +++---+ drivers/ata/libahci.c | 4 +-+ drivers/base/firmware_loader/main.c | 2 ++ drivers/cpufreq/intel_pstate.c | 7 ++++ drivers/idle/intel_idle.c | 44 +++++++++---------+ drivers/input/serio/i8042.c | 10 ++--+ drivers/net/dummy.c | 2 +-+ drivers/nvme/host/core.c | 2 +-+ drivers/pci/pci.c | 2 +-+ drivers/powercap/intel_rapl_common.c | 2 +-+ drivers/thermal/intel/intel_powerclamp.c | 10 +++++ fs/xattr.c | 15 +++---+ include/linux/jbd2.h | 2 +-+ include/linux/sched.h | 1 ++ include/linux/wait.h | 2 ++ include/uapi/linux/if_bonding.h | 2 +-+ init/do_mounts.c | 16 ++++++-+ kernel/locking/rwsem.c | 4 +-+ kernel/module.c | 20 +++++++++ kernel/sched/debug.c | 1 ++ kernel/sched/wait.c | 24 +++++++++++ kernel/watchdog.c | 2 +-+ mm/compaction.c | 2 +-+ mm/ksm.c | 11 +++--+ mm/page_alloc.c | 5 +-+ net/ipv4/inet_connection_sock.c | 2 +-+ net/ipv4/tcp.c | 4 +-+ 37 files changed, 258 insertions(+), 71 deletions(-)++diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h+index 2f0b6be8eaab..c31f81e2ea05 100644+--- a/arch/x86/include/asm/topology.h++++ b/arch/x86/include/asm/topology.h+@@ -174,6 +174,7 @@ extern unsigned int __read_mostly sysctl_sched_itmt_enabled;++ /* Interface to set priority of a cpu */+ void sched_set_itmt_core_prio(int prio, int core_cpu);++void sched_set_itmt_power_ratio(int power_ratio, int core_cpu);++ /* Interface to notify scheduler that system supports ITMT */+ int sched_set_itmt_support(void);+diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h+index 57b1a7034c64..e2c45674f989 100644+--- a/arch/x86/include/asm/vdso/processor.h++++ b/arch/x86/include/asm/vdso/processor.h+@@ -10,7 +10,7 @@+ /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */+ static __always_inline void rep_nop(void)+ {+- asm volatile("rep; nop" ::: "memory");++ asm volatile("lfence" ::: "memory");+ }++ static __always_inline void cpu_relax(void)+diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c+index 5007c3ffe96f..c8d04206b2e9 100644+--- a/arch/x86/kernel/alternative.c++++ b/arch/x86/kernel/alternative.c+@@ -829,7 +829,9 @@ void __init alternative_instructions(void)+ * Then patch alternatives, such that those paravirt calls that are in+ * alternatives can be overwritten by their immediate fragments.+ */++ printk("clr: Applying alternatives\n");+ apply_alternatives(__alt_instructions, __alt_instructions_end);++ printk("clr: Applying alternatives done\n");++ #ifdef CONFIG_SMP+ /* Patch to UP if other cpus not imminent. */+diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c+index fbaf12e43f41..c8c2d6f1a8ac 100644+--- a/arch/x86/kernel/cpu/intel_epb.c++++ b/arch/x86/kernel/cpu/intel_epb.c+@@ -166,6 +166,10 @@ static ssize_t energy_perf_bias_store(struct device *dev,+ if (ret < 0)+ return ret;+++ /* update the ITMT scheduler logic to use the power policy data */++ /* scale the val up by 2 so the range is 224 - 256 */++ sched_set_itmt_power_ratio(256 - val * 2, cpu);+++ return count;+ }++diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c+index f955d25076ba..fb3925d1f788 100644+--- a/arch/x86/kernel/cpu/microcode/core.c++++ b/arch/x86/kernel/cpu/microcode/core.c+@@ -587,6 +587,7 @@ static int microcode_reload_late(void)+ atomic_set(&late_cpus_in, 0);+ atomic_set(&late_cpus_out, 0);+++ printk ("Going to do stop_machine\n");+ ret = stop_machine_cpuslocked(__reload_late, NULL, cpu_online_mask);+ if (ret == 0)+ microcode_check();+@@ -600,18 +601,23 @@ static ssize_t reload_store(struct device *dev,+ struct device_attribute *attr,+ const char *buf, size_t size)+ {++ extern bool force_ucode_load;+ enum ucode_state tmp_ret = UCODE_OK;+ int bsp = boot_cpu_data.cpu_index;+ unsigned long val;++ bool orig_cmd_line = force_ucode_load;+ ssize_t ret = 0;++ ret = kstrtoul(buf, 0, &val);+ if (ret)+ return ret;++- if (val != 1)++ if (!val || val > 2)+ return size;+++ if (val == 2)++ force_ucode_load = true;+++ cpus_read_lock();++ ret = check_online_cpus();+@@ -627,6 +633,7 @@ static ssize_t reload_store(struct device *dev,+ mutex_unlock(µcode_mutex);++ put:++ force_ucode_load = orig_cmd_line;+ cpus_read_unlock();++ if (ret == 0)+diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c+index d28a9f8f3fec..4cd155b0a42f 100644+--- a/arch/x86/kernel/cpu/microcode/intel.c++++ b/arch/x86/kernel/cpu/microcode/intel.c+@@ -30,6 +30,7 @@+ #include <linux/uio.h>+ #include <linux/mm.h>+++#include <asm/cmdline.h>+ #include <asm/microcode_intel.h>+ #include <asm/intel-family.h>+ #include <asm/processor.h>+@@ -38,6 +39,7 @@+ #include <asm/msr.h>++ static const char ucode_path[] = "kernel/x86/microcode/GenuineIntel.bin";++bool force_ucode_load = false;++ /* Current microcode patch used in early patching on the APs. */+ static struct microcode_intel *intel_ucode_patch;+@@ -94,8 +96,18 @@ static int has_newer_microcode(void *mc, unsigned int csig, int cpf, int new_rev+ {+ struct microcode_header_intel *mc_hdr = mc;++- if (mc_hdr->rev <= new_rev)++ //if (mc_hdr->rev <= new_rev)++ if (mc_hdr->rev < new_rev) {++ printk ("Returning NO_NEW old = 0x%x new = 0x%x\n",++ mc_hdr->rev, new_rev);+ return 0;++ }++ if ((mc_hdr->rev == new_rev) && !force_ucode_load) {++ printk ("SAME REV: no_force Returning NO_NEW old = 0x%x new = 0x%x\n",++ mc_hdr->rev, new_rev);++ return 0;++ }++ printk ("ucode: force loading same rev\n");++ return find_matching_signature(mc, csig, cpf);+ }+@@ -557,11 +569,20 @@ static int apply_microcode_early(struct ucode_cpu_info *uci, bool early)+ * already.+ */+ rev = intel_get_microcode_revision();+- if (rev >= mc->hdr.rev) {++ if (rev > mc->hdr.rev) {+ uci->cpu_sig.rev = rev;+ return UCODE_OK;+ }+++ if (rev == mc->hdr.rev) {++ if (!force_ucode_load) {++ printk ("Matching ucode rev, no update\n");++ return UCODE_OK;++ } else {++ printk ("Matching ucode rev.. force updating\n");++ }++ }+++ /*+ * Writeback and invalidate caches before updating microcode to avoid+ * internal issues depending on what the microcode is updating.+@@ -613,6 +634,29 @@ int __init save_microcode_in_initrd_intel(void)+ return 0;+ }+++static bool check_force_ucode_bsp(void)++{++ static const char *__force_ucode_str = "force_ucode_load";++++#ifdef CONFIG_X86_32++ const char *cmdline = (const char *)__pa_nodebug(boot_command_line);++ const char *option = (const char *)__pa_nodebug(__force_ucode_str);++ bool *res = (bool *)__pa_nodebug(&force_ucode_load);++++#else /* CONFIG_X86_64 */++ const char *cmdline = boot_command_line;++ const char *option = __force_ucode_str;++ bool *res = &force_ucode_load;++#endif++++ if (cmdline_find_option_bool(cmdline, option)) {++ printk("cmdline forcing ucode update for same rev\n");++ *res = true;++ }++++ return *res;++}+++ /*+ * @res_patch, output: a pointer to the patch we found.+ */+@@ -646,6 +690,9 @@ void __init load_ucode_intel_bsp(void)+ {+ struct microcode_intel *patch;+ struct ucode_cpu_info uci;++ bool force_bsp;++++ force_bsp = check_force_ucode_bsp();++ patch = __load_ucode_intel(&uci);+ if (!patch)+@@ -694,8 +741,12 @@ static struct microcode_intel *find_patch(struct ucode_cpu_info *uci)++ phdr = (struct microcode_header_intel *)iter->data;++- if (phdr->rev <= uci->cpu_sig.rev)++ if (phdr->rev < uci->cpu_sig.rev)+ continue;++ if (phdr->rev == uci->cpu_sig.rev && !force_ucode_load)++ continue;++ else++ printk ("same rev forcing ucode\n");++ if (!find_matching_signature(phdr,+ uci->cpu_sig.sig,+@@ -779,7 +830,7 @@ static enum ucode_state apply_microcode_intel(int cpu)+ * already.+ */+ rev = intel_get_microcode_revision();+- if (rev >= mc->hdr.rev) {++ if (rev > mc->hdr.rev || (rev == mc->hdr.rev && !force_ucode_load)) {+ ret = UCODE_OK;+ goto out;+ }+diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c+index 9ff480e94511..d4326e050fb7 100644+--- a/arch/x86/kernel/itmt.c++++ b/arch/x86/kernel/itmt.c+@@ -25,6 +25,7 @@++ static DEFINE_MUTEX(itmt_update_mutex);+ DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority);++DEFINE_PER_CPU_READ_MOSTLY(int, sched_power_ratio);++ /* Boolean to track if system has ITMT capabilities */+ static bool __read_mostly sched_itmt_capable;+@@ -169,7 +170,12 @@ void sched_clear_itmt_support(void)++ int arch_asym_cpu_priority(int cpu)+ {+- return per_cpu(sched_core_priority, cpu);++ int power_ratio = per_cpu(sched_power_ratio, cpu);++++ /* a power ratio of 0 (uninitialized) is assumed to be maximum */++ if (power_ratio == 0)++ power_ratio = 256 - 2 * 6;++ return per_cpu(sched_core_priority, cpu) * power_ratio / 256;+ }++ /**+@@ -203,3 +209,24 @@ void sched_set_itmt_core_prio(int prio, int core_cpu)+ i++;+ }+ }++++/**++ * sched_set_itmt_power_ratio() - Set CPU priority based on ITMT++ * @power_ratio: The power scaling ratio [1..256] for the core++ * @core_cpu: The cpu number associated with the core++ *++ * Set a scaling to the cpu performance based on long term power++ * settings (like EPB).++ *++ * Note this is for the policy not for the actual dynamic frequency;++ * the frequency will increase itself as workloads run on a core.++ */++++void sched_set_itmt_power_ratio(int power_ratio, int core_cpu)++{++ int cpu;++++ for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) {++ per_cpu(sched_power_ratio, cpu) = power_ratio;++ }++}+diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c+index a698196377be..5f3ee7c31c8a 100644+--- a/arch/x86/kernel/tsc.c++++ b/arch/x86/kernel/tsc.c+@@ -1569,6 +1569,9 @@ unsigned long calibrate_delay_is_known(void)+ if (!constant_tsc || !mask)+ return 0;+++ if (cpu != 0)++ return cpu_data(0).loops_per_jiffy;+++ sibling = cpumask_any_but(mask, cpu);+ if (sibling < nr_cpu_ids)+ return cpu_data(sibling).loops_per_jiffy;+diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c+index d0074c6ed31a..aeeae77fe5dd 100644+--- a/arch/x86/mm/fault.c++++ b/arch/x86/mm/fault.c+@@ -776,9 +776,9 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code,+ if (!printk_ratelimit())+ return;++- printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx",++ printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx cpu %i",+ loglvl, tsk->comm, task_pid_nr(tsk), address,+- (void *)regs->ip, (void *)regs->sp, error_code);++ (void *)regs->ip, (void *)regs->sp, error_code, raw_smp_processor_id());++ print_vma_addr(KERN_CONT " in ", regs->ip);++diff --git a/block/blk-core.c b/block/blk-core.c+index 97f8bc8d3a79..004628768c61 100644+--- a/block/blk-core.c++++ b/block/blk-core.c+@@ -946,6 +946,9 @@ void submit_bio(struct bio *bio)+ } else {+ task_io_account_read(bio->bi_iter.bi_size);+ count_vm_events(PGPGIN, count);++++ if (bio->bi_opf & REQ_PREFLUSH)++ current->fsync_count++;+ }+ }++diff --git a/drivers/Makefile b/drivers/Makefile+index a110338c860c..f91099276a78 100644+--- a/drivers/Makefile++++ b/drivers/Makefile+@@ -59,15 +59,8 @@ obj-y += char/+ # iommu/ comes before gpu as gpu are using iommu controllers+ obj-y += iommu/++-# gpu/ comes after char for AGP vs DRM startup and after iommu+-obj-y += gpu/+-+ obj-$(CONFIG_CONNECTOR) += connector/++-# i810fb and intelfb depend on char/agp/+-obj-$(CONFIG_FB_I810) += video/fbdev/i810/+-obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/+-+ obj-$(CONFIG_PARPORT) += parport/+ obj-y += base/ block/ misc/ mfd/ nfc/+ obj-$(CONFIG_LIBNVDIMM) += nvdimm/+@@ -79,6 +72,14 @@ obj-y += macintosh/+ obj-y += scsi/+ obj-y += nvme/+ obj-$(CONFIG_ATA) += ata/++++# gpu/ comes after char for AGP vs DRM startup and after iommu++obj-y += gpu/++++# i810fb and intelfb depend on char/agp/++obj-$(CONFIG_FB_I810) += video/fbdev/i810/++obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/+++ obj-$(CONFIG_TARGET_CORE) += target/+ obj-$(CONFIG_MTD) += mtd/+ obj-$(CONFIG_SPI) += spi/+diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c+index 0ed484e04fd6..bbcb54da7a31 100644+--- a/drivers/ata/libahci.c++++ b/drivers/ata/libahci.c+@@ -33,14 +33,14 @@+ #include "libata.h"++ static int ahci_skip_host_reset;+-int ahci_ignore_sss;++int ahci_ignore_sss=1;+ EXPORT_SYMBOL_GPL(ahci_ignore_sss);++ module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);+ MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");++ module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);+-MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");++MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore [default])");++ static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,+ unsigned hints);+diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c+index 94d1789a233e..678b81768bd9 100644+--- a/drivers/base/firmware_loader/main.c++++ b/drivers/base/firmware_loader/main.c+@@ -407,6 +407,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv,+ static char fw_path_para[256];+ static const char * const fw_path[] = {+ fw_path_para,++ "/etc/firmware/" UTS_RELEASE,++ "/etc/firmware",+ "/lib/firmware/updates/" UTS_RELEASE,+ "/lib/firmware/updates",+ "/lib/firmware/" UTS_RELEASE,+diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c+index bc7f7e6759bd..ee33ad7f6f28 100644+--- a/drivers/cpufreq/intel_pstate.c++++ b/drivers/cpufreq/intel_pstate.c+@@ -364,6 +364,13 @@ static void intel_pstate_set_itmt_prio(int cpu)+ * update them at any time after it has been called.+ */+ sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu);++ /*++ * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff.++ * In this case we can't use CPPC.highest_perf to enable ITMT.++ * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide.++ */++ if (cppc_perf.highest_perf == 0xff)++ cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached));++ if (max_highest_perf <= min_highest_perf) {+ if (cppc_perf.highest_perf > max_highest_perf)+diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c+index 0b66e25c0e2d..406b41b387d5 100644+--- a/drivers/idle/intel_idle.c++++ b/drivers/idle/intel_idle.c+@@ -511,7 +511,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {+ .desc = "MWAIT 0x01",+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,+ .exit_latency = 10,+- .target_residency = 20,++ .target_residency = 120,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -519,7 +519,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {+ .desc = "MWAIT 0x10",+ .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 33,+- .target_residency = 100,++ .target_residency = 900,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -527,7 +527,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {+ .desc = "MWAIT 0x20",+ .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 133,+- .target_residency = 400,++ .target_residency = 1000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -535,7 +535,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {+ .desc = "MWAIT 0x32",+ .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 166,+- .target_residency = 500,++ .target_residency = 1500,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -543,7 +543,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {+ .desc = "MWAIT 0x40",+ .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 300,+- .target_residency = 900,++ .target_residency = 2000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -551,7 +551,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {+ .desc = "MWAIT 0x50",+ .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 600,+- .target_residency = 1800,++ .target_residency = 5000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -559,7 +559,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {+ .desc = "MWAIT 0x60",+ .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 2600,+- .target_residency = 7700,++ .target_residency = 9000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -579,7 +579,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {+ .desc = "MWAIT 0x01",+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,+ .exit_latency = 10,+- .target_residency = 20,++ .target_residency = 120,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -587,7 +587,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {+ .desc = "MWAIT 0x10",+ .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 40,+- .target_residency = 100,++ .target_residency = 1000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -595,7 +595,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {+ .desc = "MWAIT 0x20",+ .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 133,+- .target_residency = 400,++ .target_residency = 1000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -603,7 +603,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {+ .desc = "MWAIT 0x32",+ .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 166,+- .target_residency = 500,++ .target_residency = 2000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -611,7 +611,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {+ .desc = "MWAIT 0x40",+ .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 300,+- .target_residency = 900,++ .target_residency = 4000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -619,7 +619,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {+ .desc = "MWAIT 0x50",+ .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 600,+- .target_residency = 1800,++ .target_residency = 7000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -627,7 +627,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {+ .desc = "MWAIT 0x60",+ .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 2600,+- .target_residency = 7700,++ .target_residency = 9000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -648,7 +648,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {+ .desc = "MWAIT 0x01",+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,+ .exit_latency = 10,+- .target_residency = 20,++ .target_residency = 120,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -656,7 +656,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {+ .desc = "MWAIT 0x10",+ .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 70,+- .target_residency = 100,++ .target_residency = 1000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -664,7 +664,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {+ .desc = "MWAIT 0x20",+ .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 85,+- .target_residency = 200,++ .target_residency = 600,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -672,7 +672,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {+ .desc = "MWAIT 0x33",+ .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 124,+- .target_residency = 800,++ .target_residency = 3000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -680,7 +680,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {+ .desc = "MWAIT 0x40",+ .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 200,+- .target_residency = 800,++ .target_residency = 3200,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -688,7 +688,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {+ .desc = "MWAIT 0x50",+ .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 480,+- .target_residency = 5000,++ .target_residency = 9000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -696,7 +696,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {+ .desc = "MWAIT 0x60",+ .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,+ .exit_latency = 890,+- .target_residency = 5000,++ .target_residency = 9000,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+@@ -717,7 +717,7 @@ static struct cpuidle_state skx_cstates[] __initdata = {+ .desc = "MWAIT 0x01",+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,+ .exit_latency = 10,+- .target_residency = 20,++ .target_residency = 300,+ .enter = &intel_idle,+ .enter_s2idle = intel_idle_s2idle, },+ {+diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c+index 3fc0a89cc785..a7c103f9dfd3 100644+--- a/drivers/input/serio/i8042.c++++ b/drivers/input/serio/i8042.c+@@ -621,7 +621,7 @@ static int i8042_enable_kbd_port(void)+ if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) {+ i8042_ctr &= ~I8042_CTR_KBDINT;+ i8042_ctr |= I8042_CTR_KBDDIS;+- pr_err("Failed to enable KBD port\n");++ pr_info("Failed to enable KBD port\n");+ return -EIO;+ }++@@ -640,7 +640,7 @@ static int i8042_enable_aux_port(void)+ if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) {+ i8042_ctr &= ~I8042_CTR_AUXINT;+ i8042_ctr |= I8042_CTR_AUXDIS;+- pr_err("Failed to enable AUX port\n");++ pr_info("Failed to enable AUX port\n");+ return -EIO;+ }++@@ -732,7 +732,7 @@ static int i8042_check_mux(void)+ i8042_ctr &= ~I8042_CTR_AUXINT;++ if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) {+- pr_err("Failed to disable AUX port, can't use MUX\n");++ pr_info("Failed to disable AUX port, can't use MUX\n");+ return -EIO;+ }++@@ -955,7 +955,7 @@ static int i8042_controller_selftest(void)+ do {++ if (i8042_command(¶m, I8042_CMD_CTL_TEST)) {+- pr_err("i8042 controller selftest timeout\n");++ pr_info("i8042 controller selftest timeout\n");+ return -ENODEV;+ }++@@ -977,7 +977,7 @@ static int i8042_controller_selftest(void)+ pr_info("giving up on controller selftest, continuing anyway...\n");+ return 0;+ #else+- pr_err("i8042 controller selftest failed\n");++ pr_info("i8042 controller selftest failed\n");+ return -EIO;+ #endif+ }+diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c+index f82ad7419508..5e8faa70aad6 100644+--- a/drivers/net/dummy.c++++ b/drivers/net/dummy.c+@@ -43,7 +43,7 @@++ #define DRV_NAME "dummy"++-static int numdummies = 1;++static int numdummies = 0;++ /* fake multicast ability */+ static void set_multicast_list(struct net_device *dev)+diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c+index 5e0bfda04bd7..3024a992d5ac 100644+--- a/drivers/nvme/host/core.c++++ b/drivers/nvme/host/core.c+@@ -48,7 +48,7 @@ static u8 nvme_max_retries = 5;+ module_param_named(max_retries, nvme_max_retries, byte, 0644);+ MODULE_PARM_DESC(max_retries, "max number of retries a command may have");++-static unsigned long default_ps_max_latency_us = 100000;++static unsigned long default_ps_max_latency_us = 200;+ module_param(default_ps_max_latency_us, ulong, 0644);+ MODULE_PARM_DESC(default_ps_max_latency_us,+ "max power saving latency for new devices; use PM QOS to change per device");+diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c+index 9ecce435fb3f..796f280eccfe 100644+--- a/drivers/pci/pci.c++++ b/drivers/pci/pci.c+@@ -60,7 +60,7 @@ struct pci_pme_device {+ struct pci_dev *dev;+ };++-#define PME_TIMEOUT 1000 /* How long between PME checks */++#define PME_TIMEOUT 4000 /* How long between PME checks */++ static void pci_dev_d3_sleep(struct pci_dev *dev)+ {+diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c+index 07611a00b78f..4031d810def5 100644+--- a/drivers/powercap/intel_rapl_common.c++++ b/drivers/powercap/intel_rapl_common.c+@@ -1512,7 +1512,7 @@ static int __init rapl_init(void)++ id = x86_match_cpu(rapl_ids);+ if (!id) {+- pr_err("driver does not support CPU family %d model %d\n",++ pr_info("driver does not support CPU family %d model %d\n",+ boot_cpu_data.x86, boot_cpu_data.x86_model);++ return -ENODEV;+diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c+index 14256421d98c..8df2e604ceef 100644+--- a/drivers/thermal/intel/intel_powerclamp.c++++ b/drivers/thermal/intel/intel_powerclamp.c+@@ -647,6 +647,11 @@ static const struct thermal_cooling_device_ops powerclamp_cooling_ops = {+ .set_cur_state = powerclamp_set_cur_state,+ };+++static const struct x86_cpu_id amd_cpu[] = {++ { X86_VENDOR_AMD },++ {},++};+++ static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = {+ X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL),+ {}+@@ -656,6 +661,11 @@ MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids);+ static int __init powerclamp_probe(void)+ {+++ if (x86_match_cpu(amd_cpu)){++ pr_info("Intel PowerClamp does not support AMD CPUs\n");++ return -ENODEV;++ }+++ if (!x86_match_cpu(intel_powerclamp_ids)) {+ pr_err("CPU does not support MWAIT\n");+ return -ENODEV;+diff --git a/fs/xattr.c b/fs/xattr.c+index 5c8c5175b385..0f98df2ea2e2 100644+--- a/fs/xattr.c++++ b/fs/xattr.c+@@ -120,16 +120,17 @@ xattr_permission(struct user_namespace *mnt_userns, struct inode *inode,+ }++ /*+- * In the user.* namespace, only regular files and directories can have+- * extended attributes. For sticky directories, only the owner and+- * privileged users can write attributes.++ * In the user.* namespace, only regular files, symbolic links, and++ * directories can have extended attributes. For symbolic links and++ * sticky directories, only the owner and privileged users can write++ * attributes.+ */+ if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) {+- if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))++ if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode))+ return (mask & MAY_WRITE) ? -EPERM : -ENODATA;+- if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) &&+- (mask & MAY_WRITE) &&+- !inode_owner_or_capable(mnt_userns, inode))++ if (((S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX))++ || S_ISLNK(inode->i_mode)) && (mask & MAY_WRITE)++ && !inode_owner_or_capable(mnt_userns, inode))+ return -EPERM;+ }++diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h+index fd933c45281a..e70409a77cfa 100644+--- a/include/linux/jbd2.h++++ b/include/linux/jbd2.h+@@ -45,7 +45,7 @@+ /*+ * The default maximum commit age, in seconds.+ */+-#define JBD2_DEFAULT_MAX_COMMIT_AGE 5++#define JBD2_DEFAULT_MAX_COMMIT_AGE 30++ #ifdef CONFIG_JBD2_DEBUG+ /*+diff --git a/include/linux/sched.h b/include/linux/sched.h+index 508b91d57470..c1b38df8ac72 100644+--- a/include/linux/sched.h++++ b/include/linux/sched.h+@@ -1050,6 +1050,7 @@ struct task_struct {+ /* Cached requested key. */+ struct key *cached_requested_key;+ #endif++ int fsync_count;++ /*+ * executable name, excluding path.+diff --git a/include/linux/wait.h b/include/linux/wait.h+index 851e07da2583..85653fc33274 100644+--- a/include/linux/wait.h++++ b/include/linux/wait.h+@@ -165,6 +165,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)++ extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);+ extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);++extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);+ extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);+ extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);++@@ -1163,6 +1164,7 @@ do { \+ */+ void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);+ bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);++void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);+ long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);+ void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);+ long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);+diff --git a/include/uapi/linux/if_bonding.h b/include/uapi/linux/if_bonding.h+index d174914a837d..bf8e2af101a3 100644+--- a/include/uapi/linux/if_bonding.h++++ b/include/uapi/linux/if_bonding.h+@@ -82,7 +82,7 @@+ #define BOND_STATE_ACTIVE 0 /* link is active */+ #define BOND_STATE_BACKUP 1 /* link is backup */++-#define BOND_DEFAULT_MAX_BONDS 1 /* Default maximum number of devices to support */++#define BOND_DEFAULT_MAX_BONDS 0 /* Default maximum number of devices to support */++ #define BOND_DEFAULT_TX_QUEUES 16 /* Default number of tx queues per device */++diff --git a/init/do_mounts.c b/init/do_mounts.c+index 762b534978d9..107b96927049 100644+--- a/init/do_mounts.c++++ b/init/do_mounts.c+@@ -284,8 +284,18 @@ dev_t name_to_dev_t(const char *name)+ if (strcmp(name, "/dev/ram") == 0)+ return Root_RAM0;+ #ifdef CONFIG_BLOCK+- if (strncmp(name, "PARTUUID=", 9) == 0)+- return devt_from_partuuid(name + 9);++ if (strncmp(name, "PARTUUID=", 9) == 0) {++ dev_t res;++ int needtowait = 40<<1;++ res = devt_from_partuuid(name + 9);++ while (!res && needtowait) {++ /* waiting 0.5 sec */++ msleep(500);++ res = devt_from_partuuid(name + 9);++ needtowait--;++ }++ return res;++ }+ if (strncmp(name, "PARTLABEL=", 10) == 0)+ return devt_from_partlabel(name + 10);+ if (strncmp(name, "/dev/", 5) == 0)+@@ -613,7 +623,9 @@ void __init prepare_namespace(void)+ * For example, it is not atypical to wait 5 seconds here+ * for the touchpad of a laptop to initialize.+ */++ async_synchronize_full();+ wait_for_device_probe();++ async_synchronize_full();++ md_run_setup();++diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c+index 69aba4abe104..b4818ccad5de 100644+--- a/kernel/locking/rwsem.c++++ b/kernel/locking/rwsem.c+@@ -707,6 +707,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem)+ struct task_struct *new, *owner;+ unsigned long flags, new_flags;+ enum owner_state state;++ int i = 0;++ lockdep_assert_preemption_disabled();++@@ -743,7 +744,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem)+ break;+ }++- cpu_relax();++ if (i++ > 1000)++ cpu_relax();+ }++ return state;+diff --git a/kernel/module.c b/kernel/module.c+index 24dab046e16c..2ac7eea9f93b 100644+--- a/kernel/module.c++++ b/kernel/module.c+@@ -57,6 +57,7 @@+ #include <linux/bsearch.h>+ #include <linux/dynamic_debug.h>+ #include <linux/audit.h>++#include <linux/efi.h>+ #include <uapi/linux/module.h>+ #include "module-internal.h"++@@ -270,6 +271,10 @@ static void module_assert_mutex_or_preempt(void)+ #ifdef CONFIG_MODULE_SIG+ static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE);+ module_param(sig_enforce, bool_enable_only, 0644);++/* Allow disabling module signature requirement by adding boot param */++static bool sig_unenforce = false;++module_param(sig_unenforce, bool_enable_only, 0644);++++ void set_module_sig_enforced(void)+ {+@@ -416,6 +421,8 @@ extern const struct kernel_symbol __stop___ksymtab_gpl[];+ extern const s32 __start___kcrctab[];+ extern const s32 __start___kcrctab_gpl[];+++extern struct boot_params boot_params;+++ #ifndef CONFIG_MODVERSIONS+ #define symversion(base, idx) NULL+ #else+@@ -4676,6 +4683,19 @@ static const struct proc_ops modules_proc_ops = {+ static int __init proc_modules_init(void)+ {+ proc_create("modules", 0, NULL, &modules_proc_ops);++++#ifdef CONFIG_MODULE_SIG_FORCE++ switch (boot_params.secure_boot) {++ case efi_secureboot_mode_unset:++ case efi_secureboot_mode_unknown:++ case efi_secureboot_mode_disabled:++ /*++ * sig_unenforce is only applied if SecureBoot is not++ * enabled.++ */++ sig_enforce = !sig_unenforce;++ }++#endif+ return 0;+ }+ module_init(proc_modules_init);+diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c+index aa29211de1bf..22bbc3687cdf 100644+--- a/kernel/sched/debug.c++++ b/kernel/sched/debug.c+@@ -970,6 +970,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,+ PN(se.exec_start);+ PN(se.vruntime);+ PN(se.sum_exec_runtime);++ P(fsync_count);++ nr_switches = p->nvcsw + p->nivcsw;++diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c+index eca38107b32f..0306fa23b4f8 100644+--- a/kernel/sched/wait.c++++ b/kernel/sched/wait.c+@@ -48,6 +48,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_+ }+ EXPORT_SYMBOL_GPL(add_wait_queue_priority);+++void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)++{++ unsigned long flags;++++ wq_entry->flags |= WQ_FLAG_EXCLUSIVE;++ spin_lock_irqsave(&wq_head->lock, flags);++ __add_wait_queue(wq_head, wq_entry);++ spin_unlock_irqrestore(&wq_head->lock, flags);++}++EXPORT_SYMBOL(add_wait_queue_exclusive_lifo);+++ void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)+ {+ unsigned long flags;+@@ -290,6 +301,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent+ }+ EXPORT_SYMBOL(prepare_to_wait_exclusive);+++void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)++{++ unsigned long flags;++++ wq_entry->flags |= WQ_FLAG_EXCLUSIVE;++ spin_lock_irqsave(&wq_head->lock, flags);++ if (list_empty(&wq_entry->entry))++ __add_wait_queue(wq_head, wq_entry);++ set_current_state(state);++ spin_unlock_irqrestore(&wq_head->lock, flags);++}++EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo);+++ void init_wait_entry(struct wait_queue_entry *wq_entry, int flags)+ {+ wq_entry->flags = flags;+diff --git a/kernel/watchdog.c b/kernel/watchdog.c+index 99afb88d2e85..e24483521e17 100644+--- a/kernel/watchdog.c++++ b/kernel/watchdog.c+@@ -41,7 +41,7 @@ unsigned long __read_mostly watchdog_enabled;+ int __read_mostly watchdog_user_enabled = 1;+ int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT;+ int __read_mostly soft_watchdog_user_enabled = 1;+-int __read_mostly watchdog_thresh = 10;++int __read_mostly watchdog_thresh = 40;+ static int __read_mostly nmi_watchdog_available;++ struct cpumask watchdog_cpumask __read_mostly;+diff --git a/mm/compaction.c b/mm/compaction.c+index b4e94cda3019..e9a36942c1fa 100644+--- a/mm/compaction.c++++ b/mm/compaction.c+@@ -53,7 +53,7 @@ static inline void count_compact_events(enum vm_event_item item, long delta)+ /*+ * Fragmentation score check interval for proactive compaction purposes.+ */+-static const unsigned int HPAGE_FRAG_CHECK_INTERVAL_MSEC = 500;++static const unsigned int HPAGE_FRAG_CHECK_INTERVAL_MSEC = 5000;++ /*+ * Page order with-respect-to which proactive compaction+diff --git a/mm/ksm.c b/mm/ksm.c+index c20bd4d9a0d9..702a2a111fef 100644+--- a/mm/ksm.c++++ b/mm/ksm.c+@@ -2421,9 +2421,14 @@ static int ksm_scan_thread(void *nothing)++ if (ksmd_should_run()) {+ sleep_ms = READ_ONCE(ksm_thread_sleep_millisecs);+- wait_event_interruptible_timeout(ksm_iter_wait,+- sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs),+- msecs_to_jiffies(sleep_ms));++ if (sleep_ms >= 1000)++ wait_event_interruptible_timeout(ksm_iter_wait,++ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs),++ msecs_to_jiffies(round_jiffies_relative(sleep_ms)));++ else++ wait_event_interruptible_timeout(ksm_iter_wait,++ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs),++ msecs_to_jiffies(sleep_ms));+ } else {+ wait_event_freezable(ksm_thread_wait,+ ksmd_should_run() || kthread_should_stop());+diff --git a/mm/page_alloc.c b/mm/page_alloc.c+index 3589febc6d31..aa5f6d702694 100644+--- a/mm/page_alloc.c++++ b/mm/page_alloc.c+@@ -6836,11 +6836,11 @@ static int zone_batchsize(struct zone *zone)++ /*+ * The number of pages to batch allocate is either ~0.1%+- * of the zone or 1MB, whichever is smaller. The batch++ * of the zone or 4MB, whichever is smaller. The batch+ * size is striking a balance between allocation latency+ * and zone lock contention.+ */+- batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE);++ batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE);+ batch /= 4; /* We effectively *= 4 below */+ if (batch < 1)+ batch = 1;+@@ -6918,6 +6918,7 @@ static int zone_highsize(struct zone *zone, int batch, int cpu_online)+ * historical relationship between high and batch.+ */+ high = max(high, batch << 2);++ high = max(high, 1024);++ return high;+ #else+diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c+index fc2a985f6064..02dc861de3d5 100644+--- a/net/ipv4/inet_connection_sock.c++++ b/net/ipv4/inet_connection_sock.c+@@ -441,7 +441,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo)+ * having to remove and re-insert us on the wait queue.+ */+ for (;;) {+- prepare_to_wait_exclusive(sk_sleep(sk), &wait,++ prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait,+ TASK_INTERRUPTIBLE);+ release_sock(sk);+ if (reqsk_queue_empty(&icsk->icsk_accept_queue))+diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c+index 3b75836db19b..51b67c96868d 100644+--- a/net/ipv4/tcp.c++++ b/net/ipv4/tcp.c+@@ -4593,8 +4593,8 @@ void __init tcp_init(void)+ tcp_init_mem();+ /* Set per-socket limits to no more than 1/128 the pressure threshold */+ limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);+- max_wshare = min(4UL*1024*1024, limit);+- max_rshare = min(6UL*1024*1024, limit);++ max_wshare = min(16UL*1024*1024, limit);++ max_rshare = min(16UL*1024*1024, limit);++ init_net.ipv4.sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;+ init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;+--+2.34.1++From 1611858732c35596279cd94f57afda62f3b83493 Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sun, 23 Jan 2022 16:53:23 +0000+Subject: [PATCH] cpu++---+ arch/x86/Kconfig.cpu | 332 ++++++++++++++++++++++++++++++--+ arch/x86/Makefile | 40 +++-+ arch/x86/include/asm/vermagic.h | 66 ++++++++ init/Kconfig | 1 -+ 4 files changed, 424 insertions(+), 15 deletions(-)++diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu+index 542377cd419d..22b919cdb6d1 100644+--- a/arch/x86/Kconfig.cpu++++ b/arch/x86/Kconfig.cpu+@@ -157,7 +157,7 @@ config MPENTIUM4+++ config MK6+- bool "K6/K6-II/K6-III"++ bool "AMD K6/K6-II/K6-III"+ depends on X86_32+ help+ Select this for an AMD K6-family processor. Enables use of+@@ -165,7 +165,7 @@ config MK6+ flags to GCC.++ config MK7+- bool "Athlon/Duron/K7"++ bool "AMD Athlon/Duron/K7"+ depends on X86_32+ help+ Select this for an AMD Athlon K7-family processor. Enables use of+@@ -173,12 +173,98 @@ config MK7+ flags to GCC.++ config MK8+- bool "Opteron/Athlon64/Hammer/K8"++ bool "AMD Opteron/Athlon64/Hammer/K8"+ help+ Select this for an AMD Opteron or Athlon64 Hammer-family processor.+ Enables use of some extended instructions, and passes appropriate+ optimization flags to GCC.+++config MK8SSE3++ bool "AMD Opteron/Athlon64/Hammer/K8 with SSE3"++ help++ Select this for improved AMD Opteron or Athlon64 Hammer-family processors.++ Enables use of some extended instructions, and passes appropriate++ optimization flags to GCC.++++config MK10++ bool "AMD 61xx/7x50/PhenomX3/X4/II/K10"++ help++ Select this for an AMD 61xx Eight-Core Magny-Cours, Athlon X2 7x50,++ Phenom X3/X4/II, Athlon II X2/X3/X4, or Turion II-family processor.++ Enables use of some extended instructions, and passes appropriate++ optimization flags to GCC.++++config MBARCELONA++ bool "AMD Barcelona"++ help++ Select this for AMD Family 10h Barcelona processors.++++ Enables -march=barcelona++++config MBOBCAT++ bool "AMD Bobcat"++ help++ Select this for AMD Family 14h Bobcat processors.++++ Enables -march=btver1++++config MJAGUAR++ bool "AMD Jaguar"++ help++ Select this for AMD Family 16h Jaguar processors.++++ Enables -march=btver2++++config MBULLDOZER++ bool "AMD Bulldozer"++ help++ Select this for AMD Family 15h Bulldozer processors.++++ Enables -march=bdver1++++config MPILEDRIVER++ bool "AMD Piledriver"++ help++ Select this for AMD Family 15h Piledriver processors.++++ Enables -march=bdver2++++config MSTEAMROLLER++ bool "AMD Steamroller"++ help++ Select this for AMD Family 15h Steamroller processors.++++ Enables -march=bdver3++++config MEXCAVATOR++ bool "AMD Excavator"++ help++ Select this for AMD Family 15h Excavator processors.++++ Enables -march=bdver4++++config MZEN++ bool "AMD Zen"++ help++ Select this for AMD Family 17h Zen processors.++++ Enables -march=znver1++++config MZEN2++ bool "AMD Zen 2"++ help++ Select this for AMD Family 17h Zen 2 processors.++++ Enables -march=znver2++++config MZEN3++ bool "AMD Zen 3"++ depends on (CC_IS_GCC && GCC_VERSION >= 100300) || (CC_IS_CLANG && CLANG_VERSION >= 120000)++ help++ Select this for AMD Family 19h Zen 3 processors.++++ Enables -march=znver3+++ config MCRUSOE+ bool "Crusoe"+ depends on X86_32+@@ -270,7 +356,7 @@ config MPSC+ in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one.++ config MCORE2+- bool "Core 2/newer Xeon"++ bool "Intel Core 2"+ help++ Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and+@@ -278,6 +364,8 @@ config MCORE2+ family in /proc/cpuinfo. Newer ones have 6 and older ones 15+ (not a typo)+++ Enables -march=core2+++ config MATOM+ bool "Intel Atom"+ help+@@ -287,6 +375,182 @@ config MATOM+ accordingly optimized code. Use a recent GCC with specific Atom+ support in order to fully benefit from selecting this option.+++config MNEHALEM++ bool "Intel Nehalem"++ select X86_P6_NOP++ help++++ Select this for 1st Gen Core processors in the Nehalem family.++++ Enables -march=nehalem++++config MWESTMERE++ bool "Intel Westmere"++ select X86_P6_NOP++ help++++ Select this for the Intel Westmere formerly Nehalem-C family.++++ Enables -march=westmere++++config MSILVERMONT++ bool "Intel Silvermont"++ select X86_P6_NOP++ help++++ Select this for the Intel Silvermont platform.++++ Enables -march=silvermont++++config MGOLDMONT++ bool "Intel Goldmont"++ select X86_P6_NOP++ help++++ Select this for the Intel Goldmont platform including Apollo Lake and Denverton.++++ Enables -march=goldmont++++config MGOLDMONTPLUS++ bool "Intel Goldmont Plus"++ select X86_P6_NOP++ help++++ Select this for the Intel Goldmont Plus platform including Gemini Lake.++++ Enables -march=goldmont-plus++++config MSANDYBRIDGE++ bool "Intel Sandy Bridge"++ select X86_P6_NOP++ help++++ Select this for 2nd Gen Core processors in the Sandy Bridge family.++++ Enables -march=sandybridge++++config MIVYBRIDGE++ bool "Intel Ivy Bridge"++ select X86_P6_NOP++ help++++ Select this for 3rd Gen Core processors in the Ivy Bridge family.++++ Enables -march=ivybridge++++config MHASWELL++ bool "Intel Haswell"++ select X86_P6_NOP++ help++++ Select this for 4th Gen Core processors in the Haswell family.++++ Enables -march=haswell++++config MBROADWELL++ bool "Intel Broadwell"++ select X86_P6_NOP++ help++++ Select this for 5th Gen Core processors in the Broadwell family.++++ Enables -march=broadwell++++config MSKYLAKE++ bool "Intel Skylake"++ select X86_P6_NOP++ help++++ Select this for 6th Gen Core processors in the Skylake family.++++ Enables -march=skylake++++config MSKYLAKEX++ bool "Intel Skylake X"++ select X86_P6_NOP++ help++++ Select this for 6th Gen Core processors in the Skylake X family.++++ Enables -march=skylake-avx512++++config MCANNONLAKE++ bool "Intel Cannon Lake"++ select X86_P6_NOP++ help++++ Select this for 8th Gen Core processors++++ Enables -march=cannonlake++++config MICELAKE++ bool "Intel Ice Lake"++ select X86_P6_NOP++ help++++ Select this for 10th Gen Core processors in the Ice Lake family.++++ Enables -march=icelake-client++++config MCASCADELAKE++ bool "Intel Cascade Lake"++ select X86_P6_NOP++ help++++ Select this for Xeon processors in the Cascade Lake family.++++ Enables -march=cascadelake++++config MCOOPERLAKE++ bool "Intel Cooper Lake"++ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000)++ select X86_P6_NOP++ help++++ Select this for Xeon processors in the Cooper Lake family.++++ Enables -march=cooperlake++++config MTIGERLAKE++ bool "Intel Tiger Lake"++ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000)++ select X86_P6_NOP++ help++++ Select this for third-generation 10 nm process processors in the Tiger Lake family.++++ Enables -march=tigerlake++++config MSAPPHIRERAPIDS++ bool "Intel Sapphire Rapids"++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)++ select X86_P6_NOP++ help++++ Select this for third-generation 10 nm process processors in the Sapphire Rapids family.++++ Enables -march=sapphirerapids++++config MROCKETLAKE++ bool "Intel Rocket Lake"++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)++ select X86_P6_NOP++ help++++ Select this for eleventh-generation processors in the Rocket Lake family.++++ Enables -march=rocketlake++++config MALDERLAKE++ bool "Intel Alder Lake"++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)++ select X86_P6_NOP++ help++++ Select this for twelfth-generation processors in the Alder Lake family.++++ Enables -march=alderlake+++ config GENERIC_CPU+ bool "Generic-x86-64"+ depends on X86_64+@@ -294,6 +558,50 @@ config GENERIC_CPU+ Generic x86-64 CPU.+ Run equally well on all x86-64 CPUs.+++config GENERIC_CPU2++ bool "Generic-x86-64-v2"++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)++ depends on X86_64++ help++ Generic x86-64 CPU.++ Run equally well on all x86-64 CPUs with min support of x86-64-v2.++++config GENERIC_CPU3++ bool "Generic-x86-64-v3"++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)++ depends on X86_64++ help++ Generic x86-64-v3 CPU with v3 instructions.++ Run equally well on all x86-64 CPUs with min support of x86-64-v3.++++config GENERIC_CPU4++ bool "Generic-x86-64-v4"++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)++ depends on X86_64++ help++ Generic x86-64 CPU with v4 instructions.++ Run equally well on all x86-64 CPUs with min support of x86-64-v4.++++config MNATIVE_INTEL++ bool "Intel-Native optimizations autodetected by the compiler"++ help++++ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects++ the optimum settings to use based on your processor. Do NOT use this++ for AMD CPUs. Intel Only!++++ Enables -march=native++++config MNATIVE_AMD++ bool "AMD-Native optimizations autodetected by the compiler"++ help++++ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects++ the optimum settings to use based on your processor. Do NOT use this++ for Intel CPUs. AMD Only!++++ Enables -march=native+++ endchoice++ config X86_GENERIC+@@ -318,7 +626,7 @@ config X86_INTERNODE_CACHE_SHIFT+ config X86_L1_CACHE_SHIFT+ int+ default "7" if MPENTIUM4 || MPSC+- default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU++ default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD || X86_GENERIC || GENERIC_CPU || GENERIC_CPU2 || GENERIC_CPU3 || GENERIC_CPU4+ default "4" if MELAN || M486SX || M486 || MGEODEGX1+ default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX++@@ -336,11 +644,11 @@ config X86_ALIGNMENT_16++ config X86_INTEL_USERCOPY+ def_bool y+- depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2++ depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL++ config X86_USE_PPRO_CHECKSUM+ def_bool y+- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM++ depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD++ #+ # P6_NOPs are a relatively minor optimization that require a family >=+@@ -356,26 +664,26 @@ config X86_USE_PPRO_CHECKSUM+ config X86_P6_NOP+ def_bool y+ depends on X86_64+- depends on (MCORE2 || MPENTIUM4 || MPSC)++ depends on (MCORE2 || MPENTIUM4 || MPSC || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL)++ config X86_TSC+ def_bool y+- depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64++ depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) || X86_64++ config X86_CMPXCHG64+ def_bool y+- depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8++ depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD++ # this should be set for all -march=.. options where the compiler+ # generates cmov.+ config X86_CMOV+ def_bool y+- depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX)++ depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD)++ config X86_MINIMUM_CPU_FAMILY+ int+ default "64" if X86_64+- default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8)++ default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD)+ default "5" if X86_32 && X86_CMPXCHG64+ default "4"++diff --git a/arch/x86/Makefile b/arch/x86/Makefile+index e84cdd409b64..7d3bbf060079 100644+--- a/arch/x86/Makefile++++ b/arch/x86/Makefile+@@ -131,8 +131,44 @@ else+ # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu)+ cflags-$(CONFIG_MK8) += -march=k8+ cflags-$(CONFIG_MPSC) += -march=nocona+- cflags-$(CONFIG_MCORE2) += -march=core2+- cflags-$(CONFIG_MATOM) += -march=atom++ cflags-$(CONFIG_MK8SSE3) += -march=k8-sse3++ cflags-$(CONFIG_MK10) += -march=amdfam10++ cflags-$(CONFIG_MBARCELONA) += -march=barcelona++ cflags-$(CONFIG_MBOBCAT) += -march=btver1++ cflags-$(CONFIG_MJAGUAR) += -march=btver2++ cflags-$(CONFIG_MBULLDOZER) += -march=bdver1++ cflags-$(CONFIG_MPILEDRIVER) += -march=bdver2 -mno-tbm++ cflags-$(CONFIG_MSTEAMROLLER) += -march=bdver3 -mno-tbm++ cflags-$(CONFIG_MEXCAVATOR) += -march=bdver4 -mno-tbm++ cflags-$(CONFIG_MZEN) += -march=znver1++ cflags-$(CONFIG_MZEN2) += -march=znver2++ cflags-$(CONFIG_MZEN3) += -march=znver3++ cflags-$(CONFIG_MNATIVE_INTEL) += -march=native++ cflags-$(CONFIG_MNATIVE_AMD) += -march=native++ cflags-$(CONFIG_MATOM) += -march=bonnell++ cflags-$(CONFIG_MCORE2) += -march=core2++ cflags-$(CONFIG_MNEHALEM) += -march=nehalem++ cflags-$(CONFIG_MWESTMERE) += -march=westmere++ cflags-$(CONFIG_MSILVERMONT) += -march=silvermont++ cflags-$(CONFIG_MGOLDMONT) += -march=goldmont++ cflags-$(CONFIG_MGOLDMONTPLUS) += -march=goldmont-plus++ cflags-$(CONFIG_MSANDYBRIDGE) += -march=sandybridge++ cflags-$(CONFIG_MIVYBRIDGE) += -march=ivybridge++ cflags-$(CONFIG_MHASWELL) += -march=haswell++ cflags-$(CONFIG_MBROADWELL) += -march=broadwell++ cflags-$(CONFIG_MSKYLAKE) += -march=skylake++ cflags-$(CONFIG_MSKYLAKEX) += -march=skylake-avx512++ cflags-$(CONFIG_MCANNONLAKE) += -march=cannonlake++ cflags-$(CONFIG_MICELAKE) += -march=icelake-client++ cflags-$(CONFIG_MCASCADELAKE) += -march=cascadelake++ cflags-$(CONFIG_MCOOPERLAKE) += -march=cooperlake++ cflags-$(CONFIG_MTIGERLAKE) += -march=tigerlake++ cflags-$(CONFIG_MSAPPHIRERAPIDS) += -march=sapphirerapids++ cflags-$(CONFIG_MROCKETLAKE) += -march=rocketlake++ cflags-$(CONFIG_MALDERLAKE) += -march=alderlake++ cflags-$(CONFIG_GENERIC_CPU2) += -march=x86-64-v2++ cflags-$(CONFIG_GENERIC_CPU3) += -march=x86-64-v3++ cflags-$(CONFIG_GENERIC_CPU4) += -march=x86-64-v4+ cflags-$(CONFIG_GENERIC_CPU) += -mtune=generic+ KBUILD_CFLAGS += $(cflags-y)++diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h+index 75884d2cdec3..4e6a08d4c7e5 100644+--- a/arch/x86/include/asm/vermagic.h++++ b/arch/x86/include/asm/vermagic.h+@@ -17,6 +17,48 @@+ #define MODULE_PROC_FAMILY "586MMX "+ #elif defined CONFIG_MCORE2+ #define MODULE_PROC_FAMILY "CORE2 "++#elif defined CONFIG_MNATIVE_INTEL++#define MODULE_PROC_FAMILY "NATIVE_INTEL "++#elif defined CONFIG_MNATIVE_AMD++#define MODULE_PROC_FAMILY "NATIVE_AMD "++#elif defined CONFIG_MNEHALEM++#define MODULE_PROC_FAMILY "NEHALEM "++#elif defined CONFIG_MWESTMERE++#define MODULE_PROC_FAMILY "WESTMERE "++#elif defined CONFIG_MSILVERMONT++#define MODULE_PROC_FAMILY "SILVERMONT "++#elif defined CONFIG_MGOLDMONT++#define MODULE_PROC_FAMILY "GOLDMONT "++#elif defined CONFIG_MGOLDMONTPLUS++#define MODULE_PROC_FAMILY "GOLDMONTPLUS "++#elif defined CONFIG_MSANDYBRIDGE++#define MODULE_PROC_FAMILY "SANDYBRIDGE "++#elif defined CONFIG_MIVYBRIDGE++#define MODULE_PROC_FAMILY "IVYBRIDGE "++#elif defined CONFIG_MHASWELL++#define MODULE_PROC_FAMILY "HASWELL "++#elif defined CONFIG_MBROADWELL++#define MODULE_PROC_FAMILY "BROADWELL "++#elif defined CONFIG_MSKYLAKE++#define MODULE_PROC_FAMILY "SKYLAKE "++#elif defined CONFIG_MSKYLAKEX++#define MODULE_PROC_FAMILY "SKYLAKEX "++#elif defined CONFIG_MCANNONLAKE++#define MODULE_PROC_FAMILY "CANNONLAKE "++#elif defined CONFIG_MICELAKE++#define MODULE_PROC_FAMILY "ICELAKE "++#elif defined CONFIG_MCASCADELAKE++#define MODULE_PROC_FAMILY "CASCADELAKE "++#elif defined CONFIG_MCOOPERLAKE++#define MODULE_PROC_FAMILY "COOPERLAKE "++#elif defined CONFIG_MTIGERLAKE++#define MODULE_PROC_FAMILY "TIGERLAKE "++#elif defined CONFIG_MSAPPHIRERAPIDS++#define MODULE_PROC_FAMILY "SAPPHIRERAPIDS "++#elif defined CONFIG_ROCKETLAKE++#define MODULE_PROC_FAMILY "ROCKETLAKE "++#elif defined CONFIG_MALDERLAKE++#define MODULE_PROC_FAMILY "ALDERLAKE "+ #elif defined CONFIG_MATOM+ #define MODULE_PROC_FAMILY "ATOM "+ #elif defined CONFIG_M686+@@ -35,6 +77,30 @@+ #define MODULE_PROC_FAMILY "K7 "+ #elif defined CONFIG_MK8+ #define MODULE_PROC_FAMILY "K8 "++#elif defined CONFIG_MK8SSE3++#define MODULE_PROC_FAMILY "K8SSE3 "++#elif defined CONFIG_MK10++#define MODULE_PROC_FAMILY "K10 "++#elif defined CONFIG_MBARCELONA++#define MODULE_PROC_FAMILY "BARCELONA "++#elif defined CONFIG_MBOBCAT++#define MODULE_PROC_FAMILY "BOBCAT "++#elif defined CONFIG_MBULLDOZER++#define MODULE_PROC_FAMILY "BULLDOZER "++#elif defined CONFIG_MPILEDRIVER++#define MODULE_PROC_FAMILY "PILEDRIVER "++#elif defined CONFIG_MSTEAMROLLER++#define MODULE_PROC_FAMILY "STEAMROLLER "++#elif defined CONFIG_MJAGUAR++#define MODULE_PROC_FAMILY "JAGUAR "++#elif defined CONFIG_MEXCAVATOR++#define MODULE_PROC_FAMILY "EXCAVATOR "++#elif defined CONFIG_MZEN++#define MODULE_PROC_FAMILY "ZEN "++#elif defined CONFIG_MZEN2++#define MODULE_PROC_FAMILY "ZEN2 "++#elif defined CONFIG_MZEN3++#define MODULE_PROC_FAMILY "ZEN3 "+ #elif defined CONFIG_MELAN+ #define MODULE_PROC_FAMILY "ELAN "+ #elif defined CONFIG_MCRUSOE+diff --git a/init/Kconfig b/init/Kconfig+index e9119bf54b1f..98b648e3cef6 100644+--- a/init/Kconfig++++ b/init/Kconfig+@@ -1370,7 +1370,6 @@ config CC_OPTIMIZE_FOR_PERFORMANCE++ config CC_OPTIMIZE_FOR_PERFORMANCE_O3+ bool "Optimize more for performance (-O3)"+- depends on ARC+ help+ Choosing this option will pass "-O3" to your compiler to optimize+ the kernel yet more for performance.+--+2.34.1++From ef65b3a3f9e6dd347d2ae8735dcf347fa5135c8f Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sun, 6 Feb 2022 20:28:25 +0000+Subject: [PATCH] misc++Signed-off-by: P. Jung <[email protected]>+---+ .../admin-guide/kernel-parameters.txt | 9 ++ Makefile | 5 -+ drivers/cpufreq/Kconfig | 2 +-+ drivers/cpufreq/cpufreq_ondemand.c | 6 +-+ drivers/hid/usbhid/hid-core.c | 8 ++ drivers/i2c/busses/Kconfig | 9 ++ drivers/i2c/busses/Makefile | 1 ++ drivers/i2c/busses/i2c-nct6775.c | 647 +++++++++++++++++++ drivers/i2c/busses/i2c-piix4.c | 4 +-+ drivers/infiniband/core/addr.c | 1 ++ drivers/pci/quirks.c | 102 ++++ drivers/scsi/sd.c | 43 +-+ drivers/scsi/sd.h | 1 ++ drivers/tty/Kconfig | 13 ++ include/linux/pagemap.h | 2 +-+ include/linux/user_namespace.h | 4 ++ include/uapi/linux/futex.h | 13 ++ include/uapi/linux/vt.h | 15 +-+ init/Kconfig | 16 ++ kernel/Kconfig.hz | 38 ++ kernel/cpu.c | 1 ++ kernel/fork.c | 14 ++ kernel/futex/syscalls.c | 75 +-+ kernel/sched/core.c | 2 +-+ kernel/sched/fair.c | 14 +-+ kernel/sysctl.c | 12 ++ kernel/user_namespace.c | 7 ++ mm/internal.h | 1 ++ mm/page-writeback.c | 4 +-+ mm/page_alloc.c | 26 +-+ mm/vmscan.c | 5 +-+ net/sched/Kconfig | 4 ++ 32 files changed, 1068 insertions(+), 36 deletions(-)+ create mode 100644 drivers/i2c/busses/i2c-nct6775.c++diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt+index f5a27f067db9..cbc216dc0a74 100644+--- a/Documentation/admin-guide/kernel-parameters.txt++++ b/Documentation/admin-guide/kernel-parameters.txt+@@ -3938,6 +3938,15 @@+ nomsi [MSI] If the PCI_MSI kernel config parameter is+ enabled, this kernel boot option can be used to+ disable the use of MSI interrupts system-wide.++ pcie_acs_override =++ [PCIE] Override missing PCIe ACS support for:++ downstream++ All downstream ports - full ACS capabilities++ multifunction++ All multifunction devices - multifunction ACS subset++ id:nnnn:nnnn++ Specific device - full ACS capabilities++ Specified as vid:did (vendor/device ID) in hex+ noioapicquirk [APIC] Disable all boot interrupt quirks.+ Safety option to keep boot IRQs enabled. This+ should never be necessary.+diff --git a/Makefile b/Makefile+index 1fc3491096cb..4de96651f8d1 100644+--- a/Makefile++++ b/Makefile+@@ -984,11 +984,6 @@ KBUILD_CFLAGS += -fno-strict-overflow+ # Make sure -fstack-check isn't enabled (like gentoo apparently did)+ KBUILD_CFLAGS += -fno-stack-check++-# conserve stack if available+-ifdef CONFIG_CC_IS_GCC+-KBUILD_CFLAGS += -fconserve-stack+-endif+-+ # Prohibit date/time macros, which would make the build non-deterministic+ KBUILD_CFLAGS += -Werror=date-time++diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig+index c3038cdc6865..25bf88b571ae 100644+--- a/drivers/cpufreq/Kconfig++++ b/drivers/cpufreq/Kconfig+@@ -72,7 +72,7 @@ config CPU_FREQ_DEFAULT_GOV_USERSPACE++ config CPU_FREQ_DEFAULT_GOV_ONDEMAND+ bool "ondemand"+- depends on !(X86_INTEL_PSTATE && SMP)++ depends on SMP+ select CPU_FREQ_GOV_ONDEMAND+ select CPU_FREQ_GOV_PERFORMANCE+ help+diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c+index 6a41ea4729b8..370af073c27f 100644+--- a/drivers/cpufreq/cpufreq_ondemand.c++++ b/drivers/cpufreq/cpufreq_ondemand.c+@@ -18,10 +18,10 @@+ #include "cpufreq_ondemand.h"++ /* On-demand governor macros */+-#define DEF_FREQUENCY_UP_THRESHOLD (80)+-#define DEF_SAMPLING_DOWN_FACTOR (1)++#define DEF_FREQUENCY_UP_THRESHOLD (65)++#define DEF_SAMPLING_DOWN_FACTOR (10)+ #define MAX_SAMPLING_DOWN_FACTOR (100000)+-#define MICRO_FREQUENCY_UP_THRESHOLD (95)++#define MICRO_FREQUENCY_UP_THRESHOLD (85)+ #define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000)+ #define MIN_FREQUENCY_UP_THRESHOLD (1)+ #define MAX_FREQUENCY_UP_THRESHOLD (100)+diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c+index 54752c85604b..6fa8de3ce7aa 100644+--- a/drivers/hid/usbhid/hid-core.c++++ b/drivers/hid/usbhid/hid-core.c+@@ -45,6 +45,10 @@+ * Module parameters.+ */+++static unsigned int hid_poll_interval;++module_param_named(poll, hid_poll_interval, uint, 0644);++MODULE_PARM_DESC(poll, "Polling interval of any device");+++ static unsigned int hid_mousepoll_interval;+ module_param_named(mousepoll, hid_mousepoll_interval, uint, 0644);+ MODULE_PARM_DESC(mousepoll, "Polling interval of mice");+@@ -1107,6 +1111,10 @@ static int usbhid_start(struct hid_device *hid)+ hid->name, endpoint->bInterval, interval);+ }+++ /* First set polling interval for all devices */++ if (hid_poll_interval > 0)++ interval = hid_poll_interval;+++ /* Change the polling interval of mice, joysticks+ * and keyboards.+ */+diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig+index 42da31c1ab70..c3707e64d132 100644+--- a/drivers/i2c/busses/Kconfig++++ b/drivers/i2c/busses/Kconfig+@@ -219,6 +219,15 @@ config I2C_CHT_WC+ combined with a FUSB302 Type-C port-controller as such it is advised+ to also select CONFIG_TYPEC_FUSB302=m.+++config I2C_NCT6775++ tristate "Nuvoton NCT6775 and compatible SMBus controller"++ help++ If you say yes to this option, support will be included for the++ Nuvoton NCT6775 and compatible SMBus controllers.++++ This driver can also be built as a module. If so, the module++ will be called i2c-nct6775.+++ config I2C_NFORCE2+ tristate "Nvidia nForce2, nForce3 and nForce4"+ depends on PCI+diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile+index 1d00dce77098..ec8a1197c8bc 100644+--- a/drivers/i2c/busses/Makefile++++ b/drivers/i2c/busses/Makefile+@@ -17,6 +17,7 @@ obj-$(CONFIG_I2C_CHT_WC) += i2c-cht-wc.o+ obj-$(CONFIG_I2C_I801) += i2c-i801.o+ obj-$(CONFIG_I2C_ISCH) += i2c-isch.o+ obj-$(CONFIG_I2C_ISMT) += i2c-ismt.o++obj-$(CONFIG_I2C_NCT6775) += i2c-nct6775.o+ obj-$(CONFIG_I2C_NFORCE2) += i2c-nforce2.o+ obj-$(CONFIG_I2C_NFORCE2_S4985) += i2c-nforce2-s4985.o+ obj-$(CONFIG_I2C_NVIDIA_GPU) += i2c-nvidia-gpu.o+diff --git a/drivers/i2c/busses/i2c-nct6775.c b/drivers/i2c/busses/i2c-nct6775.c+new file mode 100644+index 000000000000..b59f842f7278+--- /dev/null++++ b/drivers/i2c/busses/i2c-nct6775.c+@@ -0,0 +1,647 @@++/*++ * i2c-nct6775 - Driver for the SMBus master functionality of++ * Nuvoton NCT677x Super-I/O chips++ *++ * Copyright (C) 2019 Adam Honse <[email protected]>++ *++ * Derived from nct6775 hwmon driver++ * Copyright (C) 2012 Guenter Roeck <[email protected]>++ *++ * This program is free software; you can redistribute it and/or modify++ * it under the terms of the GNU General Public License as published by++ * the Free Software Foundation; either version 2 of the License, or++ * (at your option) any later version.++ *++ * This program is distributed in the hope that it will be useful,++ * but WITHOUT ANY WARRANTY; without even the implied warranty of++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the++ * GNU General Public License for more details.++ *++ * You should have received a copy of the GNU General Public License++ * along with this program; if not, write to the Free Software++ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.++ *++ */++++#include <linux/module.h>++#include <linux/init.h>++#include <linux/slab.h>++#include <linux/jiffies.h>++#include <linux/platform_device.h>++#include <linux/hwmon.h>++#include <linux/hwmon-sysfs.h>++#include <linux/hwmon-vid.h>++#include <linux/err.h>++#include <linux/mutex.h>++#include <linux/delay.h>++#include <linux/ioport.h>++#include <linux/i2c.h>++#include <linux/acpi.h>++#include <linux/bitops.h>++#include <linux/dmi.h>++#include <linux/io.h>++#include <linux/nospec.h>++++#define DRVNAME "i2c-nct6775"++++/* Nuvoton SMBus address offsets */++#define SMBHSTDAT (0 + nuvoton_nct6793d_smba)++#define SMBBLKSZ (1 + nuvoton_nct6793d_smba)++#define SMBHSTCMD (2 + nuvoton_nct6793d_smba)++#define SMBHSTIDX (3 + nuvoton_nct6793d_smba) //Index field is the Command field on other controllers++#define SMBHSTCTL (4 + nuvoton_nct6793d_smba)++#define SMBHSTADD (5 + nuvoton_nct6793d_smba)++#define SMBHSTERR (9 + nuvoton_nct6793d_smba)++#define SMBHSTSTS (0xE + nuvoton_nct6793d_smba)++++/* Command register */++#define NCT6793D_READ_BYTE 0++#define NCT6793D_READ_WORD 1++#define NCT6793D_READ_BLOCK 2++#define NCT6793D_BLOCK_WRITE_READ_PROC_CALL 3++#define NCT6793D_PROC_CALL 4++#define NCT6793D_WRITE_BYTE 8++#define NCT6793D_WRITE_WORD 9++#define NCT6793D_WRITE_BLOCK 10++++/* Control register */++#define NCT6793D_MANUAL_START 128++#define NCT6793D_SOFT_RESET 64++++/* Error register */++#define NCT6793D_NO_ACK 32++++/* Status register */++#define NCT6793D_FIFO_EMPTY 1++#define NCT6793D_FIFO_FULL 2++#define NCT6793D_MANUAL_ACTIVE 4++++#define NCT6775_LD_SMBUS 0x0B++++/* Other settings */++#define MAX_RETRIES 400++++enum kinds { nct6106, nct6775, nct6776, nct6779, nct6791, nct6792, nct6793,++ nct6795, nct6796, nct6798 };++++struct nct6775_sio_data {++ int sioreg;++ enum kinds kind;++};++++/* used to set data->name = nct6775_device_names[data->sio_kind] */++static const char * const nct6775_device_names[] = {++ "nct6106",++ "nct6775",++ "nct6776",++ "nct6779",++ "nct6791",++ "nct6792",++ "nct6793",++ "nct6795",++ "nct6796",++ "nct6798",++};++++static const char * const nct6775_sio_names[] __initconst = {++ "NCT6106D",++ "NCT6775F",++ "NCT6776D/F",++ "NCT6779D",++ "NCT6791D",++ "NCT6792D",++ "NCT6793D",++ "NCT6795D",++ "NCT6796D",++ "NCT6798D",++};++++#define SIO_REG_LDSEL 0x07 /* Logical device select */++#define SIO_REG_DEVID 0x20 /* Device ID (2 bytes) */++#define SIO_REG_SMBA 0x62 /* SMBus base address register */++++#define SIO_NCT6106_ID 0xc450++#define SIO_NCT6775_ID 0xb470++#define SIO_NCT6776_ID 0xc330++#define SIO_NCT6779_ID 0xc560++#define SIO_NCT6791_ID 0xc800++#define SIO_NCT6792_ID 0xc910++#define SIO_NCT6793_ID 0xd120++#define SIO_NCT6795_ID 0xd350++#define SIO_NCT6796_ID 0xd420++#define SIO_NCT6798_ID 0xd428++#define SIO_ID_MASK 0xFFF0++++static inline void++superio_outb(int ioreg, int reg, int val)++{++ outb(reg, ioreg);++ outb(val, ioreg + 1);++}++++static inline int++superio_inb(int ioreg, int reg)++{++ outb(reg, ioreg);++ return inb(ioreg + 1);++}++++static inline void++superio_select(int ioreg, int ld)++{++ outb(SIO_REG_LDSEL, ioreg);++ outb(ld, ioreg + 1);++}++++static inline int++superio_enter(int ioreg)++{++ /*++ * Try to reserve <ioreg> and <ioreg + 1> for exclusive access.++ */++ if (!request_muxed_region(ioreg, 2, DRVNAME))++ return -EBUSY;++++ outb(0x87, ioreg);++ outb(0x87, ioreg);++++ return 0;++}++++static inline void++superio_exit(int ioreg)++{++ outb(0xaa, ioreg);++ outb(0x02, ioreg);++ outb(0x02, ioreg + 1);++ release_region(ioreg, 2);++}++++/*++ * ISA constants++ */++++#define IOREGION_ALIGNMENT (~7)++#define IOREGION_LENGTH 2++#define ADDR_REG_OFFSET 0++#define DATA_REG_OFFSET 1++++#define NCT6775_REG_BANK 0x4E++#define NCT6775_REG_CONFIG 0x40++++static struct i2c_adapter *nct6775_adapter;++++struct i2c_nct6775_adapdata {++ unsigned short smba;++};++++/* Return negative errno on error. */++static s32 nct6775_access(struct i2c_adapter * adap, u16 addr,++ unsigned short flags, char read_write,++ u8 command, int size, union i2c_smbus_data * data)++{++ struct i2c_nct6775_adapdata *adapdata = i2c_get_adapdata(adap);++ unsigned short nuvoton_nct6793d_smba = adapdata->smba;++ int i, len, cnt;++ union i2c_smbus_data tmp_data;++ int timeout = 0;++++ tmp_data.word = 0;++ cnt = 0;++ len = 0;++++ outb_p(NCT6793D_SOFT_RESET, SMBHSTCTL);++++ switch (size) {++ case I2C_SMBUS_QUICK:++ outb_p((addr << 1) | read_write,++ SMBHSTADD);++ break;++ case I2C_SMBUS_BYTE_DATA:++ tmp_data.byte = data->byte;++ case I2C_SMBUS_BYTE:++ outb_p((addr << 1) | read_write,++ SMBHSTADD);++ outb_p(command, SMBHSTIDX);++ if (read_write == I2C_SMBUS_WRITE) {++ outb_p(tmp_data.byte, SMBHSTDAT);++ outb_p(NCT6793D_WRITE_BYTE, SMBHSTCMD);++ }++ else {++ outb_p(NCT6793D_READ_BYTE, SMBHSTCMD);++ }++ break;++ case I2C_SMBUS_WORD_DATA:++ outb_p((addr << 1) | read_write,++ SMBHSTADD);++ outb_p(command, SMBHSTIDX);++ if (read_write == I2C_SMBUS_WRITE) {++ outb_p(data->word & 0xff, SMBHSTDAT);++ outb_p((data->word & 0xff00) >> 8, SMBHSTDAT);++ outb_p(NCT6793D_WRITE_WORD, SMBHSTCMD);++ }++ else {++ outb_p(NCT6793D_READ_WORD, SMBHSTCMD);++ }++ break;++ case I2C_SMBUS_BLOCK_DATA:++ outb_p((addr << 1) | read_write,++ SMBHSTADD);++ outb_p(command, SMBHSTIDX);++ if (read_write == I2C_SMBUS_WRITE) {++ len = data->block[0];++ if (len == 0 || len > I2C_SMBUS_BLOCK_MAX)++ return -EINVAL;++ outb_p(len, SMBBLKSZ);++++ cnt = 1;++ if (len >= 4) {++ for (i = cnt; i <= 4; i++) {++ outb_p(data->block[i], SMBHSTDAT);++ }++++ len -= 4;++ cnt += 4;++ }++ else {++ for (i = cnt; i <= len; i++ ) {++ outb_p(data->block[i], SMBHSTDAT);++ }++++ len = 0;++ }++++ outb_p(NCT6793D_WRITE_BLOCK, SMBHSTCMD);++ }++ else {++ return -ENOTSUPP;++ }++ break;++ default:++ dev_warn(&adap->dev, "Unsupported transaction %d\n", size);++ return -EOPNOTSUPP;++ }++++ outb_p(NCT6793D_MANUAL_START, SMBHSTCTL);++++ while ((size == I2C_SMBUS_BLOCK_DATA) && (len > 0)) {++ if (read_write == I2C_SMBUS_WRITE) {++ timeout = 0;++ while ((inb_p(SMBHSTSTS) & NCT6793D_FIFO_EMPTY) == 0)++ {++ if(timeout > MAX_RETRIES)++ {++ return -ETIMEDOUT;++ }++ usleep_range(250, 500);++ timeout++;++ }++++ //Load more bytes into FIFO++ if (len >= 4) {++ for (i = cnt; i <= (cnt + 4); i++) {++ outb_p(data->block[i], SMBHSTDAT);++ }++++ len -= 4;++ cnt += 4;++ }++ else {++ for (i = cnt; i <= (cnt + len); i++) {++ outb_p(data->block[i], SMBHSTDAT);++ }++++ len = 0;++ }++ }++ else {++ return -ENOTSUPP;++ }++++ }++++ //wait for manual mode to complete++ timeout = 0;++ while ((inb_p(SMBHSTSTS) & NCT6793D_MANUAL_ACTIVE) != 0)++ {++ if(timeout > MAX_RETRIES)++ {++ return -ETIMEDOUT;++ }++ usleep_range(250, 500);++ timeout++;++ }++++ if ((inb_p(SMBHSTERR) & NCT6793D_NO_ACK) != 0) {++ return -ENXIO;++ }++ else if ((read_write == I2C_SMBUS_WRITE) || (size == I2C_SMBUS_QUICK)) {++ return 0;++ }++++ switch (size) {++ case I2C_SMBUS_QUICK:++ case I2C_SMBUS_BYTE_DATA:++ data->byte = inb_p(SMBHSTDAT);++ break;++ case I2C_SMBUS_WORD_DATA:++ data->word = inb_p(SMBHSTDAT) + (inb_p(SMBHSTDAT) << 8);++ break;++ }++ return 0;++}++++static u32 nct6775_func(struct i2c_adapter *adapter)++{++ return I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SMBUS_BYTE |++ I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA |++ I2C_FUNC_SMBUS_BLOCK_DATA;++}++++static const struct i2c_algorithm smbus_algorithm = {++ .smbus_xfer = nct6775_access,++ .functionality = nct6775_func,++};++++static int nct6775_add_adapter(unsigned short smba, const char *name, struct i2c_adapter **padap)++{++ struct i2c_adapter *adap;++ struct i2c_nct6775_adapdata *adapdata;++ int retval;++++ adap = kzalloc(sizeof(*adap), GFP_KERNEL);++ if (adap == NULL) {++ return -ENOMEM;++ }++++ adap->owner = THIS_MODULE;++ adap->class = I2C_CLASS_HWMON | I2C_CLASS_SPD;++ adap->algo = &smbus_algorithm;++++ adapdata = kzalloc(sizeof(*adapdata), GFP_KERNEL);++ if (adapdata == NULL) {++ kfree(adap);++ return -ENOMEM;++ }++++ adapdata->smba = smba;++++ snprintf(adap->name, sizeof(adap->name),++ "SMBus NCT67xx adapter%s at %04x", name, smba);++++ i2c_set_adapdata(adap, adapdata);++++ retval = i2c_add_adapter(adap);++ if (retval) {++ kfree(adapdata);++ kfree(adap);++ return retval;++ }++++ *padap = adap;++ return 0;++}++++static void nct6775_remove_adapter(struct i2c_adapter *adap)++{++ struct i2c_nct6775_adapdata *adapdata = i2c_get_adapdata(adap);++++ if (adapdata->smba) {++ i2c_del_adapter(adap);++ kfree(adapdata);++ kfree(adap);++ }++}++++//static SIMPLE_DEV_PM_OPS(nct6775_dev_pm_ops, nct6775_suspend, nct6775_resume);++++/*++ * when Super-I/O functions move to a separate file, the Super-I/O++ * bus will manage the lifetime of the device and this module will only keep++ * track of the nct6775 driver. But since we use platform_device_alloc(), we++ * must keep track of the device++ */++static struct platform_device *pdev[2];++++static int nct6775_probe(struct platform_device *pdev)++{++ struct device *dev = &pdev->dev;++ struct nct6775_sio_data *sio_data = dev_get_platdata(dev);++ struct resource *res;++++ res = platform_get_resource(pdev, IORESOURCE_IO, 0);++ if (!devm_request_region(&pdev->dev, res->start, IOREGION_LENGTH,++ DRVNAME))++ return -EBUSY;++++ switch (sio_data->kind) {++ case nct6791:++ case nct6792:++ case nct6793:++ case nct6795:++ case nct6796:++ case nct6798:++ nct6775_add_adapter(res->start, "", &nct6775_adapter);++ break;++ default:++ return -ENODEV;++ }++++ return 0;++}++/*++static void nct6791_enable_io_mapping(int sioaddr)++{++ int val;++++ val = superio_inb(sioaddr, NCT6791_REG_HM_IO_SPACE_LOCK_ENABLE);++ if (val & 0x10) {++ pr_info("Enabling hardware monitor logical device mappings.\n");++ superio_outb(sioaddr, NCT6791_REG_HM_IO_SPACE_LOCK_ENABLE,++ val & ~0x10);++ }++}*/++++static struct platform_driver i2c_nct6775_driver = {++ .driver = {++ .name = DRVNAME,++// .pm = &nct6775_dev_pm_ops,++ },++ .probe = nct6775_probe,++};++++static void __exit i2c_nct6775_exit(void)++{++ int i;++++ if(nct6775_adapter)++ nct6775_remove_adapter(nct6775_adapter);++++ for (i = 0; i < ARRAY_SIZE(pdev); i++) {++ if (pdev[i])++ platform_device_unregister(pdev[i]);++ }++ platform_driver_unregister(&i2c_nct6775_driver);++}++++/* nct6775_find() looks for a '627 in the Super-I/O config space */++static int __init nct6775_find(int sioaddr, struct nct6775_sio_data *sio_data)++{++ u16 val;++ int err;++ int addr;++++ err = superio_enter(sioaddr);++ if (err)++ return err;++++ val = (superio_inb(sioaddr, SIO_REG_DEVID) << 8) |++ superio_inb(sioaddr, SIO_REG_DEVID + 1);++++ switch (val & SIO_ID_MASK) {++ case SIO_NCT6106_ID:++ sio_data->kind = nct6106;++ break;++ case SIO_NCT6775_ID:++ sio_data->kind = nct6775;++ break;++ case SIO_NCT6776_ID:++ sio_data->kind = nct6776;++ break;++ case SIO_NCT6779_ID:++ sio_data->kind = nct6779;++ break;++ case SIO_NCT6791_ID:++ sio_data->kind = nct6791;++ break;++ case SIO_NCT6792_ID:++ sio_data->kind = nct6792;++ break;++ case SIO_NCT6793_ID:++ sio_data->kind = nct6793;++ break;++ case SIO_NCT6795_ID:++ sio_data->kind = nct6795;++ break;++ case SIO_NCT6796_ID:++ sio_data->kind = nct6796;++ break;++ case SIO_NCT6798_ID:++ sio_data->kind = nct6798;++ break;++ default:++ if (val != 0xffff)++ pr_debug("unsupported chip ID: 0x%04x\n", val);++ superio_exit(sioaddr);++ return -ENODEV;++ }++++ /* We have a known chip, find the SMBus I/O address */++ superio_select(sioaddr, NCT6775_LD_SMBUS);++ val = (superio_inb(sioaddr, SIO_REG_SMBA) << 8)++ | superio_inb(sioaddr, SIO_REG_SMBA + 1);++ addr = val & IOREGION_ALIGNMENT;++ if (addr == 0) {++ pr_err("Refusing to enable a Super-I/O device with a base I/O port 0\n");++ superio_exit(sioaddr);++ return -ENODEV;++ }++++ //if (sio_data->kind == nct6791 || sio_data->kind == nct6792 ||++ // sio_data->kind == nct6793 || sio_data->kind == nct6795 ||++ // sio_data->kind == nct6796)++ // nct6791_enable_io_mapping(sioaddr);++++ superio_exit(sioaddr);++ pr_info("Found %s or compatible chip at %#x:%#x\n",++ nct6775_sio_names[sio_data->kind], sioaddr, addr);++ sio_data->sioreg = sioaddr;++++ return addr;++}++++static int __init i2c_nct6775_init(void)++{++ int i, err;++ bool found = false;++ int address;++ struct resource res;++ struct nct6775_sio_data sio_data;++ int sioaddr[2] = { 0x2e, 0x4e };++++ err = platform_driver_register(&i2c_nct6775_driver);++ if (err)++ return err;++++ /*++ * initialize sio_data->kind and sio_data->sioreg.++ *++ * when Super-I/O functions move to a separate file, the Super-I/O++ * driver will probe 0x2e and 0x4e and auto-detect the presence of a++ * nct6775 hardware monitor, and call probe()++ */++ for (i = 0; i < ARRAY_SIZE(pdev); i++) {++ address = nct6775_find(sioaddr[i], &sio_data);++ if (address <= 0)++ continue;++++ found = true;++++ pdev[i] = platform_device_alloc(DRVNAME, address);++ if (!pdev[i]) {++ err = -ENOMEM;++ goto exit_device_unregister;++ }++++ err = platform_device_add_data(pdev[i], &sio_data,++ sizeof(struct nct6775_sio_data));++ if (err)++ goto exit_device_put;++++ memset(&res, 0, sizeof(res));++ res.name = DRVNAME;++ res.start = address;++ res.end = address + IOREGION_LENGTH - 1;++ res.flags = IORESOURCE_IO;++++ err = acpi_check_resource_conflict(&res);++ if (err) {++ platform_device_put(pdev[i]);++ pdev[i] = NULL;++ continue;++ }++++ err = platform_device_add_resources(pdev[i], &res, 1);++ if (err)++ goto exit_device_put;++++ /* platform_device_add calls probe() */++ err = platform_device_add(pdev[i]);++ if (err)++ goto exit_device_put;++ }++ if (!found) {++ err = -ENODEV;++ goto exit_unregister;++ }++++ return 0;++++exit_device_put:++ platform_device_put(pdev[i]);++exit_device_unregister:++ while (--i >= 0) {++ if (pdev[i])++ platform_device_unregister(pdev[i]);++ }++exit_unregister:++ platform_driver_unregister(&i2c_nct6775_driver);++ return err;++}++++MODULE_AUTHOR("Adam Honse <[email protected]>");++MODULE_DESCRIPTION("SMBus driver for NCT6775F and compatible chips");++MODULE_LICENSE("GPL");++++module_init(i2c_nct6775_init);++module_exit(i2c_nct6775_exit);+diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c+index 8c1b31ed0c42..0b1b36fa0b71 100644+--- a/drivers/i2c/busses/i2c-piix4.c++++ b/drivers/i2c/busses/i2c-piix4.c+@@ -467,11 +467,11 @@ static int piix4_transaction(struct i2c_adapter *piix4_adapter)+ if (srvrworks_csb5_delay) /* Extra delay for SERVERWORKS_CSB5 */+ usleep_range(2000, 2100);+ else+- usleep_range(250, 500);++ usleep_range(25, 50);++ while ((++timeout < MAX_TIMEOUT) &&+ ((temp = inb_p(SMBHSTSTS)) & 0x01))+- usleep_range(250, 500);++ usleep_range(25, 50);++ /* If the SMBus is still busy, we give up */+ if (timeout == MAX_TIMEOUT) {+diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c+index 65e3e7df8a4b..b41afee7747e 100644+--- a/drivers/infiniband/core/addr.c++++ b/drivers/infiniband/core/addr.c+@@ -821,6 +821,7 @@ int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,+ union {+ struct sockaddr_in _sockaddr_in;+ struct sockaddr_in6 _sockaddr_in6;++ struct sockaddr_ib _sockaddr_ib;+ } sgid_addr, dgid_addr;+ int ret;++diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c+index d2dd6a6cda60..ccb05be311ff 100644+--- a/drivers/pci/quirks.c++++ b/drivers/pci/quirks.c+@@ -3600,6 +3600,106 @@ static void quirk_nvidia_no_bus_reset(struct pci_dev *dev)+ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,+ quirk_nvidia_no_bus_reset);+++static bool acs_on_downstream;++static bool acs_on_multifunction;++++#define NUM_ACS_IDS 16++struct acs_on_id {++ unsigned short vendor;++ unsigned short device;++};++static struct acs_on_id acs_on_ids[NUM_ACS_IDS];++static u8 max_acs_id;++++static __init int pcie_acs_override_setup(char *p)++{++ if (!p)++ return -EINVAL;++++ while (*p) {++ if (!strncmp(p, "downstream", 10))++ acs_on_downstream = true;++ if (!strncmp(p, "multifunction", 13))++ acs_on_multifunction = true;++ if (!strncmp(p, "id:", 3)) {++ char opt[5];++ int ret;++ long val;++++ if (max_acs_id >= NUM_ACS_IDS - 1) {++ pr_warn("Out of PCIe ACS override slots (%d)\n",++ NUM_ACS_IDS);++ goto next;++ }++++ p += 3;++ snprintf(opt, 5, "%s", p);++ ret = kstrtol(opt, 16, &val);++ if (ret) {++ pr_warn("PCIe ACS ID parse error %d\n", ret);++ goto next;++ }++ acs_on_ids[max_acs_id].vendor = val;++++ p += strcspn(p, ":");++ if (*p != ':') {++ pr_warn("PCIe ACS invalid ID\n");++ goto next;++ }++++ p++;++ snprintf(opt, 5, "%s", p);++ ret = kstrtol(opt, 16, &val);++ if (ret) {++ pr_warn("PCIe ACS ID parse error %d\n", ret);++ goto next;++ }++ acs_on_ids[max_acs_id].device = val;++ max_acs_id++;++ }++next:++ p += strcspn(p, ",");++ if (*p == ',')++ p++;++ }++++ if (acs_on_downstream || acs_on_multifunction || max_acs_id)++ pr_warn("Warning: PCIe ACS overrides enabled; This may allow non-IOMMU protected peer-to-peer DMA\n");++++ return 0;++}++early_param("pcie_acs_override", pcie_acs_override_setup);++++static int pcie_acs_overrides(struct pci_dev *dev, u16 acs_flags)++{++ int i;++++ /* Never override ACS for legacy devices or devices with ACS caps */++ if (!pci_is_pcie(dev) ||++ pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS))++ return -ENOTTY;++++ for (i = 0; i < max_acs_id; i++)++ if (acs_on_ids[i].vendor == dev->vendor &&++ acs_on_ids[i].device == dev->device)++ return 1;++++ switch (pci_pcie_type(dev)) {++ case PCI_EXP_TYPE_DOWNSTREAM:++ case PCI_EXP_TYPE_ROOT_PORT:++ if (acs_on_downstream)++ return 1;++ break;++ case PCI_EXP_TYPE_ENDPOINT:++ case PCI_EXP_TYPE_UPSTREAM:++ case PCI_EXP_TYPE_LEG_END:++ case PCI_EXP_TYPE_RC_END:++ if (acs_on_multifunction && dev->multifunction)++ return 1;++ }++++ return -ENOTTY;++}+ /*+ * Some Atheros AR9xxx and QCA988x chips do not behave after a bus reset.+ * The device will throw a Link Down error on AER-capable systems and+@@ -4953,6 +5053,8 @@ static const struct pci_dev_acs_enabled {+ { PCI_VENDOR_ID_NXP, 0x8d9b, pci_quirk_nxp_rp_acs },+ /* Zhaoxin Root/Downstream Ports */+ { PCI_VENDOR_ID_ZHAOXIN, PCI_ANY_ID, pci_quirk_zhaoxin_pcie_ports_acs },++ /* PCIe ACS overrides */++ { PCI_ANY_ID, PCI_ANY_ID, pcie_acs_overrides },+ { 0 }+ };++diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c+index 62eb9921cc94..58952be562f6 100644+--- a/drivers/scsi/sd.c++++ b/drivers/scsi/sd.c+@@ -2947,7 +2947,6 @@ static void sd_read_app_tag_own(struct scsi_disk *sdkp, unsigned char *buffer)+ */+ static void sd_read_block_limits(struct scsi_disk *sdkp)+ {+- unsigned int sector_sz = sdkp->device->sector_size;+ const int vpd_len = 64;+ unsigned char *buffer = kmalloc(vpd_len, GFP_KERNEL);++@@ -2956,9 +2955,7 @@ static void sd_read_block_limits(struct scsi_disk *sdkp)+ scsi_get_vpd_page(sdkp->device, 0xb0, buffer, vpd_len))+ goto out;++- blk_queue_io_min(sdkp->disk->queue,+- get_unaligned_be16(&buffer[6]) * sector_sz);+-++ sdkp->min_xfer_blocks = get_unaligned_be16(&buffer[6]);+ sdkp->max_xfer_blocks = get_unaligned_be32(&buffer[8]);+ sdkp->opt_xfer_blocks = get_unaligned_be32(&buffer[12]);++@@ -3214,6 +3211,29 @@ static void sd_read_cpr(struct scsi_disk *sdkp)+ kfree(buffer);+ }+++static bool sd_validate_min_xfer_size(struct scsi_disk *sdkp)++{++ struct scsi_device *sdp = sdkp->device;++ unsigned int min_xfer_bytes =++ logical_to_bytes(sdp, sdkp->min_xfer_blocks);++++ if (sdkp->min_xfer_blocks == 0)++ return false;++++ if (min_xfer_bytes & (sdkp->physical_block_size - 1)) {++ sd_first_printk(KERN_WARNING, sdkp,++ "Preferred minimum I/O size %u bytes not a " \++ "multiple of physical block size (%u bytes)\n",++ min_xfer_bytes, sdkp->physical_block_size);++ sdkp->min_xfer_blocks = 0;++ return false;++ }++++ sd_first_printk(KERN_INFO, sdkp, "Preferred minimum I/O size %u bytes\n",++ min_xfer_bytes);++ return true;++}+++ /*+ * Determine the device's preferred I/O size for reads and writes+ * unless the reported value is unreasonably small, large, not a+@@ -3225,6 +3245,8 @@ static bool sd_validate_opt_xfer_size(struct scsi_disk *sdkp,+ struct scsi_device *sdp = sdkp->device;+ unsigned int opt_xfer_bytes =+ logical_to_bytes(sdp, sdkp->opt_xfer_blocks);++ unsigned int min_xfer_bytes =++ logical_to_bytes(sdp, sdkp->min_xfer_blocks);++ if (sdkp->opt_xfer_blocks == 0)+ return false;+@@ -3261,6 +3283,15 @@ static bool sd_validate_opt_xfer_size(struct scsi_disk *sdkp,+ return false;+ }+++ if (min_xfer_bytes && opt_xfer_bytes & (min_xfer_bytes - 1)) {++ sd_first_printk(KERN_WARNING, sdkp,++ "Optimal transfer size %u bytes not a " \++ "multiple of preferred minimum block " \++ "size (%u bytes)\n",++ opt_xfer_bytes, min_xfer_bytes);++ return false;++ }+++ sd_first_printk(KERN_INFO, sdkp, "Optimal transfer size %u bytes\n",+ opt_xfer_bytes);+ return true;+@@ -3345,6 +3376,10 @@ static int sd_revalidate_disk(struct gendisk *disk)+ dev_max = min_not_zero(dev_max, sdkp->max_xfer_blocks);+ q->limits.max_dev_sectors = logical_to_sectors(sdp, dev_max);+++ if (sd_validate_min_xfer_size(sdkp))++ blk_queue_io_min(sdkp->disk->queue,++ logical_to_bytes(sdp, sdkp->min_xfer_blocks));+++ if (sd_validate_opt_xfer_size(sdkp, dev_max)) {+ q->limits.io_opt = logical_to_bytes(sdp, sdkp->opt_xfer_blocks);+ rw_max = logical_to_sectors(sdp, sdkp->opt_xfer_blocks);+diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h+index 2e5932bde43d..f4fbca90e997 100644+--- a/drivers/scsi/sd.h++++ b/drivers/scsi/sd.h+@@ -91,6 +91,7 @@ struct scsi_disk {+ atomic_t openers;+ sector_t capacity; /* size in logical blocks */+ int max_retries;++ u32 min_xfer_blocks;+ u32 max_xfer_blocks;+ u32 opt_xfer_blocks;+ u32 max_ws_blocks;+diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig+index cc30ff93e2e4..812116d48064 100644+--- a/drivers/tty/Kconfig++++ b/drivers/tty/Kconfig+@@ -75,6 +75,19 @@ config VT_CONSOLE_SLEEP+ def_bool y+ depends on VT_CONSOLE && PM_SLEEP+++config NR_TTY_DEVICES++ int "Maximum tty device number"++ depends on VT++ range 12 63++ default 63++ help++ This option is used to change the number of tty devices in /dev.++ The default value is 63. The lowest number you can set is 12,++ 63 is also the upper limit so we don't overrun the serial++ consoles.++++ If unsure, say 63.+++ config HW_CONSOLE+ bool+ depends on VT+diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h+index 270bf5136c34..13207a15b326 100644+--- a/include/linux/pagemap.h++++ b/include/linux/pagemap.h+@@ -1024,7 +1024,7 @@ struct readahead_control {+ ._index = i, \+ }++-#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE)++#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE)++ void page_cache_ra_unbounded(struct readahead_control *,+ unsigned long nr_to_read, unsigned long lookahead_count);+diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h+index 33a4240e6a6f..82213f9c4c17 100644+--- a/include/linux/user_namespace.h++++ b/include/linux/user_namespace.h+@@ -139,6 +139,8 @@ static inline void set_rlimit_ucount_max(struct user_namespace *ns,++ #ifdef CONFIG_USER_NS+++extern int unprivileged_userns_clone;+++ static inline struct user_namespace *get_user_ns(struct user_namespace *ns)+ {+ if (ns)+@@ -172,6 +174,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns);+ struct ns_common *ns_get_owner(struct ns_common *ns);+ #else+++#define unprivileged_userns_clone 0+++ static inline struct user_namespace *get_user_ns(struct user_namespace *ns)+ {+ return &init_user_ns;+diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h+index 71a5df8d2689..cf7fc988c24c 100644+--- a/include/uapi/linux/futex.h++++ b/include/uapi/linux/futex.h+@@ -22,6 +22,7 @@+ #define FUTEX_WAIT_REQUEUE_PI 11+ #define FUTEX_CMP_REQUEUE_PI 12+ #define FUTEX_LOCK_PI2 13++#define FUTEX_WAIT_MULTIPLE 31++ #define FUTEX_PRIVATE_FLAG 128+ #define FUTEX_CLOCK_REALTIME 256+@@ -68,6 +69,18 @@ struct futex_waitv {+ __u32 __reserved;+ };+++/**++ * struct futex_wait_block - Block of futexes to be waited for++ * @uaddr: User address of the futex++ * @val: Futex value expected by userspace++ * @bitset: Bitset for the optional bitmasked wakeup++ */++struct futex_wait_block {++ __u32 __user *uaddr;++ __u32 val;++ __u32 bitset;++};+++ /*+ * Support for robust futexes: the kernel cleans up held futexes at+ * thread exit time.+diff --git a/include/uapi/linux/vt.h b/include/uapi/linux/vt.h+index e9d39c48520a..3bceead8da40 100644+--- a/include/uapi/linux/vt.h++++ b/include/uapi/linux/vt.h+@@ -3,12 +3,25 @@+ #define _UAPI_LINUX_VT_H++++/*++ * We will make this definition solely for the purpose of making packages++ * such as splashutils build, because they can not understand that++ * NR_TTY_DEVICES is defined in the kernel configuration.++ */++#ifndef CONFIG_NR_TTY_DEVICES++#define CONFIG_NR_TTY_DEVICES 63++#endif+++ /*+ * These constants are also useful for user-level apps (e.g., VC+ * resizing).+ */+ #define MIN_NR_CONSOLES 1 /* must be at least 1 */+-#define MAX_NR_CONSOLES 63 /* serial lines start at 64 */++/*++ * NR_TTY_DEVICES:++ * Value MUST be at least 12 and must never be higher then 63++ */++#define MAX_NR_CONSOLES CONFIG_NR_TTY_DEVICES /* serial lines start above this */+ /* Note: the ioctl VT_GETSTATE does not work for+ consoles 16 and higher (since it returns a short) */++diff --git a/init/Kconfig b/init/Kconfig+index e9119bf54b1f..a1a5a68aea07 100644+--- a/init/Kconfig++++ b/init/Kconfig+@@ -1231,6 +1231,22 @@ config USER_NS++ If unsure, say N.+++config USER_NS_UNPRIVILEGED++ bool "Allow unprivileged users to create namespaces"++ default y++ depends on USER_NS++ help++ When disabled, unprivileged users will not be able to create++ new namespaces. Allowing users to create their own namespaces++ has been part of several recent local privilege escalation++ exploits, so if you need user namespaces but are++ paranoid^Wsecurity-conscious you want to disable this.++++ This setting can be overridden at runtime via the++ kernel.unprivileged_userns_clone sysctl.++++ If unsure, say Y.+++ config PID_NS+ bool "PID Namespaces"+ default y+diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz+index 38ef6d06888e..9948a1f1fb42 100644+--- a/kernel/Kconfig.hz++++ b/kernel/Kconfig.hz+@@ -40,6 +40,41 @@ choice+ on SMP and NUMA systems and exactly dividing by both PAL and+ NTSC frame rates for video and multimedia work.+++ config HZ_500++ bool "500 HZ"++ help++ 500 Hz is a balanced timer frequency. Provides fast interactivity++ on desktops with good smoothness without increasing CPU power++ consumption and sacrificing the battery life on laptops.++++ config HZ_600++ bool "600 HZ"++ help++ 600 Hz is a balanced timer frequency. Provides fast interactivity++ on desktops with good smoothness without increasing CPU power++ consumption and sacrificing the battery life on laptops.++++ config HZ_750++ bool "750 HZ"++ help++ 750 Hz is a balanced timer frequency. Provides fast interactivity++ on desktops with good smoothness without increasing CPU power++ consumption and sacrificing the battery life on laptops.++++ config HZ_600++ bool "600 HZ"++ help++ 600 Hz is a balanced timer frequency. Provides fast interactivity++ on desktops with good smoothness without increasing CPU power++ consumption and sacrificing the battery life on laptops.++++ config HZ_750++ bool "750 HZ"++ help++ 750 Hz is a balanced timer frequency. Provides fast interactivity++ on desktops with good smoothness without increasing CPU power++ consumption and sacrificing the battery life on laptops.+++ config HZ_1000+ bool "1000 HZ"+ help+@@ -53,6 +88,9 @@ config HZ+ default 100 if HZ_100+ default 250 if HZ_250+ default 300 if HZ_300++ default 500 if HZ_500++ default 600 if HZ_600++ default 750 if HZ_750+ default 1000 if HZ_1000++ config SCHED_HRTICK+diff --git a/kernel/cpu.c b/kernel/cpu.c+index 407a2568f35e..cd7605204d4d 100644+--- a/kernel/cpu.c++++ b/kernel/cpu.c+@@ -2616,6 +2616,7 @@ void init_cpu_possible(const struct cpumask *src)+ void init_cpu_online(const struct cpumask *src)+ {+ cpumask_copy(&__cpu_online_mask, src);++ atomic_set(&__num_online_cpus, cpumask_weight(cpu_online_mask));+ }++ void set_cpu_online(unsigned int cpu, bool online)+diff --git a/kernel/fork.c b/kernel/fork.c+index d75a528f7b21..cc89aa4d3a1f 100644+--- a/kernel/fork.c++++ b/kernel/fork.c+@@ -98,6 +98,10 @@+ #include <linux/io_uring.h>+ #include <linux/bpf.h>+++#ifdef CONFIG_USER_NS++#include <linux/user_namespace.h>++#endif+++ #include <asm/pgalloc.h>+ #include <linux/uaccess.h>+ #include <asm/mmu_context.h>+@@ -1922,6 +1926,10 @@ static __latent_entropy struct task_struct *copy_process(+ if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))+ return ERR_PTR(-EINVAL);+++ if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone)++ if (!capable(CAP_SYS_ADMIN))++ return ERR_PTR(-EPERM);+++ /*+ * Thread groups must share signals as well, and detached threads+ * can only be started up within the thread group.+@@ -3026,6 +3034,12 @@ int ksys_unshare(unsigned long unshare_flags)+ if (unshare_flags & CLONE_NEWNS)+ unshare_flags |= CLONE_FS;+++ if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) {++ err = -EPERM;++ if (!capable(CAP_SYS_ADMIN))++ goto bad_unshare_out;++ }+++ err = check_unshare_flags(unshare_flags);+ if (err)+ goto bad_unshare_out;+diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c+index 086a22d1adb7..c6f5f1e84e09 100644+--- a/kernel/futex/syscalls.c++++ b/kernel/futex/syscalls.c+@@ -142,6 +142,7 @@ static __always_inline bool futex_cmd_has_timeout(u32 cmd)+ case FUTEX_LOCK_PI2:+ case FUTEX_WAIT_BITSET:+ case FUTEX_WAIT_REQUEUE_PI:++ case FUTEX_WAIT_MULTIPLE:+ return true;+ }+ return false;+@@ -154,13 +155,79 @@ futex_init_timeout(u32 cmd, u32 op, struct timespec64 *ts, ktime_t *t)+ return -EINVAL;++ *t = timespec64_to_ktime(*ts);+- if (cmd == FUTEX_WAIT)++ if (cmd == FUTEX_WAIT || cmd == FUTEX_WAIT_MULTIPLE)+ *t = ktime_add_safe(ktime_get(), *t);+ else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME))+ *t = timens_ktime_to_host(CLOCK_MONOTONIC, *t);+ return 0;+ }+++/**++ * futex_read_wait_block - Read an array of futex_wait_block from userspace++ * @uaddr: Userspace address of the block++ * @count: Number of blocks to be read++ *++ * This function creates and allocate an array of futex_q (we zero it to++ * initialize the fields) and then, for each futex_wait_block element from++ * userspace, fill a futex_q element with proper values.++ */++inline struct futex_vector *futex_read_wait_block(u32 __user *uaddr, u32 count)++{++ unsigned int i;++ struct futex_vector *futexv;++ struct futex_wait_block fwb;++ struct futex_wait_block __user *entry =++ (struct futex_wait_block __user *)uaddr;++++ if (!count || count > FUTEX_WAITV_MAX)++ return ERR_PTR(-EINVAL);++++ futexv = kcalloc(count, sizeof(*futexv), GFP_KERNEL);++ if (!futexv)++ return ERR_PTR(-ENOMEM);++++ for (i = 0; i < count; i++) {++ if (copy_from_user(&fwb, &entry[i], sizeof(fwb))) {++ kfree(futexv);++ return ERR_PTR(-EFAULT);++ }++++ futexv[i].w.flags = FUTEX_32;++ futexv[i].w.val = fwb.val;++ futexv[i].w.uaddr = (uintptr_t) (fwb.uaddr);++ futexv[i].q = futex_q_init;++ }++++ return futexv;++}++++int futex_wait_multiple(struct futex_vector *vs, unsigned int count,++ struct hrtimer_sleeper *to);++++int futex_opcode_31(ktime_t *abs_time, u32 __user *uaddr, int count)++{++ int ret;++ struct futex_vector *vs;++ struct hrtimer_sleeper *to = NULL, timeout;++++ to = futex_setup_timer(abs_time, &timeout, 0, 0);++++ vs = futex_read_wait_block(uaddr, count);++++ if (IS_ERR(vs))++ return PTR_ERR(vs);++++ ret = futex_wait_multiple(vs, count, abs_time ? to : NULL);++ kfree(vs);++++ if (to) {++ hrtimer_cancel(&to->timer);++ destroy_hrtimer_on_stack(&to->timer);++ }++++ return ret;++}+++ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,+ const struct __kernel_timespec __user *, utime,+ u32 __user *, uaddr2, u32, val3)+@@ -180,6 +247,9 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,+ tp = &t;+ }+++ if (cmd == FUTEX_WAIT_MULTIPLE)++ return futex_opcode_31(tp, uaddr, val);+++ return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);+ }++@@ -370,6 +440,9 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,+ tp = &t;+ }+++ if (cmd == FUTEX_WAIT_MULTIPLE)++ return futex_opcode_31(tp, uaddr, val);+++ return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);+ }+ #endif /* CONFIG_COMPAT_32BIT_TIME */+diff --git a/kernel/sched/core.c b/kernel/sched/core.c+index 848eaa0efe0e..afb671d71c59 100644+--- a/kernel/sched/core.c++++ b/kernel/sched/core.c+@@ -383,7 +383,7 @@ sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }+ * part of the period that we allow rt tasks to run in us.+ * default: 0.95s+ */+-int sysctl_sched_rt_runtime = 950000;++int sysctl_sched_rt_runtime = 980000;+++ /*+diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c+index 5146163bfabb..47d9243bf250 100644+--- a/kernel/sched/fair.c++++ b/kernel/sched/fair.c+@@ -35,8 +35,8 @@+ *+ * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)+ */+-unsigned int sysctl_sched_latency = 6000000ULL;+-static unsigned int normalized_sysctl_sched_latency = 6000000ULL;++unsigned int sysctl_sched_latency = 4000000ULL;++static unsigned int normalized_sysctl_sched_latency = 4000000ULL;++ /*+ * The initial- and re-scaling of tunables is configurable+@@ -56,8 +56,8 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;+ *+ * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)+ */+-unsigned int sysctl_sched_min_granularity = 750000ULL;+-static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL;++unsigned int sysctl_sched_min_granularity = 500000ULL;++static unsigned int normalized_sysctl_sched_min_granularity = 500000ULL;++ /*+ * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks.+@@ -87,8 +87,8 @@ unsigned int sysctl_sched_child_runs_first __read_mostly;+ *+ * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)+ */+-unsigned int sysctl_sched_wakeup_granularity = 1000000UL;+-static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;++unsigned int sysctl_sched_wakeup_granularity = 800000UL;++static unsigned int normalized_sysctl_sched_wakeup_granularity = 800000UL;++ const_debug unsigned int sysctl_sched_migration_cost = 500000UL;++@@ -141,7 +141,7 @@ int __weak arch_asym_cpu_priority(int cpu)+ *+ * (default: 5 msec, units: microseconds)+ */+-unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;++unsigned int sysctl_sched_cfs_bandwidth_slice = 4000UL;+ #endif++ static inline void update_load_add(struct load_weight *lw, unsigned long inc)+diff --git a/kernel/sysctl.c b/kernel/sysctl.c+index 5ae443b2882e..bfbd398505d2 100644+--- a/kernel/sysctl.c++++ b/kernel/sysctl.c+@@ -99,6 +99,9 @@+ #ifdef CONFIG_PERF_EVENTS+ static const int six_hundred_forty_kb = 640 * 1024;+ #endif++#ifdef CONFIG_USER_NS++#include <linux/user_namespace.h>++#endif++ /* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */+ static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE;+@@ -2005,6 +2008,15 @@ static struct ctl_table kern_table[] = {+ .proc_handler = sysrq_sysctl_handler,+ },+ #endif++#ifdef CONFIG_USER_NS++ {++ .procname = "unprivileged_userns_clone",++ .data = &unprivileged_userns_clone,++ .maxlen = sizeof(int),++ .mode = 0644,++ .proc_handler = proc_dointvec,++ },++#endif+ #ifdef CONFIG_PROC_SYSCTL+ {+ .procname = "cad_pid",+diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c+index 6b2e3ca7ee99..0253002184f1 100644+--- a/kernel/user_namespace.c++++ b/kernel/user_namespace.c+@@ -21,6 +21,13 @@+ #include <linux/bsearch.h>+ #include <linux/sort.h>+++/* sysctl */++#ifdef CONFIG_USER_NS_UNPRIVILEGED++int unprivileged_userns_clone = 1;++#else++int unprivileged_userns_clone;++#endif+++ static struct kmem_cache *user_ns_cachep __read_mostly;+ static DEFINE_MUTEX(userns_state_mutex);++diff --git a/mm/internal.h b/mm/internal.h+index d80300392a19..96aa50e46119 100644+--- a/mm/internal.h++++ b/mm/internal.h+@@ -248,6 +248,7 @@ extern void prep_compound_page(struct page *page, unsigned int order);+ extern void post_alloc_hook(struct page *page, unsigned int order,+ gfp_t gfp_flags);+ extern int user_min_free_kbytes;++extern atomic_long_t kswapd_waiters;++ extern void free_unref_page(struct page *page, unsigned int order);+ extern void free_unref_page_list(struct list_head *list);+diff --git a/mm/page-writeback.c b/mm/page-writeback.c+index 91d163f8d36b..9ba34ad877ae 100644+--- a/mm/page-writeback.c++++ b/mm/page-writeback.c+@@ -70,7 +70,7 @@ static long ratelimit_pages = 32;+ /*+ * Start background writeback (via writeback threads) at this percentage+ */+-int dirty_background_ratio = 10;++int dirty_background_ratio = 5;++ /*+ * dirty_background_bytes starts at 0 (disabled) so that it is a function of+@@ -98,7 +98,7 @@ unsigned long vm_dirty_bytes;+ /*+ * The interval between `kupdate'-style writebacks+ */+-unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */++unsigned int dirty_writeback_interval = 10 * 100; /* centiseconds */++ EXPORT_SYMBOL_GPL(dirty_writeback_interval);++diff --git a/mm/page_alloc.c b/mm/page_alloc.c+index 3589febc6d31..9c51d9c8ca13 100644+--- a/mm/page_alloc.c++++ b/mm/page_alloc.c+@@ -121,6 +121,8 @@ typedef int __bitwise fpi_t;+ */+ #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2))+++atomic_long_t kswapd_waiters = ATOMIC_LONG_INIT(0);+++ /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */+ static DEFINE_MUTEX(pcp_batch_high_lock);+ #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8)+@@ -341,7 +343,7 @@ compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = {++ int min_free_kbytes = 1024;+ int user_min_free_kbytes = -1;+-int watermark_boost_factor __read_mostly = 15000;++int watermark_boost_factor __read_mostly;+ int watermark_scale_factor = 10;++ static unsigned long nr_kernel_pages __initdata;+@@ -2683,8 +2685,11 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page,+ * likelihood of future fallbacks. Wake kswapd now as the node+ * may be balanced overall and kswapd will not wake naturally.+ */+- if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))+- set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);++ if (alloc_flags & ALLOC_KSWAPD) {++ boost_watermark(zone);++ if (zone->watermark_boost)++ set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);++ }++ /* We are not allowed to try stealing from the whole block */+ if (!whole_block)+@@ -4878,6 +4883,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,+ int no_progress_loops;+ unsigned int cpuset_mems_cookie;+ int reserve_flags;++ bool woke_kswapd = false;++ /*+ * We also sanity check to catch abuse of atomic reserves being used by+@@ -4924,8 +4930,13 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,+ goto nopage;+ }++- if (alloc_flags & ALLOC_KSWAPD)++ if (alloc_flags & ALLOC_KSWAPD) {++ if (!woke_kswapd) {++ atomic_long_inc(&kswapd_waiters);++ woke_kswapd = true;++ }+ wake_all_kswapds(order, gfp_mask, ac);++ }++ /*+ * The adjusted alloc_flags might result in immediate success, so try+@@ -5130,9 +5141,12 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,+ goto retry;+ }+ fail:+- warn_alloc(gfp_mask, ac->nodemask,+- "page allocation failure: order:%u", order);+ got_pg:++ if (woke_kswapd)++ atomic_long_dec(&kswapd_waiters);++ if (!page)++ warn_alloc(gfp_mask, ac->nodemask,++ "page allocation failure: order:%u", order);+ return page;+ }++diff --git a/mm/vmscan.c b/mm/vmscan.c+index 090bfb605ecf..236e119e2e6a 100644+--- a/mm/vmscan.c++++ b/mm/vmscan.c+@@ -174,7 +174,7 @@ struct scan_control {+ /*+ * From 0 .. 200. Higher means more swappy.+ */+-int vm_swappiness = 60;++int vm_swappiness = 30;++ static void set_task_reclaim_state(struct task_struct *task,+ struct reclaim_state *rs)+@@ -4273,7 +4273,8 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)+ __fs_reclaim_release(_THIS_IP_);+ ret = try_to_freeze();+ __fs_reclaim_acquire(_THIS_IP_);+- if (ret || kthread_should_stop())++ if (ret || kthread_should_stop() ||++ !atomic_long_read(&kswapd_waiters))+ break;++ /*+diff --git a/net/sched/Kconfig b/net/sched/Kconfig+index 1e8ab4749c6c..75122fd6506b 100644+--- a/net/sched/Kconfig++++ b/net/sched/Kconfig+@@ -474,6 +474,9 @@ choice+ config DEFAULT_SFQ+ bool "Stochastic Fair Queue" if NET_SCH_SFQ+++ config DEFAULT_CAKE++ bool "Common Applications Kept Enhanced" if NET_SCH_CAKE+++ config DEFAULT_PFIFO_FAST+ bool "Priority FIFO Fast"+ endchoice+@@ -485,6 +488,7 @@ config DEFAULT_NET_SCH+ default "fq_codel" if DEFAULT_FQ_CODEL+ default "fq_pie" if DEFAULT_FQ_PIE+ default "sfq" if DEFAULT_SFQ++ default "cake" if DEFAULT_CAKE+ default "pfifo_fast"+ endif++--+2.35.1++From 7fc0e544875f4ba792f2720d1cdbb490d17561c2 Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sat, 29 Jan 2022 08:12:02 +0000+Subject: [PATCH] pf-patches++---+ drivers/i2c/i2c-core-base.c | 2 +-+ include/linux/irqdesc.h | 1 ++ kernel/irq/irqdesc.c | 21 ++++++++++++++++++++++ net/mac80211/airtime.c | 12 ++++++++++--+ net/mac80211/rate.c | 12 +++++++++---+ net/mac80211/rc80211_minstrel_ht.c | 7 ++++++-+ 6 files changed, 48 insertions(+), 7 deletions(-)++diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c+index 2c59dd748a49..121bbad56cfa 100644+--- a/drivers/i2c/i2c-core-base.c++++ b/drivers/i2c/i2c-core-base.c+@@ -1424,7 +1424,7 @@ int i2c_handle_smbus_host_notify(struct i2c_adapter *adap, unsigned short addr)+ if (irq <= 0)+ return -ENXIO;++- generic_handle_irq(irq);++ generic_dispatch_irq(irq);++ return 0;+ }+diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h+index 93d270ca0c56..fbec0c019e3b 100644+--- a/include/linux/irqdesc.h++++ b/include/linux/irqdesc.h+@@ -160,6 +160,7 @@ static inline void generic_handle_irq_desc(struct irq_desc *desc)++ int handle_irq_desc(struct irq_desc *desc);+ int generic_handle_irq(unsigned int irq);++int generic_dispatch_irq(unsigned int irq);++ #ifdef CONFIG_IRQ_DOMAIN+ /*+diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c+index 2267e6527db3..3e129f2e19d1 100644+--- a/kernel/irq/irqdesc.c++++ b/kernel/irq/irqdesc.c+@@ -681,6 +681,27 @@ int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq)+ }+ EXPORT_SYMBOL_GPL(generic_handle_domain_irq);+++/**++ * generic_dispatch_irq - Dispatch an interrupt from an interrupt handler++ * @irq: The irq number to handle++ *++ * A wrapper around generic_handle_irq() which ensures that interrupts are++ * disabled when the primary handler of the dispatched irq is invoked.++ * This is useful for interrupt handlers with dispatching to be safe for++ * the forced threaded case.++ */++int generic_dispatch_irq(unsigned int irq)++{++ unsigned long flags;++ int ret;++++ local_irq_save(flags);++ ret = generic_handle_irq(irq);++ local_irq_restore(flags);++ return ret;++}++EXPORT_SYMBOL_GPL(generic_dispatch_irq);+++ /**+ * generic_handle_domain_nmi - Invoke the handler for a HW nmi belonging+ * to a domain.+diff --git a/net/mac80211/airtime.c b/net/mac80211/airtime.c+index 26d2f8ba7029..03fb8070bdae 100644+--- a/net/mac80211/airtime.c++++ b/net/mac80211/airtime.c+@@ -453,9 +453,17 @@ static u32 ieee80211_get_rate_duration(struct ieee80211_hw *hw,+ return 0;+ }++- if (WARN_ON_ONCE((status->encoding != RX_ENC_HE && streams > 4) ||+- (status->encoding == RX_ENC_HE && streams > 8)))++ if (unlikely(status->encoding != RX_ENC_HE && streams > 4)) {++ pr_warn_once("%s: status->encoding != RX_ENC_HE (%u != %d) && streams > 4 (%d > 4)\n",++ __func__, status->encoding, RX_ENC_HE, streams);+ return 0;++ }++++ if (unlikely(status->encoding == RX_ENC_HE && streams > 8)) {++ pr_warn_once("%s: status->encoding == RX_ENC_HE (%u == %d) && streams > 8 (%d > 8)\n",++ __func__, status->encoding, RX_ENC_HE, streams);++ return 0;++ }++ duration = airtime_mcs_groups[group].duration[idx];+ duration <<= airtime_mcs_groups[group].shift;+diff --git a/net/mac80211/rate.c b/net/mac80211/rate.c+index 8c6416129d5b..810ea5cd6681 100644+--- a/net/mac80211/rate.c++++ b/net/mac80211/rate.c+@@ -664,7 +664,9 @@ static void rate_fixup_ratelist(struct ieee80211_vif *vif,+ * needs to be fixed.+ */+ if (rates[i].flags & IEEE80211_TX_RC_MCS) {+- WARN_ON(rates[i].idx > 76);++ if (unlikely(rates[i].idx > 76))++ pr_warn("%s: rates[%d].idx == %d > 76\n",++ __func__, i, rates[i].idx);++ if (!(rates[i].flags & IEEE80211_TX_RC_USE_RTS_CTS) &&+ info->control.use_cts_prot)+@@ -674,7 +676,9 @@ static void rate_fixup_ratelist(struct ieee80211_vif *vif,+ }++ if (rates[i].flags & IEEE80211_TX_RC_VHT_MCS) {+- WARN_ON(ieee80211_rate_get_vht_mcs(&rates[i]) > 9);++ if (unlikely(ieee80211_rate_get_vht_mcs(&rates[i]) > 9))++ pr_warn("%s: ieee80211_rate_get_vht_mcs(&rates[%d]) == %d > 9\n",++ __func__, i, ieee80211_rate_get_vht_mcs(&rates[i]));+ continue;+ }++@@ -685,7 +689,9 @@ static void rate_fixup_ratelist(struct ieee80211_vif *vif,+ }++ /* RC is busted */+- if (WARN_ON_ONCE(rates[i].idx >= sband->n_bitrates)) {++ if (unlikely(rates[i].idx >= sband->n_bitrates)) {++ pr_warn_once("%s: rates[%d].idx == %d >= sband->n_bitrates == %d\n",++ __func__, i, rates[i].idx, sband->n_bitrates);+ rates[i].idx = -1;+ continue;+ }+diff --git a/net/mac80211/rc80211_minstrel_ht.c b/net/mac80211/rc80211_minstrel_ht.c+index 9c3b7fc377c1..a65d55bbb46b 100644+--- a/net/mac80211/rc80211_minstrel_ht.c++++ b/net/mac80211/rc80211_minstrel_ht.c+@@ -438,8 +438,13 @@ minstrel_ht_get_tp_avg(struct minstrel_ht_sta *mi, int group, int rate,++ if (minstrel_ht_is_legacy_group(group))+ overhead = mi->overhead_legacy;+- else++ else {+ ampdu_len = minstrel_ht_avg_ampdu_len(mi);++ if (unlikely(!ampdu_len)) {++ pr_err_once("%s: ampdu_len == 0!\n", __func__);++ ampdu_len = 1;++ }++ }++ nsecs = 1000 * overhead / ampdu_len;+ nsecs += minstrel_mcs_groups[group].duration[rate] <<+--+2.35.0++From 1baca42af85c292893389760d4ee46c0e9a29b22 Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sun, 6 Feb 2022 20:21:57 +0000+Subject: [PATCH] spf-anon++---+ arch/arm64/Kconfig | 1 ++ arch/arm64/mm/fault.c | 62 +++++ arch/powerpc/Kconfig | 1 ++ arch/powerpc/mm/fault.c | 64 +++++ arch/x86/Kconfig | 1 ++ arch/x86/boot/compressed/kaslr.c | 2 ++ arch/x86/mm/fault.c | 63 +++++ drivers/gpu/drm/i915/i915_gpu_error.c | 4 +-+ include/linux/mm.h | 68 +++-+ include/linux/mm_types.h | 33 +-+ include/linux/mmap_lock.h | 109 ++++--+ include/linux/mmu_notifier.h | 52 ++-+ include/linux/percpu-rwsem.h | 13 +-+ include/linux/vm_event.h | 111 +++++++ include/linux/vm_event_item.h | 25 +++ include/linux/vmstat.h | 95 +-----+ kernel/fork.c | 18 +-+ kernel/locking/percpu-rwsem.c | 32 +++ mm/Kconfig | 22 +++ mm/Kconfig.debug | 7 ++ mm/debug.c | 1 ++ mm/memory.c | 470 +++++++++++++++++++-------+ mm/mmap.c | 13 +-+ mm/vmstat.c | 25 +++ scripts/package/builddeb | 4 +-+ 25 files changed, 1042 insertions(+), 254 deletions(-)+ create mode 100644 include/linux/vm_event.h++diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig+index cbcd42decb2a..15ba7cc583d3 100644+--- a/arch/arm64/Kconfig++++ b/arch/arm64/Kconfig+@@ -222,6 +222,7 @@ config ARM64+ select THREAD_INFO_IN_TASK+ select HAVE_ARCH_USERFAULTFD_MINOR if USERFAULTFD+ select TRACE_IRQFLAGS_SUPPORT++ select ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT+ help+ ARM 64-bit (AArch64) Linux support.++diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c+index 77341b160aca..2598795f4e70 100644+--- a/arch/arm64/mm/fault.c++++ b/arch/arm64/mm/fault.c+@@ -25,6 +25,7 @@+ #include <linux/perf_event.h>+ #include <linux/preempt.h>+ #include <linux/hugetlb.h>++#include <linux/vm_event_item.h>++ #include <asm/acpi.h>+ #include <asm/bug.h>+@@ -524,6 +525,11 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,+ unsigned long vm_flags;+ unsigned int mm_flags = FAULT_FLAG_DEFAULT;+ unsigned long addr = untagged_addr(far);++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ struct vm_area_struct *vma;++ struct vm_area_struct pvma;++ unsigned long seq;++#endif++ if (kprobe_page_fault(regs, esr))+ return 0;+@@ -574,6 +580,59 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,++ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);+++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ /*++ * No need to try speculative faults for kernel or++ * single threaded user space.++ */++ if (!(mm_flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1)++ goto no_spf;++++ count_vm_event(SPF_ATTEMPT);++ seq = mmap_seq_read_start(mm);++ if (seq & 1) {++ count_vm_spf_event(SPF_ABORT_ODD);++ goto spf_abort;++ }++ rcu_read_lock();++ vma = __find_vma(mm, addr);++ if (!vma || vma->vm_start > addr) {++ rcu_read_unlock();++ count_vm_spf_event(SPF_ABORT_UNMAPPED);++ goto spf_abort;++ }++ if (!vma_is_anonymous(vma)) {++ rcu_read_unlock();++ count_vm_spf_event(SPF_ABORT_NO_SPECULATE);++ goto spf_abort;++ }++ pvma = *vma;++ rcu_read_unlock();++ if (!mmap_seq_read_check(mm, seq, SPF_ABORT_VMA_COPY))++ goto spf_abort;++ vma = &pvma;++ if (!(vma->vm_flags & vm_flags)) {++ count_vm_spf_event(SPF_ABORT_ACCESS_ERROR);++ goto spf_abort;++ }++ fault = do_handle_mm_fault(vma, addr & PAGE_MASK,++ mm_flags | FAULT_FLAG_SPECULATIVE, seq, regs);++++ /* Quick path to respond to signals */++ if (fault_signal_pending(fault, regs)) {++ if (!user_mode(regs))++ goto no_context;++ return 0;++ }++ if (!(fault & VM_FAULT_RETRY))++ goto done;++++spf_abort:++ count_vm_event(SPF_ABORT);++no_spf:++++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */+++ /*+ * As per x86, we may deadlock here. However, since the kernel only+ * validly references user space from well defined areas of the code,+@@ -612,6 +671,9 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,+ goto retry;+ }+ mmap_read_unlock(mm);++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++done:++#endif++ /*+ * Handle the "normal" (no error) case first.+diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig+index b779603978e1..5f82bc7eee0b 100644+--- a/arch/powerpc/Kconfig++++ b/arch/powerpc/Kconfig+@@ -144,6 +144,7 @@ config PPC+ select ARCH_STACKWALK+ select ARCH_SUPPORTS_ATOMIC_RMW+ select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx || 40x++ select ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT if PPC_BOOK3S_64+ select ARCH_USE_BUILTIN_BSWAP+ select ARCH_USE_CMPXCHG_LOCKREF if PPC64+ select ARCH_USE_MEMTEST+diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c+index eb8ecd7343a9..3f039504e8fd 100644+--- a/arch/powerpc/mm/fault.c++++ b/arch/powerpc/mm/fault.c+@@ -395,6 +395,10 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address,+ int is_write = page_fault_is_write(error_code);+ vm_fault_t fault, major = 0;+ bool kprobe_fault = kprobe_page_fault(regs, 11);++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ struct vm_area_struct pvma;++ unsigned long seq;++#endif++ if (unlikely(debugger_fault_handler(regs) || kprobe_fault))+ return 0;+@@ -451,6 +455,63 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address,+ if (is_exec)+ flags |= FAULT_FLAG_INSTRUCTION;+++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ /*++ * No need to try speculative faults for kernel or++ * single threaded user space.++ */++ if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1)++ goto no_spf;++++ count_vm_event(SPF_ATTEMPT);++ seq = mmap_seq_read_start(mm);++ if (seq & 1) {++ count_vm_spf_event(SPF_ABORT_ODD);++ goto spf_abort;++ }++ rcu_read_lock();++ vma = __find_vma(mm, address);++ if (!vma || vma->vm_start > address) {++ rcu_read_unlock();++ count_vm_spf_event(SPF_ABORT_UNMAPPED);++ goto spf_abort;++ }++ if (!vma_is_anonymous(vma)) {++ rcu_read_unlock();++ count_vm_spf_event(SPF_ABORT_NO_SPECULATE);++ goto spf_abort;++ }++ pvma = *vma;++ rcu_read_unlock();++ if (!mmap_seq_read_check(mm, seq, SPF_ABORT_VMA_COPY))++ goto spf_abort;++ vma = &pvma;++#ifdef CONFIG_PPC_MEM_KEYS++ if (unlikely(access_pkey_error(is_write, is_exec,++ (error_code & DSISR_KEYFAULT), vma))) {++ count_vm_spf_event(SPF_ABORT_ACCESS_ERROR);++ goto spf_abort;++ }++#endif /* CONFIG_PPC_MEM_KEYS */++ if (unlikely(access_error(is_write, is_exec, vma))) {++ count_vm_spf_event(SPF_ABORT_ACCESS_ERROR);++ goto spf_abort;++ }++ fault = do_handle_mm_fault(vma, address,++ flags | FAULT_FLAG_SPECULATIVE, seq, regs);++ major |= fault & VM_FAULT_MAJOR;++++ if (fault_signal_pending(fault, regs))++ return user_mode(regs) ? 0 : SIGBUS;++ if (!(fault & VM_FAULT_RETRY))++ goto done;++++spf_abort:++ count_vm_event(SPF_ABORT);++no_spf:++++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */+++ /* When running in the kernel we expect faults to occur only to+ * addresses in user space. All other faults represent errors in the+ * kernel and should generate an OOPS. Unfortunately, in the case of an+@@ -522,6 +583,9 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address,+ }++ mmap_read_unlock(current->mm);++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++done:++#endif++ if (unlikely(fault & VM_FAULT_ERROR))+ return mm_fault_error(regs, address, fault);+diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig+index 9f5bd41bf660..3697b840a34c 100644+--- a/arch/x86/Kconfig++++ b/arch/x86/Kconfig+@@ -34,6 +34,7 @@ config X86_64+ select SWIOTLB+ select ARCH_HAS_ELFCORE_COMPAT+ select ZONE_DMA32++ select ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT++ config FORCE_DYNAMIC_FTRACE+ def_bool y+diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c+index 411b268bc0a2..dd3c2112a1e7 100644+--- a/arch/x86/boot/compressed/kaslr.c++++ b/arch/x86/boot/compressed/kaslr.c+@@ -625,7 +625,9 @@ static bool process_mem_region(struct mem_vector *region,+ unsigned long minimum,+ unsigned long image_size)+ {++#if defined(CONFIG_MEMORY_HOTREMOVE) && defined(CONFIG_ACPI)+ int i;++#endif+ /*+ * If no immovable memory found, or MEMORY_HOTREMOVE disabled,+ * use @region directly.+diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c+index d0074c6ed31a..a5a19561c319 100644+--- a/arch/x86/mm/fault.c++++ b/arch/x86/mm/fault.c+@@ -1226,6 +1226,10 @@ void do_user_addr_fault(struct pt_regs *regs,+ struct mm_struct *mm;+ vm_fault_t fault;+ unsigned int flags = FAULT_FLAG_DEFAULT;++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ struct vm_area_struct pvma;++ unsigned long seq;++#endif++ tsk = current;+ mm = tsk->mm;+@@ -1323,6 +1327,62 @@ void do_user_addr_fault(struct pt_regs *regs,+ }+ #endif+++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ /*++ * No need to try speculative faults for kernel or++ * single threaded user space.++ */++ if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1)++ goto no_spf;++++ count_vm_event(SPF_ATTEMPT);++ seq = mmap_seq_read_start(mm);++ if (seq & 1) {++ count_vm_spf_event(SPF_ABORT_ODD);++ goto spf_abort;++ }++ rcu_read_lock();++ vma = __find_vma(mm, address);++ if (!vma || vma->vm_start > address) {++ rcu_read_unlock();++ count_vm_spf_event(SPF_ABORT_UNMAPPED);++ goto spf_abort;++ }++ if (!vma_is_anonymous(vma)) {++ rcu_read_unlock();++ count_vm_spf_event(SPF_ABORT_NO_SPECULATE);++ goto spf_abort;++ }++ pvma = *vma;++ rcu_read_unlock();++ if (!mmap_seq_read_check(mm, seq, SPF_ABORT_VMA_COPY))++ goto spf_abort;++ vma = &pvma;++ if (unlikely(access_error(error_code, vma))) {++ count_vm_spf_event(SPF_ABORT_ACCESS_ERROR);++ goto spf_abort;++ }++ fault = do_handle_mm_fault(vma, address,++ flags | FAULT_FLAG_SPECULATIVE, seq, regs);++++ if (!(fault & VM_FAULT_RETRY))++ goto done;++++ /* Quick path to respond to signals */++ if (fault_signal_pending(fault, regs)) {++ if (!user_mode(regs))++ kernelmode_fixup_or_oops(regs, error_code, address,++ SIGBUS, BUS_ADRERR,++ ARCH_DEFAULT_PKEY);++ return;++ }++++spf_abort:++ count_vm_event(SPF_ABORT);++no_spf:++++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */+++ /*+ * Kernel-mode access to the user address space should only occur+ * on well-defined single instructions listed in the exception+@@ -1419,6 +1479,9 @@ void do_user_addr_fault(struct pt_regs *regs,+ }++ mmap_read_unlock(mm);++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++done:++#endif+ if (likely(!(fault & VM_FAULT_ERROR)))+ return;++diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c+index 0633888a411e..0e1ddc5911a5 100644+--- a/drivers/gpu/drm/i915/i915_gpu_error.c++++ b/drivers/gpu/drm/i915/i915_gpu_error.c+@@ -515,7 +515,7 @@ static void error_print_context(struct drm_i915_error_state_buf *m,+ }++ static struct i915_vma_coredump *+-__find_vma(struct i915_vma_coredump *vma, const char *name)++__i915_find_vma(struct i915_vma_coredump *vma, const char *name)+ {+ while (vma) {+ if (strcmp(vma->name, name) == 0)+@@ -529,7 +529,7 @@ __find_vma(struct i915_vma_coredump *vma, const char *name)+ static struct i915_vma_coredump *+ find_batch(const struct intel_engine_coredump *ee)+ {+- return __find_vma(ee->vma, "batch");++ return __i915_find_vma(ee->vma, "batch");+ }++ static void error_print_engine(struct drm_i915_error_state_buf *m,+diff --git a/include/linux/mm.h b/include/linux/mm.h+index 213cc569b192..ad3b5aef9431 100644+--- a/include/linux/mm.h++++ b/include/linux/mm.h+@@ -461,7 +461,8 @@ static inline bool fault_flag_allow_retry_first(enum fault_flag flags)+ { FAULT_FLAG_USER, "USER" }, \+ { FAULT_FLAG_REMOTE, "REMOTE" }, \+ { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \+- { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }++ { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \++ { FAULT_FLAG_SPECULATIVE, "SPECULATIVE" }++ /*+ * vm_fault is filled by the pagefault handler and passed to the vma's+@@ -482,6 +483,10 @@ struct vm_fault {+ };+ enum fault_flag flags; /* FAULT_FLAG_xxx flags+ * XXX: should really be 'const' */++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ unsigned long seq;++ pmd_t orig_pmd;++#endif+ pmd_t *pmd; /* Pointer to pmd entry matching+ * the 'address' */+ pud_t *pud; /* Pointer to pud entry matching+@@ -489,9 +494,11 @@ struct vm_fault {+ */+ union {+ pte_t orig_pte; /* Value of PTE at the time of fault */++#ifndef CONFIG_SPECULATIVE_PAGE_FAULT+ pmd_t orig_pmd; /* Value of PMD at the time of fault,+ * used by PMD fault only.+ */++#endif+ };++ struct page *cow_page; /* Page handler may use for COW fault */+@@ -1857,9 +1864,15 @@ int generic_error_remove_page(struct address_space *mapping, struct page *page);+ int invalidate_inode_page(struct page *page);++ #ifdef CONFIG_MMU+-extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,+- unsigned long address, unsigned int flags,+- struct pt_regs *regs);++extern vm_fault_t do_handle_mm_fault(struct vm_area_struct *vma,++ unsigned long address, unsigned int flags,++ unsigned long seq, struct pt_regs *regs);++static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,++ unsigned long address, unsigned int flags,++ struct pt_regs *regs)++{++ return do_handle_mm_fault(vma, address, flags, 0, regs);++}+ extern int fixup_user_fault(struct mm_struct *mm,+ unsigned long address, unsigned int fault_flags,+ bool *unlocked);+@@ -2751,10 +2764,17 @@ extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);+ #endif++ /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */+-extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);++extern struct vm_area_struct * __find_vma(struct mm_struct * mm, unsigned long addr);+ extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,+ struct vm_area_struct **pprev);+++static inline++struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)++{++ mmap_assert_locked(mm);++ return __find_vma(mm, addr);++}+++ /**+ * find_vma_intersection() - Look up the first VMA which intersects the interval+ * @mm: The process address space.+@@ -3381,5 +3401,43 @@ madvise_set_anon_name(struct mm_struct *mm, unsigned long start,+ }+ #endif+++#ifdef CONFIG_MMU++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++++bool __pte_map_lock(struct vm_fault *vmf);++++static inline bool pte_map_lock(struct vm_fault *vmf)++{++ VM_BUG_ON(vmf->pte);++ return __pte_map_lock(vmf);++}++++static inline bool pte_spinlock(struct vm_fault *vmf)++{++ VM_BUG_ON(!vmf->pte);++ return __pte_map_lock(vmf);++}++++#else /* !CONFIG_SPECULATIVE_PAGE_FAULT */++++#define pte_map_lock(__vmf) \++({ \++ struct vm_fault *vmf = __vmf; \++ vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, \++ vmf->address, &vmf->ptl); \++ true; \++})++++#define pte_spinlock(__vmf) \++({ \++ struct vm_fault *vmf = __vmf; \++ vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd); \++ spin_lock(vmf->ptl); \++ true; \++})++++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */++#endif /* CONFIG_MMU */+++ #endif /* __KERNEL__ */+ #endif /* _LINUX_MM_H */+diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h+index 5140e5feb486..f4ba3d11cbbe 100644+--- a/include/linux/mm_types.h++++ b/include/linux/mm_types.h+@@ -371,16 +371,27 @@ struct anon_vma_name {+ * per VM-area/task. A VM area is any part of the process virtual memory+ * space that has a special rule for the page-fault handlers (ie a shared+ * library, the executable area etc).++ *++ * Note that speculative page faults make an on-stack copy of the VMA,++ * so the structure size matters.++ * (TODO - it would be preferable to copy only the required vma attributes++ * rather than the entire vma).+ */+ struct vm_area_struct {+ /* The first cache line has the info for VMA tree walking. */++- unsigned long vm_start; /* Our start address within vm_mm. */+- unsigned long vm_end; /* The first byte after our end address+- within vm_mm. */++ union {++ struct {++ /* VMA covers [vm_start; vm_end) addresses within mm */++ unsigned long vm_start, vm_end;++- /* linked list of VM areas per task, sorted by address */+- struct vm_area_struct *vm_next, *vm_prev;++ /* linked list of VMAs per task, sorted by address */++ struct vm_area_struct *vm_next, *vm_prev;++ };++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ struct rcu_head vm_rcu; /* Used for deferred freeing. */++#endif++ };++ struct rb_node vm_rb;++@@ -452,6 +463,7 @@ struct vm_area_struct {+ } __randomize_layout;++ struct kioctx_table;++struct percpu_rw_semaphore;+ struct mm_struct {+ struct {+ struct vm_area_struct *mmap; /* list of VMAs */+@@ -524,6 +536,10 @@ struct mm_struct {+ * cacheline.+ */+ struct rw_semaphore mmap_lock;++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ unsigned long mmap_seq;++#endif++++ struct list_head mmlist; /* List of maybe swapped mm's. These+ * are globally strung together off+@@ -594,7 +610,10 @@ struct mm_struct {+ struct file __rcu *exe_file;+ #ifdef CONFIG_MMU_NOTIFIER+ struct mmu_notifier_subscriptions *notifier_subscriptions;+-#endif++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ struct percpu_rw_semaphore *mmu_notifier_lock;++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */++#endif /* CONFIG_MMU_NOTIFIER */+ #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS+ pgtable_t pmd_huge_pte; /* protected by page_table_lock */+ #endif+@@ -791,6 +810,7 @@ typedef struct {+ * @FAULT_FLAG_REMOTE: The fault is not for current task/mm.+ * @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch.+ * @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals.++ * @FAULT_FLAG_SPECULATIVE: The fault is handled without holding the mmap lock.+ *+ * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify+ * whether we would allow page faults to retry by specifying these two+@@ -822,6 +842,7 @@ enum fault_flag {+ FAULT_FLAG_REMOTE = 1 << 7,+ FAULT_FLAG_INSTRUCTION = 1 << 8,+ FAULT_FLAG_INTERRUPTIBLE = 1 << 9,++ FAULT_FLAG_SPECULATIVE = 1 << 10,+ };++ #endif /* _LINUX_MM_TYPES_H */+diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h+index 96e113e23d04..747805ce07b8 100644+--- a/include/linux/mmap_lock.h++++ b/include/linux/mmap_lock.h+@@ -7,9 +7,18 @@+ #include <linux/rwsem.h>+ #include <linux/tracepoint-defs.h>+ #include <linux/types.h>++#include <linux/vm_event.h>++-#define MMAP_LOCK_INITIALIZER(name) \+- .mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock),++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++#define MMAP_LOCK_SEQ_INITIALIZER(name) \++ .mmap_seq = 0,++#else++#define MMAP_LOCK_SEQ_INITIALIZER(name)++#endif++++#define MMAP_LOCK_INITIALIZER(name) \++ .mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock), \++ MMAP_LOCK_SEQ_INITIALIZER(name)++ DECLARE_TRACEPOINT(mmap_lock_start_locking);+ DECLARE_TRACEPOINT(mmap_lock_acquire_returned);+@@ -63,13 +72,66 @@ static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)+ static inline void mmap_init_lock(struct mm_struct *mm)+ {+ init_rwsem(&mm->mmap_lock);++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ mm->mmap_seq = 0;++#endif++}++++static inline void __mmap_seq_write_lock(struct mm_struct *mm)++{++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ VM_BUG_ON_MM(mm->mmap_seq & 1, mm);++ mm->mmap_seq++;++ smp_wmb();++#endif++}++++static inline void __mmap_seq_write_unlock(struct mm_struct *mm)++{++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ smp_wmb();++ mm->mmap_seq++;++ VM_BUG_ON_MM(mm->mmap_seq & 1, mm);++#endif++}++++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++static inline unsigned long mmap_seq_read_start(struct mm_struct *mm)++{++ unsigned long seq;++++ seq = READ_ONCE(mm->mmap_seq);++ smp_rmb();++ return seq;++}++++static inline bool __mmap_seq_read_check(struct mm_struct *mm,++ unsigned long seq)++{++ smp_rmb();++ return seq == READ_ONCE(mm->mmap_seq);++}++++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT_STATS++static inline bool mmap_seq_read_check(struct mm_struct *mm, unsigned long seq,++ enum vm_event_item fail_event)++{++ if (__mmap_seq_read_check(mm, seq))++ return true;++ count_vm_event(fail_event);++ return false;+ }++#else++#define mmap_seq_read_check(mm, seq, fail) __mmap_seq_read_check(mm, seq)++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT_STATS */++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */++ static inline void mmap_write_lock(struct mm_struct *mm)+ {+ __mmap_lock_trace_start_locking(mm, true);+ down_write(&mm->mmap_lock);+ __mmap_lock_trace_acquire_returned(mm, true, true);++ __mmap_seq_write_lock(mm);+ }++ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)+@@ -77,37 +139,44 @@ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)+ __mmap_lock_trace_start_locking(mm, true);+ down_write_nested(&mm->mmap_lock, subclass);+ __mmap_lock_trace_acquire_returned(mm, true, true);++ __mmap_seq_write_lock(mm);+ }++ static inline int mmap_write_lock_killable(struct mm_struct *mm)+ {+- int ret;++ int error;++ __mmap_lock_trace_start_locking(mm, true);+- ret = down_write_killable(&mm->mmap_lock);+- __mmap_lock_trace_acquire_returned(mm, true, ret == 0);+- return ret;++ error = down_write_killable(&mm->mmap_lock);++ __mmap_lock_trace_acquire_returned(mm, true, !error);++ if (likely(!error))++ __mmap_seq_write_lock(mm);++ return error;+ }++ static inline bool mmap_write_trylock(struct mm_struct *mm)+ {+- bool ret;++ bool ok;++ __mmap_lock_trace_start_locking(mm, true);+- ret = down_write_trylock(&mm->mmap_lock) != 0;+- __mmap_lock_trace_acquire_returned(mm, true, ret);+- return ret;++ ok = down_write_trylock(&mm->mmap_lock) != 0;++ __mmap_lock_trace_acquire_returned(mm, true, ok);++ if (likely(ok))++ __mmap_seq_write_lock(mm);++ return ok;+ }++ static inline void mmap_write_unlock(struct mm_struct *mm)+ {+ __mmap_lock_trace_released(mm, true);++ __mmap_seq_write_unlock(mm);+ up_write(&mm->mmap_lock);+ }++ static inline void mmap_write_downgrade(struct mm_struct *mm)+ {+ __mmap_lock_trace_acquire_returned(mm, false, true);++ __mmap_seq_write_unlock(mm);+ downgrade_write(&mm->mmap_lock);+ }++@@ -120,22 +189,22 @@ static inline void mmap_read_lock(struct mm_struct *mm)++ static inline int mmap_read_lock_killable(struct mm_struct *mm)+ {+- int ret;++ int error;++ __mmap_lock_trace_start_locking(mm, false);+- ret = down_read_killable(&mm->mmap_lock);+- __mmap_lock_trace_acquire_returned(mm, false, ret == 0);+- return ret;++ error = down_read_killable(&mm->mmap_lock);++ __mmap_lock_trace_acquire_returned(mm, false, !error);++ return error;+ }++ static inline bool mmap_read_trylock(struct mm_struct *mm)+ {+- bool ret;++ bool ok;++ __mmap_lock_trace_start_locking(mm, false);+- ret = down_read_trylock(&mm->mmap_lock) != 0;+- __mmap_lock_trace_acquire_returned(mm, false, ret);+- return ret;++ ok = down_read_trylock(&mm->mmap_lock) != 0;++ __mmap_lock_trace_acquire_returned(mm, false, ok);++ return ok;+ }++ static inline void mmap_read_unlock(struct mm_struct *mm)+@@ -162,9 +231,9 @@ static inline void mmap_assert_write_locked(struct mm_struct *mm)+ VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm);+ }++-static inline int mmap_lock_is_contended(struct mm_struct *mm)++static inline bool mmap_lock_is_contended(struct mm_struct *mm)+ {+- return rwsem_is_contended(&mm->mmap_lock);++ return rwsem_is_contended(&mm->mmap_lock) != 0;+ }++ #endif /* _LINUX_MMAP_LOCK_H */+diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h+index 45fc2c81e370..d0430410fdd8 100644+--- a/include/linux/mmu_notifier.h++++ b/include/linux/mmu_notifier.h+@@ -6,6 +6,8 @@+ #include <linux/spinlock.h>+ #include <linux/mm_types.h>+ #include <linux/mmap_lock.h>++#include <linux/percpu-rwsem.h>++#include <linux/slab.h>+ #include <linux/srcu.h>+ #include <linux/interval_tree.h>++@@ -499,15 +501,35 @@ static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,+ __mmu_notifier_invalidate_range(mm, start, end);+ }++-static inline void mmu_notifier_subscriptions_init(struct mm_struct *mm)++static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)+ {++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ mm->mmu_notifier_lock = kzalloc(sizeof(struct percpu_rw_semaphore), GFP_KERNEL);++ if (!mm->mmu_notifier_lock)++ return false;++ if (percpu_init_rwsem(mm->mmu_notifier_lock)) {++ kfree(mm->mmu_notifier_lock);++ return false;++ }++#endif+++ mm->notifier_subscriptions = NULL;++ return true;+ }++ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)+ {+ if (mm_has_notifiers(mm))+ __mmu_notifier_subscriptions_destroy(mm);++++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ if (!in_atomic()) {++ percpu_free_rwsem(mm->mmu_notifier_lock);++ kfree(mm->mmu_notifier_lock);++ } else {++ percpu_rwsem_async_destroy(mm->mmu_notifier_lock);++ }++#endif+ }+++@@ -724,8 +746,9 @@ static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,+ {+ }++-static inline void mmu_notifier_subscriptions_init(struct mm_struct *mm)++static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)+ {++ return true;+ }++ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)+@@ -749,4 +772,29 @@ static inline void mmu_notifier_synchronize(void)++ #endif /* CONFIG_MMU_NOTIFIER */+++#if defined(CONFIG_MMU_NOTIFIER) && defined(CONFIG_SPECULATIVE_PAGE_FAULT)++++static inline bool mmu_notifier_trylock(struct mm_struct *mm)++{++ return percpu_down_read_trylock(mm->mmu_notifier_lock);++}++++static inline void mmu_notifier_unlock(struct mm_struct *mm)++{++ percpu_up_read(mm->mmu_notifier_lock);++}++++#else++++static inline bool mmu_notifier_trylock(struct mm_struct *mm)++{++ return true;++}++++static inline void mmu_notifier_unlock(struct mm_struct *mm)++{++}++++#endif+++ #endif /* _LINUX_MMU_NOTIFIER_H */+diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h+index 5fda40f97fe9..bf1668fc9c5e 100644+--- a/include/linux/percpu-rwsem.h++++ b/include/linux/percpu-rwsem.h+@@ -13,7 +13,14 @@ struct percpu_rw_semaphore {+ struct rcu_sync rss;+ unsigned int __percpu *read_count;+ struct rcuwait writer;+- wait_queue_head_t waiters;++ /*++ * destroy_list_entry is used during object destruction when waiters++ * can't be used, therefore reusing the same space.++ */++ union {++ wait_queue_head_t waiters;++ struct list_head destroy_list_entry;++ };+ atomic_t block;+ #ifdef CONFIG_DEBUG_LOCK_ALLOC+ struct lockdep_map dep_map;+@@ -127,8 +134,12 @@ extern void percpu_up_write(struct percpu_rw_semaphore *);+ extern int __percpu_init_rwsem(struct percpu_rw_semaphore *,+ const char *, struct lock_class_key *);+++/* Can't be called in atomic context. */+ extern void percpu_free_rwsem(struct percpu_rw_semaphore *);+++/* Invokes percpu_free_rwsem and frees the semaphore from a worker thread. */++extern void percpu_rwsem_async_destroy(struct percpu_rw_semaphore *sem);+++ #define percpu_init_rwsem(sem) \+ ({ \+ static struct lock_class_key rwsem_key; \+diff --git a/include/linux/vm_event.h b/include/linux/vm_event.h+new file mode 100644+index 000000000000..689a21387dad+--- /dev/null++++ b/include/linux/vm_event.h+@@ -0,0 +1,111 @@++/* SPDX-License-Identifier: GPL-2.0 */++#ifndef _LINUX_VM_EVENT_H++#define _LINUX_VM_EVENT_H++++#include <linux/types.h>++#include <linux/percpu.h>++#include <linux/mmzone.h>++#include <linux/vm_event_item.h>++#include <linux/atomic.h>++++#ifdef CONFIG_VM_EVENT_COUNTERS++/*++ * Light weight per cpu counter implementation.++ *++ * Counters should only be incremented and no critical kernel component++ * should rely on the counter values.++ *++ * Counters are handled completely inline. On many platforms the code++ * generated will simply be the increment of a global address.++ */++++struct vm_event_state {++ unsigned long event[NR_VM_EVENT_ITEMS];++};++++DECLARE_PER_CPU(struct vm_event_state, vm_event_states);++++/*++ * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the++ * local_irq_disable overhead.++ */++static inline void __count_vm_event(enum vm_event_item item)++{++ raw_cpu_inc(vm_event_states.event[item]);++}++++static inline void count_vm_event(enum vm_event_item item)++{++ this_cpu_inc(vm_event_states.event[item]);++}++++static inline void __count_vm_events(enum vm_event_item item, long delta)++{++ raw_cpu_add(vm_event_states.event[item], delta);++}++++static inline void count_vm_events(enum vm_event_item item, long delta)++{++ this_cpu_add(vm_event_states.event[item], delta);++}++++extern void all_vm_events(unsigned long *);++++extern void vm_events_fold_cpu(int cpu);++++#else++++/* Disable counters */++static inline void count_vm_event(enum vm_event_item item)++{++}++static inline void count_vm_events(enum vm_event_item item, long delta)++{++}++static inline void __count_vm_event(enum vm_event_item item)++{++}++static inline void __count_vm_events(enum vm_event_item item, long delta)++{++}++static inline void all_vm_events(unsigned long *ret)++{++}++static inline void vm_events_fold_cpu(int cpu)++{++}++++#endif /* CONFIG_VM_EVENT_COUNTERS */++++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT_STATS++#define count_vm_spf_event(x) count_vm_event(x)++#else++#define count_vm_spf_event(x) do {} while (0)++#endif++++#ifdef CONFIG_NUMA_BALANCING++#define count_vm_numa_event(x) count_vm_event(x)++#define count_vm_numa_events(x, y) count_vm_events(x, y)++#else++#define count_vm_numa_event(x) do {} while (0)++#define count_vm_numa_events(x, y) do { (void)(y); } while (0)++#endif /* CONFIG_NUMA_BALANCING */++++#ifdef CONFIG_DEBUG_TLBFLUSH++#define count_vm_tlb_event(x) count_vm_event(x)++#define count_vm_tlb_events(x, y) count_vm_events(x, y)++#else++#define count_vm_tlb_event(x) do {} while (0)++#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)++#endif++++#ifdef CONFIG_DEBUG_VM_VMACACHE++#define count_vm_vmacache_event(x) count_vm_event(x)++#else++#define count_vm_vmacache_event(x) do {} while (0)++#endif++++#define __count_zid_vm_events(item, zid, delta) \++ __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)++++#endif /* _LINUX_VM_EVENT_H */+diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h+index 7b2363388bfa..0390b81b1e71 100644+--- a/include/linux/vm_event_item.h++++ b/include/linux/vm_event_item.h+@@ -133,6 +133,31 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,+ #ifdef CONFIG_X86+ DIRECT_MAP_LEVEL2_SPLIT,+ DIRECT_MAP_LEVEL3_SPLIT,++#endif++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ SPF_ATTEMPT,++ SPF_ABORT,++#endif++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT_STATS++ SPF_ABORT_ODD,++ SPF_ABORT_UNMAPPED,++ SPF_ABORT_NO_SPECULATE,++ SPF_ABORT_VMA_COPY,++ SPF_ABORT_ACCESS_ERROR,++ SPF_ABORT_PUD,++ SPF_ABORT_PMD,++ SPF_ABORT_ANON_VMA,++ SPF_ABORT_PTE_MAP_LOCK_SEQ1,++ SPF_ABORT_PTE_MAP_LOCK_PMD,++ SPF_ABORT_PTE_MAP_LOCK_PTL,++ SPF_ABORT_PTE_MAP_LOCK_SEQ2,++ SPF_ABORT_USERFAULTFD,++ SPF_ABORT_FAULT,++ SPF_ABORT_SWAP,++ SPF_ATTEMPT_ANON,++ SPF_ATTEMPT_NUMA,++ SPF_ATTEMPT_PTE,++ SPF_ATTEMPT_WP,+ #endif+ NR_VM_EVENT_ITEMS+ };+diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h+index bfe38869498d..7c3c892ce89a 100644+--- a/include/linux/vmstat.h++++ b/include/linux/vmstat.h+@@ -6,6 +6,7 @@+ #include <linux/percpu.h>+ #include <linux/mmzone.h>+ #include <linux/vm_event_item.h>++#include <linux/vm_event.h>+ #include <linux/atomic.h>+ #include <linux/static_key.h>+ #include <linux/mmdebug.h>+@@ -40,100 +41,6 @@ enum writeback_stat_item {+ NR_VM_WRITEBACK_STAT_ITEMS,+ };++-#ifdef CONFIG_VM_EVENT_COUNTERS+-/*+- * Light weight per cpu counter implementation.+- *+- * Counters should only be incremented and no critical kernel component+- * should rely on the counter values.+- *+- * Counters are handled completely inline. On many platforms the code+- * generated will simply be the increment of a global address.+- */+-+-struct vm_event_state {+- unsigned long event[NR_VM_EVENT_ITEMS];+-};+-+-DECLARE_PER_CPU(struct vm_event_state, vm_event_states);+-+-/*+- * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the+- * local_irq_disable overhead.+- */+-static inline void __count_vm_event(enum vm_event_item item)+-{+- raw_cpu_inc(vm_event_states.event[item]);+-}+-+-static inline void count_vm_event(enum vm_event_item item)+-{+- this_cpu_inc(vm_event_states.event[item]);+-}+-+-static inline void __count_vm_events(enum vm_event_item item, long delta)+-{+- raw_cpu_add(vm_event_states.event[item], delta);+-}+-+-static inline void count_vm_events(enum vm_event_item item, long delta)+-{+- this_cpu_add(vm_event_states.event[item], delta);+-}+-+-extern void all_vm_events(unsigned long *);+-+-extern void vm_events_fold_cpu(int cpu);+-+-#else+-+-/* Disable counters */+-static inline void count_vm_event(enum vm_event_item item)+-{+-}+-static inline void count_vm_events(enum vm_event_item item, long delta)+-{+-}+-static inline void __count_vm_event(enum vm_event_item item)+-{+-}+-static inline void __count_vm_events(enum vm_event_item item, long delta)+-{+-}+-static inline void all_vm_events(unsigned long *ret)+-{+-}+-static inline void vm_events_fold_cpu(int cpu)+-{+-}+-+-#endif /* CONFIG_VM_EVENT_COUNTERS */+-+-#ifdef CONFIG_NUMA_BALANCING+-#define count_vm_numa_event(x) count_vm_event(x)+-#define count_vm_numa_events(x, y) count_vm_events(x, y)+-#else+-#define count_vm_numa_event(x) do {} while (0)+-#define count_vm_numa_events(x, y) do { (void)(y); } while (0)+-#endif /* CONFIG_NUMA_BALANCING */+-+-#ifdef CONFIG_DEBUG_TLBFLUSH+-#define count_vm_tlb_event(x) count_vm_event(x)+-#define count_vm_tlb_events(x, y) count_vm_events(x, y)+-#else+-#define count_vm_tlb_event(x) do {} while (0)+-#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)+-#endif+-+-#ifdef CONFIG_DEBUG_VM_VMACACHE+-#define count_vm_vmacache_event(x) count_vm_event(x)+-#else+-#define count_vm_vmacache_event(x) do {} while (0)+-#endif+-+-#define __count_zid_vm_events(item, zid, delta) \+- __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)+-+ /*+ * Zone and node-based page accounting with per cpu differentials.+ */+diff --git a/kernel/fork.c b/kernel/fork.c+index d75a528f7b21..34600fe86743 100644+--- a/kernel/fork.c++++ b/kernel/fork.c+@@ -371,9 +371,24 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)+ return new;+ }+++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++static void __vm_area_free(struct rcu_head *head)++{++ struct vm_area_struct *vma = container_of(head, struct vm_area_struct,++ vm_rcu);++ kmem_cache_free(vm_area_cachep, vma);++}++#endif+++ void vm_area_free(struct vm_area_struct *vma)+ {+ free_vma_anon_name(vma);++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ if (atomic_read(&vma->vm_mm->mm_users) > 1) {++ call_rcu(&vma->vm_rcu, __vm_area_free);++ return;++ }++#endif+ kmem_cache_free(vm_area_cachep, vma);+ }++@@ -1056,7 +1071,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,+ mm_init_owner(mm, p);+ mm_init_pasid(mm);+ RCU_INIT_POINTER(mm->exe_file, NULL);+- mmu_notifier_subscriptions_init(mm);++ if (!mmu_notifier_subscriptions_init(mm))++ goto fail_nopgd;+ init_tlb_flush_pending(mm);+ #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS+ mm->pmd_huge_pte = NULL;+diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c+index 70a32a576f3f..a3d37bf83c60 100644+--- a/kernel/locking/percpu-rwsem.c++++ b/kernel/locking/percpu-rwsem.c+@@ -7,6 +7,7 @@+ #include <linux/rcupdate.h>+ #include <linux/sched.h>+ #include <linux/sched/task.h>++#include <linux/slab.h>+ #include <linux/errno.h>++ int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,+@@ -268,3 +269,34 @@ void percpu_up_write(struct percpu_rw_semaphore *sem)+ rcu_sync_exit(&sem->rss);+ }+ EXPORT_SYMBOL_GPL(percpu_up_write);++++static LIST_HEAD(destroy_list);++static DEFINE_SPINLOCK(destroy_list_lock);++++static void destroy_list_workfn(struct work_struct *work)++{++ struct percpu_rw_semaphore *sem, *sem2;++ LIST_HEAD(to_destroy);++++ spin_lock(&destroy_list_lock);++ list_splice_init(&destroy_list, &to_destroy);++ spin_unlock(&destroy_list_lock);++++ if (list_empty(&to_destroy))++ return;++++ list_for_each_entry_safe(sem, sem2, &to_destroy, destroy_list_entry) {++ percpu_free_rwsem(sem);++ kfree(sem);++ }++}++++static DECLARE_WORK(destroy_list_work, destroy_list_workfn);++++void percpu_rwsem_async_destroy(struct percpu_rw_semaphore *sem)++{++ spin_lock(&destroy_list_lock);++ list_add_tail(&sem->destroy_list_entry, &destroy_list);++ spin_unlock(&destroy_list_lock);++ schedule_work(&destroy_list_work);++}+diff --git a/mm/Kconfig b/mm/Kconfig+index 3326ee3903f3..d304fca0f293 100644+--- a/mm/Kconfig++++ b/mm/Kconfig+@@ -894,4 +894,26 @@ config ANON_VMA_NAME++ source "mm/damon/Kconfig"+++config ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT++ def_bool n++++config SPECULATIVE_PAGE_FAULT++ bool "Speculative page faults"++ default y++ depends on ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT && MMU && SMP++ help++ Try to handle user space page faults without holding the mmap lock.++++ Instead of blocking writers through the use of mmap lock,++ the page fault handler merely verifies, at the end of the page++ fault, that no writers have been running concurrently with it.++++ In high concurrency situations, the speculative fault handler++ gains a throughput advantage by avoiding having to update the++ mmap lock reader count.++++ If the check fails due to a concurrent writer, or due to hitting++ an unsupported case, the fault handler falls back to classical++ processing using the mmap read lock.+++ endmenu+diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug+index 5bd5bb097252..73b61cc95562 100644+--- a/mm/Kconfig.debug++++ b/mm/Kconfig.debug+@@ -174,3 +174,10 @@ config PTDUMP_DEBUGFS+ kernel.++ If in doubt, say N.++++config SPECULATIVE_PAGE_FAULT_STATS++ bool "Additional statistics for speculative page faults"++ depends on SPECULATIVE_PAGE_FAULT++ help++ Additional statistics for speculative page faults.++ If in doubt, say N.+diff --git a/mm/debug.c b/mm/debug.c+index bc9ac87f0e08..40d3f358b75c 100644+--- a/mm/debug.c++++ b/mm/debug.c+@@ -220,6 +220,7 @@ void dump_mm(const struct mm_struct *mm)+ mm->def_flags, &mm->def_flags+ );+ }++EXPORT_SYMBOL(dump_mm);++ static bool page_init_poisoning __read_mostly = true;++diff --git a/mm/memory.c b/mm/memory.c+index c125c4969913..a5754309eaae 100644+--- a/mm/memory.c++++ b/mm/memory.c+@@ -2725,6 +2725,100 @@ int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,+ }+ EXPORT_SYMBOL_GPL(apply_to_existing_page_range);+++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++++/*++ * speculative_page_walk_begin() ... speculative_page_walk_end() protects++ * against races with page table reclamation.++ *++ * This is similar to what fast GUP does, but fast GUP also needs to++ * protect against races with THP page splitting, so it always needs++ * to disable interrupts.++ * Speculative page faults only need to protect against page table reclamation,++ * so rcu_read_lock() is sufficient in the MMU_GATHER_RCU_TABLE_FREE case.++ */++#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE++#define speculative_page_walk_begin() rcu_read_lock()++#define speculative_page_walk_end() rcu_read_unlock()++#else++#define speculative_page_walk_begin() local_irq_disable()++#define speculative_page_walk_end() local_irq_enable()++#endif++++bool __pte_map_lock(struct vm_fault *vmf)++{++#ifdef CONFIG_TRANSPARENT_HUGEPAGE++ pmd_t pmdval;++#endif++ pte_t *pte = vmf->pte;++ spinlock_t *ptl;++++ if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) {++ vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);++ if (!pte)++ vmf->pte = pte_offset_map(vmf->pmd, vmf->address);++ spin_lock(vmf->ptl);++ return true;++ }++++ speculative_page_walk_begin();++ if (!mmap_seq_read_check(vmf->vma->vm_mm, vmf->seq,++ SPF_ABORT_PTE_MAP_LOCK_SEQ1))++ goto fail;++ /*++ * The mmap sequence count check guarantees that the page++ * tables are still valid at that point, and++ * speculative_page_walk_begin() ensures that they stay around.++ */++#ifdef CONFIG_TRANSPARENT_HUGEPAGE++ /*++ * We check if the pmd value is still the same to ensure that there++ * is not a huge collapse operation in progress in our back.++ */++ pmdval = READ_ONCE(*vmf->pmd);++ if (!pmd_same(pmdval, vmf->orig_pmd)) {++ count_vm_spf_event(SPF_ABORT_PTE_MAP_LOCK_PMD);++ goto fail;++ }++#endif++ ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);++ if (!pte)++ pte = pte_offset_map(vmf->pmd, vmf->address);++ /*++ * Try locking the page table.++ *++ * Note that we might race against zap_pte_range() which++ * invalidates TLBs while holding the page table lock.++ * We are still under the speculative_page_walk_begin() section,++ * and zap_pte_range() could thus deadlock with us if we tried++ * using spin_lock() here.++ *++ * We also don't want to retry until spin_trylock() succeeds,++ * because of the starvation potential against a stream of lockers.++ */++ if (unlikely(!spin_trylock(ptl))) {++ count_vm_spf_event(SPF_ABORT_PTE_MAP_LOCK_PTL);++ goto fail;++ }++ if (!mmap_seq_read_check(vmf->vma->vm_mm, vmf->seq,++ SPF_ABORT_PTE_MAP_LOCK_SEQ2))++ goto unlock_fail;++ speculative_page_walk_end();++ vmf->pte = pte;++ vmf->ptl = ptl;++ return true;++++unlock_fail:++ spin_unlock(ptl);++fail:++ if (pte)++ pte_unmap(pte);++ speculative_page_walk_end();++ return false;++}++++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */+++ /*+ * handle_pte_fault chooses page fault handler according to an entry which was+ * read non-atomically. Before making any commitment, on those architectures+@@ -2999,20 +3093,28 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)+ pte_t entry;+ int page_copied = 0;+ struct mmu_notifier_range range;++ vm_fault_t ret = VM_FAULT_OOM;++- if (unlikely(anon_vma_prepare(vma)))+- goto oom;++ if (unlikely(!vma->anon_vma)) {++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {++ count_vm_spf_event(SPF_ABORT_ANON_VMA);++ ret = VM_FAULT_RETRY;++ goto out;++ }++ if (__anon_vma_prepare(vma))++ goto out;++ }++ if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {+ new_page = alloc_zeroed_user_highpage_movable(vma,+ vmf->address);+ if (!new_page)+- goto oom;++ goto out;+ } else {+ new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,+ vmf->address);+ if (!new_page)+- goto oom;++ goto out;++ if (!cow_user_page(new_page, old_page, vmf)) {+ /*+@@ -3029,11 +3131,16 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)+ }++ if (mem_cgroup_charge(page_folio(new_page), mm, GFP_KERNEL))+- goto oom_free_new;++ goto out_free_new;+ cgroup_throttle_swaprate(new_page, GFP_KERNEL);++ __SetPageUptodate(new_page);+++ if ((vmf->flags & FAULT_FLAG_SPECULATIVE) &&++ !mmu_notifier_trylock(mm)) {++ ret = VM_FAULT_RETRY;++ goto out_free_new;++ }+ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,+ vmf->address & PAGE_MASK,+ (vmf->address & PAGE_MASK) + PAGE_SIZE);+@@ -3042,7 +3149,11 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)+ /*+ * Re-check the pte - we dropped the lock+ */+- vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);++ if (!pte_map_lock(vmf)) {++ ret = VM_FAULT_RETRY;++ /* put_page() will uncharge the page */++ goto out_notify;++ }+ if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {+ if (old_page) {+ if (!PageAnon(old_page)) {+@@ -3117,6 +3228,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)+ * the above ptep_clear_flush_notify() did already call it.+ */+ mmu_notifier_invalidate_range_only_end(&range);++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)++ mmu_notifier_unlock(mm);+ if (old_page) {+ /*+ * Don't let another task, with possibly unlocked vma,+@@ -3133,12 +3246,16 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)+ put_page(old_page);+ }+ return page_copied ? VM_FAULT_WRITE : 0;+-oom_free_new:++out_notify:++ mmu_notifier_invalidate_range_only_end(&range);++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)++ mmu_notifier_unlock(mm);++out_free_new:+ put_page(new_page);+-oom:++out:+ if (old_page)+ put_page(old_page);+- return VM_FAULT_OOM;++ return ret;+ }++ /**+@@ -3183,6 +3300,7 @@ static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)+ {+ struct vm_area_struct *vma = vmf->vma;+++ VM_BUG_ON(vmf->flags & FAULT_FLAG_SPECULATIVE);+ if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {+ vm_fault_t ret;++@@ -3203,6 +3321,8 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf)+ struct vm_area_struct *vma = vmf->vma;+ vm_fault_t ret = VM_FAULT_WRITE;+++ VM_BUG_ON(vmf->flags & FAULT_FLAG_SPECULATIVE);+++ get_page(vmf->page);++ if (vma->vm_ops && vma->vm_ops->page_mkwrite) {+@@ -3254,8 +3374,15 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)+ {+ struct vm_area_struct *vma = vmf->vma;+++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)++ count_vm_spf_event(SPF_ATTEMPT_WP);+++ if (userfaultfd_pte_wp(vma, *vmf->pte)) {+ pte_unmap_unlock(vmf->pte, vmf->ptl);++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {++ count_vm_spf_event(SPF_ABORT_USERFAULTFD);++ return VM_FAULT_RETRY;++ }+ return handle_userfault(vmf, VM_UFFD_WP);+ }++@@ -3281,6 +3408,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)+ return wp_pfn_shared(vmf);++ pte_unmap_unlock(vmf->pte, vmf->ptl);++ vmf->pte = NULL;+ return wp_page_copy(vmf);+ }++@@ -3319,6 +3447,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)+ get_page(vmf->page);++ pte_unmap_unlock(vmf->pte, vmf->ptl);++ vmf->pte = NULL;+ return wp_page_copy(vmf);+ }++@@ -3501,6 +3630,12 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)+ vm_fault_t ret = 0;+ void *shadow = NULL;+++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {++ pte_unmap(vmf->pte);++ count_vm_spf_event(SPF_ABORT_SWAP);++ return VM_FAULT_RETRY;++ }+++ if (!pte_unmap_same(vmf))+ goto out;++@@ -3726,10 +3861,13 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)+ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)+ {+ struct vm_area_struct *vma = vmf->vma;+- struct page *page;++ struct page *page = NULL;+ vm_fault_t ret = 0;+ pte_t entry;+++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)++ count_vm_spf_event(SPF_ATTEMPT_ANON);+++ /* File mapping without ->vm_ops ? */+ if (vma->vm_flags & VM_SHARED)+ return VM_FAULT_SIGBUS;+@@ -3747,7 +3885,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)+ if (pte_alloc(vma->vm_mm, vmf->pmd))+ return VM_FAULT_OOM;++- /* See comment in handle_pte_fault() */++ /* See comment in __handle_mm_fault() */+ if (unlikely(pmd_trans_unstable(vmf->pmd)))+ return 0;++@@ -3756,78 +3894,80 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)+ !mm_forbids_zeropage(vma->vm_mm)) {+ entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),+ vma->vm_page_prot));+- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,+- vmf->address, &vmf->ptl);+- if (!pte_none(*vmf->pte)) {+- update_mmu_tlb(vma, vmf->address, vmf->pte);+- goto unlock;+- }+- ret = check_stable_address_space(vma->vm_mm);+- if (ret)+- goto unlock;+- /* Deliver the page fault to userland, check inside PT lock */+- if (userfaultfd_missing(vma)) {+- pte_unmap_unlock(vmf->pte, vmf->ptl);+- return handle_userfault(vmf, VM_UFFD_MISSING);++ } else {++ /* Allocate our own private page. */++ if (unlikely(!vma->anon_vma)) {++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {++ count_vm_spf_event(SPF_ABORT_ANON_VMA);++ return VM_FAULT_RETRY;++ }++ if (__anon_vma_prepare(vma))++ goto oom;+ }+- goto setpte;+- }+-+- /* Allocate our own private page. */+- if (unlikely(anon_vma_prepare(vma)))+- goto oom;+- page = alloc_zeroed_user_highpage_movable(vma, vmf->address);+- if (!page)+- goto oom;++ page = alloc_zeroed_user_highpage_movable(vma, vmf->address);++ if (!page)++ goto oom;++- if (mem_cgroup_charge(page_folio(page), vma->vm_mm, GFP_KERNEL))+- goto oom_free_page;+- cgroup_throttle_swaprate(page, GFP_KERNEL);++ if (mem_cgroup_charge(page_folio(page), vma->vm_mm, GFP_KERNEL))++ goto oom_free_page;++ cgroup_throttle_swaprate(page, GFP_KERNEL);++- /*+- * The memory barrier inside __SetPageUptodate makes sure that+- * preceding stores to the page contents become visible before+- * the set_pte_at() write.+- */+- __SetPageUptodate(page);++ /*++ * The memory barrier inside __SetPageUptodate makes sure that++ * preceding stores to the page contents become visible before++ * the set_pte_at() write.++ */++ __SetPageUptodate(page);++- entry = mk_pte(page, vma->vm_page_prot);+- entry = pte_sw_mkyoung(entry);+- if (vma->vm_flags & VM_WRITE)+- entry = pte_mkwrite(pte_mkdirty(entry));++ entry = mk_pte(page, vma->vm_page_prot);++ entry = pte_sw_mkyoung(entry);++ if (vma->vm_flags & VM_WRITE)++ entry = pte_mkwrite(pte_mkdirty(entry));++ }++- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,+- &vmf->ptl);+- if (!pte_none(*vmf->pte)) {+- update_mmu_cache(vma, vmf->address, vmf->pte);++ if (!pte_map_lock(vmf)) {++ ret = VM_FAULT_RETRY;+ goto release;+ }++ if (!pte_none(*vmf->pte)) {++ update_mmu_tlb(vma, vmf->address, vmf->pte);++ goto unlock;++ }++ ret = check_stable_address_space(vma->vm_mm);+ if (ret)+- goto release;++ goto unlock;++ /* Deliver the page fault to userland, check inside PT lock */+ if (userfaultfd_missing(vma)) {+ pte_unmap_unlock(vmf->pte, vmf->ptl);+- put_page(page);++ if (page)++ put_page(page);++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {++ count_vm_spf_event(SPF_ABORT_USERFAULTFD);++ return VM_FAULT_RETRY;++ }+ return handle_userfault(vmf, VM_UFFD_MISSING);+ }++- inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);+- page_add_new_anon_rmap(page, vma, vmf->address, false);+- lru_cache_add_inactive_or_unevictable(page, vma);+-setpte:++ if (page) {++ inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);++ page_add_new_anon_rmap(page, vma, vmf->address, false);++ lru_cache_add_inactive_or_unevictable(page, vma);++ }+++ set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);++ /* No need to invalidate - it was non-present before */+ update_mmu_cache(vma, vmf->address, vmf->pte);++ pte_unmap_unlock(vmf->pte, vmf->ptl);++ return 0;+ unlock:+ pte_unmap_unlock(vmf->pte, vmf->ptl);+- return ret;+ release:+- put_page(page);+- goto unlock;++ if (page)++ put_page(page);++ return ret;+ oom_free_page:+ put_page(page);+ oom:+@@ -4051,7 +4191,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf)+ return VM_FAULT_OOM;+ }++- /* See comment in handle_pte_fault() */++ /* See comment in __handle_mm_fault() */+ if (pmd_devmap_trans_unstable(vmf->pmd))+ return 0;++@@ -4279,6 +4419,8 @@ static vm_fault_t do_fault(struct vm_fault *vmf)+ struct mm_struct *vm_mm = vma->vm_mm;+ vm_fault_t ret;+++ VM_BUG_ON(vmf->flags & FAULT_FLAG_SPECULATIVE);+++ /*+ * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND+ */+@@ -4348,13 +4490,16 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)+ bool was_writable = pte_savedwrite(vmf->orig_pte);+ int flags = 0;+++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)++ count_vm_spf_event(SPF_ATTEMPT_NUMA);+++ /*+ * The "pte" at this point cannot be used safely without+ * validation through pte_unmap_same(). It's of NUMA type but+ * the pfn may be screwed if the read is non atomic.+ */+- vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);+- spin_lock(vmf->ptl);++ if (!pte_spinlock(vmf))++ return VM_FAULT_RETRY;+ if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {+ pte_unmap_unlock(vmf->pte, vmf->ptl);+ goto out;+@@ -4516,53 +4661,6 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)+ {+ pte_t entry;++- if (unlikely(pmd_none(*vmf->pmd))) {+- /*+- * Leave __pte_alloc() until later: because vm_ops->fault may+- * want to allocate huge page, and if we expose page table+- * for an instant, it will be difficult to retract from+- * concurrent faults and from rmap lookups.+- */+- vmf->pte = NULL;+- } else {+- /*+- * If a huge pmd materialized under us just retry later. Use+- * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead+- * of pmd_trans_huge() to ensure the pmd didn't become+- * pmd_trans_huge under us and then back to pmd_none, as a+- * result of MADV_DONTNEED running immediately after a huge pmd+- * fault in a different thread of this mm, in turn leading to a+- * misleading pmd_trans_huge() retval. All we have to ensure is+- * that it is a regular pmd that we can walk with+- * pte_offset_map() and we can do that through an atomic read+- * in C, which is what pmd_trans_unstable() provides.+- */+- if (pmd_devmap_trans_unstable(vmf->pmd))+- return 0;+- /*+- * A regular pmd is established and it can't morph into a huge+- * pmd from under us anymore at this point because we hold the+- * mmap_lock read mode and khugepaged takes it in write mode.+- * So now it's safe to run pte_offset_map().+- */+- vmf->pte = pte_offset_map(vmf->pmd, vmf->address);+- vmf->orig_pte = *vmf->pte;+-+- /*+- * some architectures can have larger ptes than wordsize,+- * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and+- * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic+- * accesses. The code below just needs a consistent view+- * for the ifs and we later double check anyway with the+- * ptl lock held. So here a barrier will do.+- */+- barrier();+- if (pte_none(vmf->orig_pte)) {+- pte_unmap(vmf->pte);+- vmf->pte = NULL;+- }+- }+-+ if (!vmf->pte) {+ if (vma_is_anonymous(vmf->vma))+ return do_anonymous_page(vmf);+@@ -4576,8 +4674,11 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)+ if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))+ return do_numa_page(vmf);++- vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);+- spin_lock(vmf->ptl);++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)++ count_vm_spf_event(SPF_ATTEMPT_PTE);++++ if (!pte_spinlock(vmf))++ return VM_FAULT_RETRY;+ entry = vmf->orig_pte;+ if (unlikely(!pte_same(*vmf->pte, entry))) {+ update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);+@@ -4617,7 +4718,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)+ * return value. See filemap_fault() and __folio_lock_or_retry().+ */+ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,+- unsigned long address, unsigned int flags)++ unsigned long address, unsigned int flags, unsigned long seq)+ {+ struct vm_fault vmf = {+ .vma = vma,+@@ -4632,6 +4733,87 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,+ p4d_t *p4d;+ vm_fault_t ret;+++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ if (flags & FAULT_FLAG_SPECULATIVE) {++ pgd_t pgdval;++ p4d_t p4dval;++ pud_t pudval;++++ vmf.seq = seq;++++ speculative_page_walk_begin();++ pgd = pgd_offset(mm, address);++ pgdval = READ_ONCE(*pgd);++ if (pgd_none(pgdval) || unlikely(pgd_bad(pgdval))) {++ count_vm_spf_event(SPF_ABORT_PUD);++ goto spf_fail;++ }++++ p4d = p4d_offset(pgd, address);++ p4dval = READ_ONCE(*p4d);++ if (p4d_none(p4dval) || unlikely(p4d_bad(p4dval))) {++ count_vm_spf_event(SPF_ABORT_PUD);++ goto spf_fail;++ }++++ vmf.pud = pud_offset(p4d, address);++ pudval = READ_ONCE(*vmf.pud);++ if (pud_none(pudval) || unlikely(pud_bad(pudval)) ||++ unlikely(pud_trans_huge(pudval)) ||++ unlikely(pud_devmap(pudval))) {++ count_vm_spf_event(SPF_ABORT_PUD);++ goto spf_fail;++ }++++ vmf.pmd = pmd_offset(vmf.pud, address);++ vmf.orig_pmd = READ_ONCE(*vmf.pmd);++++ /*++ * pmd_none could mean that a hugepage collapse is in++ * progress in our back as collapse_huge_page() mark++ * it before invalidating the pte (which is done once++ * the IPI is catched by all CPU and we have interrupt++ * disabled). For this reason we cannot handle THP in++ * a speculative way since we can't safely identify an++ * in progress collapse operation done in our back on++ * that PMD.++ */++ if (unlikely(pmd_none(vmf.orig_pmd) ||++ is_swap_pmd(vmf.orig_pmd) ||++ pmd_trans_huge(vmf.orig_pmd) ||++ pmd_devmap(vmf.orig_pmd))) {++ count_vm_spf_event(SPF_ABORT_PMD);++ goto spf_fail;++ }++++ /*++ * The above does not allocate/instantiate page-tables because++ * doing so would lead to the possibility of instantiating++ * page-tables after free_pgtables() -- and consequently++ * leaking them.++ *++ * The result is that we take at least one non-speculative++ * fault per PMD in order to instantiate it.++ */++++ vmf.pte = pte_offset_map(vmf.pmd, address);++ vmf.orig_pte = READ_ONCE(*vmf.pte);++ barrier();++ if (pte_none(vmf.orig_pte)) {++ pte_unmap(vmf.pte);++ vmf.pte = NULL;++ }++++ speculative_page_walk_end();++++ return handle_pte_fault(&vmf);++++ spf_fail:++ speculative_page_walk_end();++ return VM_FAULT_RETRY;++ }++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */+++ pgd = pgd_offset(mm, address);+ p4d = p4d_alloc(mm, pgd, address);+ if (!p4d)+@@ -4702,6 +4884,53 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,+ }+ }+++ if (unlikely(pmd_none(*vmf.pmd))) {++ /*++ * Leave __pte_alloc() until later: because vm_ops->fault may++ * want to allocate huge page, and if we expose page table++ * for an instant, it will be difficult to retract from++ * concurrent faults and from rmap lookups.++ */++ vmf.pte = NULL;++ } else {++ /*++ * If a huge pmd materialized under us just retry later. Use++ * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead++ * of pmd_trans_huge() to ensure the pmd didn't become++ * pmd_trans_huge under us and then back to pmd_none, as a++ * result of MADV_DONTNEED running immediately after a huge pmd++ * fault in a different thread of this mm, in turn leading to a++ * misleading pmd_trans_huge() retval. All we have to ensure is++ * that it is a regular pmd that we can walk with++ * pte_offset_map() and we can do that through an atomic read++ * in C, which is what pmd_trans_unstable() provides.++ */++ if (pmd_devmap_trans_unstable(vmf.pmd))++ return 0;++ /*++ * A regular pmd is established and it can't morph into a huge++ * pmd from under us anymore at this point because we hold the++ * mmap_lock read mode and khugepaged takes it in write mode.++ * So now it's safe to run pte_offset_map().++ */++ vmf.pte = pte_offset_map(vmf.pmd, vmf.address);++ vmf.orig_pte = *vmf.pte;++++ /*++ * some architectures can have larger ptes than wordsize,++ * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and++ * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic++ * accesses. The code below just needs a consistent view++ * for the ifs and we later double check anyway with the++ * ptl lock held. So here a barrier will do.++ */++ barrier();++ if (pte_none(vmf.orig_pte)) {++ pte_unmap(vmf.pte);++ vmf.pte = NULL;++ }++ }+++ return handle_pte_fault(&vmf);+ }++@@ -4772,11 +5001,14 @@ static inline void mm_account_fault(struct pt_regs *regs,+ * The mmap_lock may have been released depending on flags and our+ * return value. See filemap_fault() and __folio_lock_or_retry().+ */+-vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,+- unsigned int flags, struct pt_regs *regs)++vm_fault_t do_handle_mm_fault(struct vm_area_struct *vma,++ unsigned long address, unsigned int flags,++ unsigned long seq, struct pt_regs *regs)+ {+ vm_fault_t ret;+++ VM_BUG_ON((flags & FAULT_FLAG_SPECULATIVE) && !vma_is_anonymous(vma));+++ __set_current_state(TASK_RUNNING);++ count_vm_event(PGFAULT);+@@ -4797,10 +5029,12 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,+ if (flags & FAULT_FLAG_USER)+ mem_cgroup_enter_user_fault();++- if (unlikely(is_vm_hugetlb_page(vma)))++ if (unlikely(is_vm_hugetlb_page(vma))) {++ VM_BUG_ON(flags & FAULT_FLAG_SPECULATIVE);+ ret = hugetlb_fault(vma->vm_mm, vma, address, flags);+- else+- ret = __handle_mm_fault(vma, address, flags);++ } else {++ ret = __handle_mm_fault(vma, address, flags, seq);++ }++ if (flags & FAULT_FLAG_USER) {+ mem_cgroup_exit_user_fault();+@@ -4818,7 +5052,7 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,++ return ret;+ }+-EXPORT_SYMBOL_GPL(handle_mm_fault);++EXPORT_SYMBOL_GPL(do_handle_mm_fault);++ #ifndef __PAGETABLE_P4D_FOLDED+ /*+diff --git a/mm/mmap.c b/mm/mmap.c+index 1e8fdb0b51ed..a67c3600d995 100644+--- a/mm/mmap.c++++ b/mm/mmap.c+@@ -2276,12 +2276,11 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,+ EXPORT_SYMBOL(get_unmapped_area);++ /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */+-struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)++struct vm_area_struct *__find_vma(struct mm_struct *mm, unsigned long addr)+ {+ struct rb_node *rb_node;+ struct vm_area_struct *vma;++- mmap_assert_locked(mm);+ /* Check the cache first. */+ vma = vmacache_find(mm, addr);+ if (likely(vma))+@@ -2308,7 +2307,7 @@ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)+ return vma;+ }++-EXPORT_SYMBOL(find_vma);++EXPORT_SYMBOL(__find_vma);++ /*+ * Same as find_vma, but also return a pointer to the previous VMA in *pprev.+@@ -3593,6 +3592,10 @@ int mm_take_all_locks(struct mm_struct *mm)++ mutex_lock(&mm_all_locks_mutex);+++#if defined(CONFIG_MMU_NOTIFIER) && defined(CONFIG_SPECULATIVE_PAGE_FAULT)++ percpu_down_write(mm->mmu_notifier_lock);++#endif+++ for (vma = mm->mmap; vma; vma = vma->vm_next) {+ if (signal_pending(current))+ goto out_unlock;+@@ -3680,6 +3683,10 @@ void mm_drop_all_locks(struct mm_struct *mm)+ vm_unlock_mapping(vma->vm_file->f_mapping);+ }+++#if defined(CONFIG_MMU_NOTIFIER) && defined(CONFIG_SPECULATIVE_PAGE_FAULT)++ percpu_up_write(mm->mmu_notifier_lock);++#endif+++ mutex_unlock(&mm_all_locks_mutex);+ }++diff --git a/mm/vmstat.c b/mm/vmstat.c+index 4057372745d0..20ac17cf582a 100644+--- a/mm/vmstat.c++++ b/mm/vmstat.c+@@ -1390,6 +1390,31 @@ const char * const vmstat_text[] = {+ "direct_map_level2_splits",+ "direct_map_level3_splits",+ #endif++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT++ "spf_attempt",++ "spf_abort",++#endif++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT_STATS++ "SPF_ABORT_ODD",++ "SPF_ABORT_UNMAPPED",++ "SPF_ABORT_NO_SPECULATE",++ "SPF_ABORT_VMA_COPY",++ "SPF_ABORT_ACCESS_ERROR",++ "SPF_ABORT_PUD",++ "SPF_ABORT_PMD",++ "SPF_ABORT_ANON_VMA",++ "SPF_ABORT_PTE_MAP_LOCK_SEQ1",++ "SPF_ABORT_PTE_MAP_LOCK_PMD",++ "SPF_ABORT_PTE_MAP_LOCK_PTL",++ "SPF_ABORT_PTE_MAP_LOCK_SEQ2",++ "SPF_ABORT_USERFAULTFD",++ "SPF_ABORT_FAULT",++ "SPF_ABORT_SWAP",++ "SPF_ATTEMPT_ANON",++ "SPF_ATTEMPT_NUMA",++ "SPF_ATTEMPT_PTE",++ "SPF_ATTEMPT_WP",++#endif+ #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */+ };+ #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */+diff --git a/scripts/package/builddeb b/scripts/package/builddeb+index 91a502bb97e8..b83a4a09a75d 100755+--- a/scripts/package/builddeb++++ b/scripts/package/builddeb+@@ -50,7 +50,7 @@ create_package() {++ # Create the package+ dpkg-gencontrol -p$pname -P"$pdir"+- dpkg-deb $dpkg_deb_opts ${KDEB_COMPRESS:+-Z$KDEB_COMPRESS} --build "$pdir" ..++ dpkg-deb $dpkg_deb_opts ${KDEB_COMPRESS:+-Z$KDEB_COMPRESS} --build "$pdir" "${UPLOAD_DIR:-..}"+ }++ deploy_kernel_headers () {+@@ -208,7 +208,7 @@ EOF+ chmod 755 "$tmpdir/DEBIAN/$script"+ done++-if [ "$ARCH" != "um" ]; then++if [ "$ARCH" != "um" -a -z "$NO_HEADER_PACKAGES" ]; then+ if is_enabled CONFIG_MODULES; then+ deploy_kernel_headers debian/linux-headers+ create_package linux-headers-$version debian/linux-headers+--+2.35.1++From 7e908601be64f8331e3b3f18c25f98c991d03c6e Mon Sep 17 00:00:00 2001+From: "P. Jung" <[email protected]>+Date: Sun, 6 Feb 2022 20:40:19 +0000+Subject: [PATCH] zstd-patches++Signed-off-by: P. Jung <[email protected]>+---+ arch/arm/boot/deflate_xip_data.sh | 2 +-+ arch/powerpc/boot/wrapper | 2 +-+ arch/x86/boot/compressed/Makefile | 5 ++ include/linux/zstd_lib.h | 4 +-+ init/Kconfig | 44 +++++++ lib/zstd/compress/zstd_double_fast.c | 61 ++++++--+ lib/zstd/compress/zstd_fast.c | 69 +++++++--+ lib/zstd/compress/zstd_lazy.c | 223 +++++++++++----------------+ scripts/Makefile.lib | 24 +--+ scripts/Makefile.modinst | 7 +-+ scripts/file-size.pl | 8 ++ scripts/file-size.sh | 4 -+ scripts/link-vmlinux.sh | 4 +-+ 13 files changed, 274 insertions(+), 183 deletions(-)+ create mode 100755 scripts/file-size.pl+ delete mode 100755 scripts/file-size.sh++diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh+index 304495c3c2c5..14cfa2babb93 100755+--- a/arch/arm/boot/deflate_xip_data.sh++++ b/arch/arm/boot/deflate_xip_data.sh+@@ -43,7 +43,7 @@ data_start=$(($__data_loc - $base_offset))+ data_end=$(($_edata_loc - $base_offset))++ # Make sure data occupies the last part of the file.+-file_end=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" "$XIPIMAGE")++file_end=$(${PERL} "${srctree}/scripts/file-size.pl" "$XIPIMAGE")+ if [ "$file_end" != "$data_end" ]; then+ printf "end of xipImage doesn't match with _edata_loc (%#x vs %#x)\n" \+ $(($file_end + $base_offset)) $_edata_loc 1>&2+diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper+index 9184eda780fd..9f9ee8613432 100755+--- a/arch/powerpc/boot/wrapper++++ b/arch/powerpc/boot/wrapper+@@ -380,7 +380,7 @@ vmz="$tmpdir/`basename \"$kernel\"`.$ext"++ # Calculate the vmlinux.strip size+ ${CROSS}objcopy $objflags "$kernel" "$vmz.$$"+-strip_size=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" "$vmz.$$")++strip_size=$(${PERL} "${srctree}/scripts/file-size.pl" "$vmz.$$")++ if [ -z "$cacheit" -o ! -f "$vmz$compression" -o "$vmz$compression" -ot "$kernel" ]; then+ # recompress the image if we need to+diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile+index 6115274fe10f..5f04502554e2 100644+--- a/arch/x86/boot/compressed/Makefile++++ b/arch/x86/boot/compressed/Makefile+@@ -135,8 +135,13 @@ $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE+ $(call if_changed,lzo_with_size)+ $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE+ $(call if_changed,lz4_with_size)++ifdef CONFIG_KERNEL_ZSTD_ULTRA+ $(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE+ $(call if_changed,zstd22_with_size)++else++$(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE++ $(call if_changed,zstd)++endif++ suffix-$(CONFIG_KERNEL_GZIP) := gz+ suffix-$(CONFIG_KERNEL_BZIP2) := bz2+diff --git a/include/linux/zstd_lib.h b/include/linux/zstd_lib.h+index b8c7dbf98390..6b91758b61af 100644+--- a/include/linux/zstd_lib.h++++ b/include/linux/zstd_lib.h+@@ -1330,7 +1330,7 @@ ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,++ /*! ZSTD_mergeBlockDelimiters() :+ * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals+- * by merging them into into the literals of the next sequence.++ * by merging them into the literals of the next sequence.+ *+ * As such, the final generated result has no explicit representation of block boundaries,+ * and the final last literals segment is not represented in the sequences.+@@ -1377,7 +1377,7 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size+ /*! ZSTD_writeSkippableFrame() :+ * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.+ *+- * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,++ * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,+ * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.+ * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so+ * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.+diff --git a/init/Kconfig b/init/Kconfig+index a1a5a68aea07..c315dfa7b107 100644+--- a/init/Kconfig++++ b/init/Kconfig+@@ -329,6 +329,25 @@ config KERNEL_UNCOMPRESSED++ endchoice+++menu "ZSTD compression options"++ depends on KERNEL_ZSTD++++config KERNEL_ZSTD_LEVEL++ int "Compression level (1-19)"++ range 1 19++ default 19++ help++ Choose a compression level for zstd kernel compression.++ Default is 19.++++config KERNEL_ZSTD_LEVEL_ULTRA++ bool "Ultra compression"++ help++ Selecting this overrides the chosen compression level and enables++ the highest possible compression level for zstd (ultra-22).++++endmenu+++ config DEFAULT_INIT+ string "Default init path"+ default ""+@@ -2306,6 +2325,31 @@ config MODULE_DECOMPRESS++ If unsure, say N.+++menu "ZSTD module compression options"++ depends on MODULE_COMPRESS_ZSTD++++config MODULE_COMPRESS_ZSTD_LEVEL++ int "Compression level (1-19)"++ range 1 19++ default 19++ help++ Compression level used by zstd for compressing modules.++++config MODULE_COMPRESS_ZSTD_ULTRA++ bool "Enable ZSTD ultra compression"++ help++ Compress modules with ZSTD using the highest possible compression.++++config MODULE_COMPRESS_ZSTD_LEVEL_ULTRA++ int "Compression level (20-22)"++ depends on MODULE_COMPRESS_ZSTD_ULTRA++ range 20 22++ default 22++ help++ Ultra compression level used by zstd for compressing modules.++++endmenu+++ config MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS+ bool "Allow loading of modules with missing namespace imports"+ help+diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c+index b0424d23ac57..fb941a5b70f5 100644+--- a/lib/zstd/compress/zstd_double_fast.c++++ b/lib/zstd/compress/zstd_double_fast.c+@@ -313,6 +313,26 @@ size_t ZSTD_compressBlock_doubleFast_generic(+ return (size_t)(iend - anchor);+ }+++#define ZSTD_GEN_FN(dictMode, mls) \++ static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \++ void const* src, size_t srcSize) \++ { \++ return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_##dictMode); \++ }++++ZSTD_GEN_FN(noDict, 4)++ZSTD_GEN_FN(noDict, 5)++ZSTD_GEN_FN(noDict, 6)++ZSTD_GEN_FN(noDict, 7)++++ZSTD_GEN_FN(dictMatchState, 4)++ZSTD_GEN_FN(dictMatchState, 5)++ZSTD_GEN_FN(dictMatchState, 6)++ZSTD_GEN_FN(dictMatchState, 7)++++#undef ZSTD_GEN_FN++++ size_t ZSTD_compressBlock_doubleFast(+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],+@@ -323,13 +343,13 @@ size_t ZSTD_compressBlock_doubleFast(+ {+ default: /* includes case 3 */+ case 4 :+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict);++ return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize);+ case 5 :+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict);++ return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize);+ case 6 :+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict);++ return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize);+ case 7 :+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict);++ return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize);+ }+ }++@@ -343,13 +363,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(+ {+ default: /* includes case 3 */+ case 4 :+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState);++ return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize);+ case 5 :+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState);++ return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize);+ case 6 :+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState);++ return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize);+ case 7 :+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState);++ return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize);+ }+ }++@@ -385,7 +405,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(++ /* if extDict is invalidated due to maxDistance, switch to "regular" variant */+ if (prefixStartIndex == dictStartIndex)+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict);++ return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize);++ /* Search Loop */+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */+@@ -499,6 +519,21 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(+ }++++#define ZSTD_GEN_FN(mls) \++ static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \++ void const* src, size_t srcSize) \++ { \++ return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \++ }++++ZSTD_GEN_FN(4)++ZSTD_GEN_FN(5)++ZSTD_GEN_FN(6)++ZSTD_GEN_FN(7)++++#undef ZSTD_GEN_FN+++ size_t ZSTD_compressBlock_doubleFast_extDict(+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],+ void const* src, size_t srcSize)+@@ -508,12 +543,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict(+ {+ default: /* includes case 3 */+ case 4 :+- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);++ return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize);+ case 5 :+- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);++ return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize);+ case 6 :+- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);++ return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize);+ case 7 :+- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);++ return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize);+ }+ }+diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c+index 96b7d48e2868..e0652e31d421 100644+--- a/lib/zstd/compress/zstd_fast.c++++ b/lib/zstd/compress/zstd_fast.c+@@ -182,6 +182,20 @@ ZSTD_compressBlock_fast_generic(+ return (size_t)(iend - anchor);+ }+++#define ZSTD_GEN_FN(mls) \++ static size_t ZSTD_compressBlock_fast_##mls( \++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \++ void const* src, size_t srcSize) \++ { \++ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); \++ }++++ZSTD_GEN_FN(4)++ZSTD_GEN_FN(5)++ZSTD_GEN_FN(6)++ZSTD_GEN_FN(7)++++#undef ZSTD_GEN_FN++ size_t ZSTD_compressBlock_fast(+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],+@@ -193,13 +207,13 @@ size_t ZSTD_compressBlock_fast(+ {+ default: /* includes case 3 */+ case 4 :+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4);++ return ZSTD_compressBlock_fast_4(ms, seqStore, rep, src, srcSize);+ case 5 :+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5);++ return ZSTD_compressBlock_fast_5(ms, seqStore, rep, src, srcSize);+ case 6 :+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6);++ return ZSTD_compressBlock_fast_6(ms, seqStore, rep, src, srcSize);+ case 7 :+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7);++ return ZSTD_compressBlock_fast_7(ms, seqStore, rep, src, srcSize);+ }+ }++@@ -351,6 +365,21 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(+ return (size_t)(iend - anchor);+ }+++#define ZSTD_GEN_FN(mls) \++ static size_t ZSTD_compressBlock_fast_dictMatchState_##mls( \++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \++ void const* src, size_t srcSize) \++ { \++ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, mls); \++ }++++ZSTD_GEN_FN(4)++ZSTD_GEN_FN(5)++ZSTD_GEN_FN(6)++ZSTD_GEN_FN(7)++++#undef ZSTD_GEN_FN+++ size_t ZSTD_compressBlock_fast_dictMatchState(+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],+ void const* src, size_t srcSize)+@@ -361,13 +390,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState(+ {+ default: /* includes case 3 */+ case 4 :+- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4);++ return ZSTD_compressBlock_fast_dictMatchState_4(ms, seqStore, rep, src, srcSize);+ case 5 :+- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5);++ return ZSTD_compressBlock_fast_dictMatchState_5(ms, seqStore, rep, src, srcSize);+ case 6 :+- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6);++ return ZSTD_compressBlock_fast_dictMatchState_6(ms, seqStore, rep, src, srcSize);+ case 7 :+- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7);++ return ZSTD_compressBlock_fast_dictMatchState_7(ms, seqStore, rep, src, srcSize);+ }+ }++@@ -402,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(++ /* switch to "regular" variant if extDict is invalidated due to maxDistance */+ if (prefixStartIndex == dictStartIndex)+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls);++ return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize);++ /* Search Loop */+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */+@@ -475,6 +504,20 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(+ return (size_t)(iend - anchor);+ }+++#define ZSTD_GEN_FN(mls) \++ static size_t ZSTD_compressBlock_fast_extDict_##mls( \++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \++ void const* src, size_t srcSize) \++ { \++ return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \++ }++++ZSTD_GEN_FN(4)++ZSTD_GEN_FN(5)++ZSTD_GEN_FN(6)++ZSTD_GEN_FN(7)++++#undef ZSTD_GEN_FN++ size_t ZSTD_compressBlock_fast_extDict(+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],+@@ -485,12 +528,12 @@ size_t ZSTD_compressBlock_fast_extDict(+ {+ default: /* includes case 3 */+ case 4 :+- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);++ return ZSTD_compressBlock_fast_extDict_4(ms, seqStore, rep, src, srcSize);+ case 5 :+- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);++ return ZSTD_compressBlock_fast_extDict_5(ms, seqStore, rep, src, srcSize);+ case 6 :+- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);++ return ZSTD_compressBlock_fast_extDict_6(ms, seqStore, rep, src, srcSize);+ case 7 :+- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);++ return ZSTD_compressBlock_fast_extDict_7(ms, seqStore, rep, src, srcSize);+ }+ }+diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c+index fb54d4e28a2b..1db22db5b1e9 100644+--- a/lib/zstd/compress/zstd_lazy.c++++ b/lib/zstd/compress/zstd_lazy.c+@@ -392,55 +392,6 @@ ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms,+ }+++-static size_t+-ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms,+- const BYTE* ip, const BYTE* const iLimit,+- size_t* offsetPtr)+-{+- switch(ms->cParams.minMatch)+- {+- default : /* includes case 3 */+- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);+- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict);+- case 7 :+- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict);+- }+-}+-+-+-static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS (+- ZSTD_matchState_t* ms,+- const BYTE* ip, const BYTE* const iLimit,+- size_t* offsetPtr)+-{+- switch(ms->cParams.minMatch)+- {+- default : /* includes case 3 */+- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);+- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState);+- case 7 :+- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState);+- }+-}+-+-+-static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (+- ZSTD_matchState_t* ms,+- const BYTE* ip, const BYTE* const iLimit,+- size_t* offsetPtr)+-{+- switch(ms->cParams.minMatch)+- {+- default : /* includes case 3 */+- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);+- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict);+- case 7 :+- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict);+- }+-}+-+-+-+ /* *********************************+ * Hash Chain+ ***********************************/+@@ -595,7 +546,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B++ /* inlining is important to hardwire a hot branch (template emulation) */+ FORCE_INLINE_TEMPLATE+-size_t ZSTD_HcFindBestMatch_generic (++size_t ZSTD_HcFindBestMatch(+ ZSTD_matchState_t* ms,+ const BYTE* const ip, const BYTE* const iLimit,+ size_t* offsetPtr,+@@ -783,76 +734,106 @@ size_t ZSTD_HcFindBestMatch_generic (+ return ml;+ }+++typedef size_t (*searchMax_f)(++ ZSTD_matchState_t* ms,++ const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);++-FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (+- ZSTD_matchState_t* ms,+- const BYTE* ip, const BYTE* const iLimit,+- size_t* offsetPtr)+-{+- switch(ms->cParams.minMatch)+- {+- default : /* includes case 3 */+- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);+- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict);+- case 7 :+- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict);+- }+-}++/*++ * This struct contains the functions necessary for lazy to search.++ * Currently, that is only searchMax. However, it is still valuable to have the++ * VTable because this makes it easier to add more functions to the VTable later.++ */++typedef struct {++ searchMax_f searchMax;++} ZSTD_LazyVTable;++++#define GEN_ZSTD_BT_VTABLE(dictMode, mls, ...) \++ static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \++ ZSTD_matchState_t* ms, \++ const BYTE* ip, const BYTE* const iLimit, \++ size_t* offsetPtr) \++ { \++ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \++ return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \++ } \++ static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \++ ZSTD_BtFindBestMatch_##dictMode##_##mls \++ };+++#define GEN_ZSTD_HC_VTABLE(dictMode, mls, ...) \++ static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \++ ZSTD_matchState_t* ms, \++ const BYTE* ip, const BYTE* const iLimit, \++ size_t* offsetPtr) \++ { \++ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \++ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \++ } \++ static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \++ ZSTD_HcFindBestMatch_##dictMode##_##mls \++ };++-static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS (+- ZSTD_matchState_t* ms,+- const BYTE* ip, const BYTE* const iLimit,+- size_t* offsetPtr)+-{+- switch(ms->cParams.minMatch)+- {+- default : /* includes case 3 */+- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);+- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState);+- case 7 :+- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState);++#define ZSTD_FOR_EACH_MLS(X, dictMode) \++ X(dictMode, 4) \++ X(dictMode, 5) \++ X(dictMode, 6)++++#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \++ X(__VA_ARGS__, noDict) \++ X(__VA_ARGS__, extDict) \++ X(__VA_ARGS__, dictMatchState) \++ X(__VA_ARGS__, dedicatedDictSearch)++++/* Generate Binary Tree VTables for each combination of (dictMode, mls) */++ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE)++/* Generate Hash Chain VTables for each combination of (dictMode, mls) */++ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE)++++#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \++ { \++ &ZSTD_BtVTable_##dictMode##_4, \++ &ZSTD_BtVTable_##dictMode##_5, \++ &ZSTD_BtVTable_##dictMode##_6 \+ }+-}+-++-static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS (+- ZSTD_matchState_t* ms,+- const BYTE* ip, const BYTE* const iLimit,+- size_t* offsetPtr)+-{+- switch(ms->cParams.minMatch)+- {+- default : /* includes case 3 */+- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch);+- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch);+- case 7 :+- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch);++#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \++ { \++ &ZSTD_HcVTable_##dictMode##_4, \++ &ZSTD_HcVTable_##dictMode##_5, \++ &ZSTD_HcVTable_##dictMode##_6 \+ }+-}++-+-FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (+- ZSTD_matchState_t* ms,+- const BYTE* ip, const BYTE* const iLimit,+- size_t* offsetPtr)+-{+- switch(ms->cParams.minMatch)+- {+- default : /* includes case 3 */+- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);+- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict);+- case 7 :+- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict);++#define GEN_ZSTD_VTABLE_ARRAY(X) \++ { \++ X(noDict), \++ X(extDict), \++ X(dictMatchState), \++ X(dedicatedDictSearch) \+ }+-}+-++ /* *******************************+ * Common parser - lazy strategy+ *********************************/+ typedef enum { search_hashChain, search_binaryTree } searchMethod_e;+++static ZSTD_LazyVTable const* ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode)++{++ /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */++ ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY);++ ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY);++ /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */++++ U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch));++ switch (searchMethod) {++ case search_hashChain:++ return hcVTables[dictMode][mls - 4];++ case search_binaryTree:++ return btVTables[dictMode][mls - 4];++ default:++ return NULL;++ }++}+++ FORCE_INLINE_TEMPLATE size_t+ ZSTD_compressBlock_lazy_generic(+ ZSTD_matchState_t* ms, seqStore_t* seqStore,+@@ -870,36 +851,13 @@ ZSTD_compressBlock_lazy_generic(+ const U32 prefixLowestIndex = ms->window.dictLimit;+ const BYTE* const prefixLowest = base + prefixLowestIndex;++- typedef size_t (*searchMax_f)(+- ZSTD_matchState_t* ms,+- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);+-+ /*+ * This table is indexed first by the four ZSTD_dictMode_e values, and then+ * by the two searchMethod_e values. NULLs are placed for configurations+ * that should never occur (extDict modes go to the other implementation+ * below and there is no DDSS for binary tree search yet).+ */+- const searchMax_f searchFuncs[4][2] = {+- {+- ZSTD_HcFindBestMatch_selectMLS,+- ZSTD_BtFindBestMatch_selectMLS+- },+- {+- NULL,+- NULL+- },+- {+- ZSTD_HcFindBestMatch_dictMatchState_selectMLS,+- ZSTD_BtFindBestMatch_dictMatchState_selectMLS+- },+- {+- ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS,+- NULL+- }+- };+-+- searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree];++ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax;+ U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0;++ const int isDMS = dictMode == ZSTD_dictMatchState;+@@ -1221,10 +1179,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(+ const BYTE* const dictStart = dictBase + ms->window.lowLimit;+ const U32 windowLog = ms->cParams.windowLog;++- typedef size_t (*searchMax_f)(+- ZSTD_matchState_t* ms,+- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);+- searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS;++ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax;++ U32 offset_1 = rep[0], offset_2 = rep[1];++diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib+index 79be57fdd32a..6c44e49f0f2b 100644+--- a/scripts/Makefile.lib++++ b/scripts/Makefile.lib+@@ -384,13 +384,8 @@ dtc-tmp = $(subst $(comma),_,$(dot-target).dts.tmp)++ # Bzip2 and LZMA do not include size in file... so we have to fake that;+ # append the size as a 32-bit littleendian number as gzip does.+-size_append = printf $(shell \+-dec_size=0; \+-for F in $(real-prereqs); do \+- fsize=$$($(CONFIG_SHELL) $(srctree)/scripts/file-size.sh $$F); \+- dec_size=$$(expr $$dec_size + $$fsize); \+-done; \+-printf "%08x\n" $$dec_size | \++total_size = $(shell $(PERL) $(srctree)/scripts/file-size.pl $(real-prereqs))++size_append = printf $(shell printf "%08x\n" $(total_size) | \+ sed 's/\(..\)/\1 /g' | { \+ read ch0 ch1 ch2 ch3; \+ for ch in $$ch3 $$ch2 $$ch1 $$ch0; do \+@@ -494,15 +489,20 @@ quiet_cmd_xzmisc = XZMISC $@+ # single pass, so zstd doesn't need to allocate a window buffer. When streaming+ # decompression is used, like initramfs decompression, zstd22 should likely not+ # be used because it would require zstd to allocate a 128 MB buffer.++#++# --stream-size to reduce zstd memory usage (otherwise zstd -22 --ultra++# allocates, but does not use, 2 GB) and potentially improve compression.++#++# --no-content-size to save three bytes which we do not use (we use size_append).++-quiet_cmd_zstd = ZSTD $@+- cmd_zstd = cat $(real-prereqs) | $(ZSTD) -19 > $@++# zstd --stream-size is only supported since 1.4.4++zstd_stream_size = $(shell $(ZSTD) -1c --stream-size=0 --no-content-size </dev/null >/dev/null 2>&1 && printf '%s' '--stream-size=$(total_size) --no-content-size')++-quiet_cmd_zstd22 = ZSTD22 $@+- cmd_zstd22 = cat $(real-prereqs) | $(ZSTD) -22 --ultra > $@++quiet_cmd_zstd = ZSTD $@++ cmd_zstd = { cat $(real-prereqs) | $(ZSTD) $(zstd_stream_size) -$(CONFIG_KERNEL_ZSTD_LEVEL) -T0; $(size_append); } > $@++ quiet_cmd_zstd22_with_size = ZSTD22 $@+- cmd_zstd22_with_size = { cat $(real-prereqs) | $(ZSTD) -22 --ultra; $(size_append); } > $@++ cmd_zstd22 = { cat $(real-prereqs) | $(ZSTD) $(zstd_stream_size) -22 --ultra -T0; $(size_append); } > $@++ # ASM offsets+ # ---------------------------------------------------------------------------+diff --git a/scripts/Makefile.modinst b/scripts/Makefile.modinst+index c2c43a0ecfe0..c15e5395fd32 100644+--- a/scripts/Makefile.modinst++++ b/scripts/Makefile.modinst+@@ -96,8 +96,13 @@ quiet_cmd_gzip = GZIP $@+ cmd_gzip = $(KGZIP) -n -f $<+ quiet_cmd_xz = XZ $@+ cmd_xz = $(XZ) --lzma2=dict=2MiB -f $<++ifdef CONFIG_MODULE_COMPRESS_ZSTD_ULTRA+ quiet_cmd_zstd = ZSTD $@+- cmd_zstd = $(ZSTD) -T0 --rm -f -q $<++ cmd_zstd = $(ZSTD) -$(CONFIG_MODULE_COMPRESS_ZSTD_LEVEL_ULTRA) --ultra --zstd=wlog=21 -T0 --rm -f -q $<++else++quiet_cmd_zstd = ZSTD $@++ cmd_zstd = $(ZSTD) -$(CONFIG_MODULE_COMPRESS_ZSTD_LEVEL) --zstd=wlog=21 -T0 --rm -f -q $<++endif++ $(dst)/%.ko.gz: $(dst)/%.ko FORCE+ $(call cmd,gzip)+diff --git a/scripts/file-size.pl b/scripts/file-size.pl+new file mode 100755+index 000000000000..170bb6d048fa+--- /dev/null++++ b/scripts/file-size.pl+@@ -0,0 +1,8 @@++#!/usr/bin/perl -w++# SPDX-License-Identifier: GPL-2.0++my $total = 0;++foreach (@ARGV) {++ @stat = stat $_ or die "$_: $!";++ $total += $stat[7];++}++print "$total\n";+diff --git a/scripts/file-size.sh b/scripts/file-size.sh+deleted file mode 100755+index 7eb7423416b5..000000000000+--- a/scripts/file-size.sh++++ /dev/null+@@ -1,4 +0,0 @@+-#!/bin/sh+-# SPDX-License-Identifier: GPL-2.0+-set -- $(ls -dn "$1")+-printf '%s\n' "$5"+diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh+index 666f7bbc13eb..ba819b9376ad 100755+--- a/scripts/link-vmlinux.sh++++ b/scripts/link-vmlinux.sh+@@ -388,8 +388,8 @@ if is_enabled CONFIG_KALLSYMS; then+ kallsyms_step 2++ # step 3+- size1=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" ${kallsymso_prev})+- size2=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" ${kallsymso})++ size1=$(${PERL} "${srctree}/scripts/file-size.pl" ${kallsymso_prev})++ size2=$(${PERL} "${srctree}/scripts/file-size.pl" ${kallsymso})++ if [ $size1 -ne $size2 ] || [ -n "${KALLSYMS_EXTRA_PASS}" ]; then+ kallsyms_step 3+--+2.35.1+