Skip to content

Archlinux Kernel based on different schedulers and some other performance improvements.

5.17-rc3-1-patch

P. Junge4ce9f5ad9fd
diff --git a/patches/linux-cachyos-rc-5.17.rc3.patch b/patches/linux-cachyos-rc-5.17.rc3.patch
new file mode 100644
index 00000000..8c6be46e
--- /dev/null
+++ b/patches/linux-cachyos-rc-5.17.rc3.patch
@@ -0,0 +1,11842 @@
+From 1acd39e254766a13905a4758a1ff1da74b289e9d Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sun, 23 Jan 2022 16:49:01 +0000
+Subject: [PATCH] anbox
+
+---
+ drivers/android/Kconfig | 2 +-
+ drivers/android/Makefile | 7 ++++---
+ drivers/android/binder_alloc.c | 2 +-
+ drivers/staging/android/Kconfig | 2 +-
+ drivers/staging/android/Makefile | 3 ++-
+ drivers/staging/android/ashmem.c | 3 +++
+ fs/file.c | 1 +
+ kernel/fork.c | 1 +
+ kernel/sched/core.c | 1 +
+ kernel/task_work.c | 1 +
+ mm/memory.c | 1 +
+ mm/shmem.c | 1 +
+ security/security.c | 4 ++++
+ 13 files changed, 22 insertions(+), 7 deletions(-)
+
+diff --git a/drivers/android/Kconfig b/drivers/android/Kconfig
+index 53b22e26266c..f3c50236e8d1 100644
+--- a/drivers/android/Kconfig
++++ b/drivers/android/Kconfig
+@@ -9,7 +9,7 @@ config ANDROID
+ if ANDROID
+
+ config ANDROID_BINDER_IPC
+- bool "Android Binder IPC Driver"
++ tristate "Android Binder IPC Driver"
+ depends on MMU
+ default n
+ help
+diff --git a/drivers/android/Makefile b/drivers/android/Makefile
+index c9d3d0c99c25..55411d9a9c2a 100644
+--- a/drivers/android/Makefile
++++ b/drivers/android/Makefile
+@@ -1,6 +1,7 @@
+ # SPDX-License-Identifier: GPL-2.0-only
+ ccflags-y += -I$(src) # needed for trace events
+
+-obj-$(CONFIG_ANDROID_BINDERFS) += binderfs.o
+-obj-$(CONFIG_ANDROID_BINDER_IPC) += binder.o binder_alloc.o
+-obj-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o
++obj-$(CONFIG_ANDROID_BINDER_IPC) += binder_linux.o
++binder_linux-y := binder.o binder_alloc.o
++binder_linux-$(CONFIG_ANDROID_BINDERFS) += binderfs.o
++binder_linux-$(CONFIG_ANDROID_BINDER_IPC_SELFTEST) += binder_alloc_selftest.o
+diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c
+index 47bc74a8c7b6..49ff7504646e 100644
+--- a/drivers/android/binder_alloc.c
++++ b/drivers/android/binder_alloc.c
+@@ -38,7 +38,7 @@ enum {
+ };
+ static uint32_t binder_alloc_debug_mask = BINDER_DEBUG_USER_ERROR;
+
+-module_param_named(debug_mask, binder_alloc_debug_mask,
++module_param_named(alloc_debug_mask, binder_alloc_debug_mask,
+ uint, 0644);
+
+ #define binder_alloc_debug(mask, x...) \
+diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig
+index 70498adb1575..5c35653ed36d 100644
+--- a/drivers/staging/android/Kconfig
++++ b/drivers/staging/android/Kconfig
+@@ -4,7 +4,7 @@ menu "Android"
+ if ANDROID
+
+ config ASHMEM
+- bool "Enable the Anonymous Shared Memory Subsystem"
++ tristate "Enable the Anonymous Shared Memory Subsystem"
+ depends on SHMEM
+ help
+ The ashmem subsystem is a new shared memory allocator, similar to
+diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile
+index e9a55a5e6529..60cb8eacc793 100644
+--- a/drivers/staging/android/Makefile
++++ b/drivers/staging/android/Makefile
+@@ -1,4 +1,5 @@
+ # SPDX-License-Identifier: GPL-2.0
+ ccflags-y += -I$(src) # needed for trace events
+
+-obj-$(CONFIG_ASHMEM) += ashmem.o
++obj-$(CONFIG_ASHMEM) += ashmem_linux.o
++ashmem_linux-y += ashmem.o
+diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c
+index ddbde3f8430e..2a92b0a7482f 100644
+--- a/drivers/staging/android/ashmem.c
++++ b/drivers/staging/android/ashmem.c
+@@ -24,6 +24,7 @@
+ #include <linux/bitops.h>
+ #include <linux/mutex.h>
+ #include <linux/shmem_fs.h>
++#include <linux/module.h>
+ #include "ashmem.h"
+
+ #define ASHMEM_NAME_PREFIX "dev/ashmem/"
+@@ -968,3 +969,5 @@ static int __init ashmem_init(void)
+ return ret;
+ }
+ device_initcall(ashmem_init);
++
++MODULE_LICENSE("GPL v2");
+diff --git a/fs/file.c b/fs/file.c
+index 97d212a9b814..b8f81bfa874e 100644
+--- a/fs/file.c
++++ b/fs/file.c
+@@ -787,6 +787,7 @@ int __close_fd_get_file(unsigned int fd, struct file **res)
+ *res = NULL;
+ return -ENOENT;
+ }
++EXPORT_SYMBOL(close_fd_get_file);
+
+ /*
+ * variant of close_fd that gets a ref on the file for later fput.
+diff --git a/kernel/fork.c b/kernel/fork.c
+index d75a528f7b21..fe337e1b7c4b 100644
+--- a/kernel/fork.c
++++ b/kernel/fork.c
+@@ -1152,6 +1152,7 @@ void mmput_async(struct mm_struct *mm)
+ schedule_work(&mm->async_put_work);
+ }
+ }
++EXPORT_SYMBOL_GPL(mmput_async);
+ #endif
+
+ /**
+diff --git a/kernel/sched/core.c b/kernel/sched/core.c
+index 2e4ae00e52d1..d0fe040e8b97 100644
+--- a/kernel/sched/core.c
++++ b/kernel/sched/core.c
+@@ -6955,6 +6955,7 @@ int can_nice(const struct task_struct *p, const int nice)
+ return (nice_rlim <= task_rlimit(p, RLIMIT_NICE) ||
+ capable(CAP_SYS_NICE));
+ }
++EXPORT_SYMBOL_GPL(can_nice);
+
+ #ifdef __ARCH_WANT_SYS_NICE
+
+diff --git a/kernel/task_work.c b/kernel/task_work.c
+index 1698fbe6f0e1..90000404af2b 100644
+--- a/kernel/task_work.c
++++ b/kernel/task_work.c
+@@ -60,6 +60,7 @@ int task_work_add(struct task_struct *task, struct callback_head *work,
+
+ return 0;
+ }
++EXPORT_SYMBOL(task_work_add);
+
+ /**
+ * task_work_cancel_match - cancel a pending work added by task_work_add()
+diff --git a/mm/memory.c b/mm/memory.c
+index c125c4969913..fe59e377b671 100644
+--- a/mm/memory.c
++++ b/mm/memory.c
+@@ -1664,6 +1664,7 @@ void zap_page_range(struct vm_area_struct *vma, unsigned long start,
+ mmu_notifier_invalidate_range_end(&range);
+ tlb_finish_mmu(&tlb);
+ }
++EXPORT_SYMBOL_GPL(zap_page_range);
+
+ /**
+ * zap_page_range_single - remove user pages in a given range
+diff --git a/mm/shmem.c b/mm/shmem.c
+index a09b29ec2b45..98f73f109f19 100644
+--- a/mm/shmem.c
++++ b/mm/shmem.c
+@@ -4140,6 +4140,7 @@ int shmem_zero_setup(struct vm_area_struct *vma)
+
+ return 0;
+ }
++EXPORT_SYMBOL_GPL(shmem_zero_setup);
+
+ /**
+ * shmem_read_mapping_page_gfp - read into page cache, using specified page allocation flags.
+diff --git a/security/security.c b/security/security.c
+index 3d4eb474f35b..429ecc13a8aa 100644
+--- a/security/security.c
++++ b/security/security.c
+@@ -751,24 +751,28 @@ int security_binder_set_context_mgr(const struct cred *mgr)
+ {
+ return call_int_hook(binder_set_context_mgr, 0, mgr);
+ }
++EXPORT_SYMBOL_GPL(security_binder_set_context_mgr);
+
+ int security_binder_transaction(const struct cred *from,
+ const struct cred *to)
+ {
+ return call_int_hook(binder_transaction, 0, from, to);
+ }
++EXPORT_SYMBOL_GPL(security_binder_transaction);
+
+ int security_binder_transfer_binder(const struct cred *from,
+ const struct cred *to)
+ {
+ return call_int_hook(binder_transfer_binder, 0, from, to);
+ }
++EXPORT_SYMBOL_GPL(security_binder_transfer_binder);
+
+ int security_binder_transfer_file(const struct cred *from,
+ const struct cred *to, struct file *file)
+ {
+ return call_int_hook(binder_transfer_file, 0, from, to, file);
+ }
++EXPORT_SYMBOL_GPL(security_binder_transfer_file);
+
+ int security_ptrace_access_check(struct task_struct *child, unsigned int mode)
+ {
+--
+2.34.1
+
+From 2d64fc35bf31bba408508cec7e3ccd45c20c7bec Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sat, 29 Jan 2022 07:47:17 +0000
+Subject: [PATCH] asus-ec
+
+---
+ Documentation/hwmon/asus_ec_sensors.rst | 52 ++
+ Documentation/hwmon/asus_wmi_ec_sensors.rst | 38 --
+ MAINTAINERS | 6 +
+ drivers/hwmon/Kconfig | 16 +-
+ drivers/hwmon/Makefile | 1 +
+ drivers/hwmon/asus-ec-sensors.c | 694 ++++++++++++++++++++
+ 6 files changed, 768 insertions(+), 39 deletions(-)
+ create mode 100644 Documentation/hwmon/asus_ec_sensors.rst
+ delete mode 100644 Documentation/hwmon/asus_wmi_ec_sensors.rst
+ create mode 100644 drivers/hwmon/asus-ec-sensors.c
+
+diff --git a/Documentation/hwmon/asus_ec_sensors.rst b/Documentation/hwmon/asus_ec_sensors.rst
+new file mode 100644
+index 000000000000..b12ac7ebeb1a
+--- /dev/null
++++ b/Documentation/hwmon/asus_ec_sensors.rst
+@@ -0,0 +1,52 @@
++.. SPDX-License-Identifier: GPL-2.0-or-later
++
++Kernel driver asus_ec_sensors
++=================================
++
++Supported boards:
++ * PRIME X570-PRO,
++ * Pro WS X570-ACE,
++ * ROG CROSSHAIR VIII DARK HERO,
++ * ROG CROSSHAIR VIII FORMULA,
++ * ROG CROSSHAIR VIII HERO,
++ * ROG CROSSHAIR VIII IMPACT,
++ * ROG STRIX B550-E GAMING,
++ * ROG STRIX B550-I GAMING,
++ * ROG STRIX X570-E GAMING,
++ * ROG STRIX X570-F GAMING,
++ * ROG STRIX X570-I GAMING
++
++Authors:
++ - Eugene Shalygin <[email protected]>
++
++Description:
++------------
++ASUS mainboards publish hardware monitoring information via Super I/O
++chip and the ACPI embedded controller (EC) registers. Some of the sensors
++are only available via the EC.
++
++The driver is aware of and reads the following sensors:
++
++1. Chipset (PCH) temperature
++2. CPU package temperature
++3. Motherboard temperature
++4. Readings from the T_Sensor header
++5. VRM temperature
++6. CPU_Opt fan RPM
++7. VRM heatsink fan RPM
++8. Chipset fan RPM
++9. Readings from the "Water flow meter" header (RPM)
++10. Readings from the "Water In" and "Water Out" temperature headers
++11. CPU current
++
++Sensor values are read from EC registers, and to avoid race with the board
++firmware the driver acquires ACPI mutex, the one used by the WMI when its
++methods access the EC.
++
++Module Parameters
++-----------------
++ * mutex_path: string
++ The driver holds path to the ACPI mutex for each board (actually,
++ the path is mostly identical for them). If ASUS changes this path
++ in a future BIOS update, this parameter can be used to override
++ the stored in the driver value until it gets updated.
+diff --git a/Documentation/hwmon/asus_wmi_ec_sensors.rst b/Documentation/hwmon/asus_wmi_ec_sensors.rst
+deleted file mode 100644
+index 1b287f229e86..000000000000
+--- a/Documentation/hwmon/asus_wmi_ec_sensors.rst
++++ /dev/null
+@@ -1,38 +0,0 @@
+-.. SPDX-License-Identifier: GPL-2.0-or-later
+-
+-Kernel driver asus_wmi_ec_sensors
+-=================================
+-
+-Supported boards:
+- * PRIME X570-PRO,
+- * Pro WS X570-ACE,
+- * ROG CROSSHAIR VIII DARK HERO,
+- * ROG CROSSHAIR VIII FORMULA,
+- * ROG CROSSHAIR VIII HERO,
+- * ROG STRIX B550-E GAMING,
+- * ROG STRIX B550-I GAMING,
+- * ROG STRIX X570-E GAMING.
+-
+-Authors:
+- - Eugene Shalygin <[email protected]>
+-
+-Description:
+-------------
+-ASUS mainboards publish hardware monitoring information via Super I/O
+-chip and the ACPI embedded controller (EC) registers. Some of the sensors
+-are only available via the EC.
+-
+-ASUS WMI interface provides a method (BREC) to read data from EC registers,
+-which is utilized by this driver to publish those sensor readings to the
+-HWMON system. The driver is aware of and reads the following sensors:
+-
+-1. Chipset (PCH) temperature
+-2. CPU package temperature
+-3. Motherboard temperature
+-4. Readings from the T_Sensor header
+-5. VRM temperature
+-6. CPU_Opt fan RPM
+-7. Chipset fan RPM
+-8. Readings from the "Water flow meter" header (RPM)
+-9. Readings from the "Water In" and "Water Out" temperature headers
+-10. CPU current
+diff --git a/MAINTAINERS b/MAINTAINERS
+index ea3e6c914384..aff67314e63e 100644
+--- a/MAINTAINERS
++++ b/MAINTAINERS
+@@ -3049,6 +3049,12 @@ L: [email protected]
+ S: Maintained
+ F: drivers/hwmon/asus_wmi_ec_sensors.c
+
++ASUS EC HARDWARE MONITOR DRIVER
++M: Eugene Shalygin <[email protected]>
++S: Maintained
++F: drivers/hwmon/asus-ec-sensors.c
++
+ ASUS WIRELESS RADIO CONTROL DRIVER
+ M: João Paulo Rechi Vita <[email protected]>
+diff --git a/drivers/hwmon/Kconfig b/drivers/hwmon/Kconfig
+index 8df25f1079ba..2acfcbd114c2 100644
+--- a/drivers/hwmon/Kconfig
++++ b/drivers/hwmon/Kconfig
+@@ -2252,16 +2252,30 @@ config SENSORS_ASUS_WMI
+
+ config SENSORS_ASUS_WMI_EC
+ tristate "ASUS WMI B550/X570"
+- depends on ACPI_WMI
++ depends on ACPI_WMI && SENSORS_ASUS_EC=n
+ help
+ If you say yes here you get support for the ACPI embedded controller
+ hardware monitoring interface found in B550/X570 ASUS motherboards.
+ This driver will provide readings of fans, voltages and temperatures
+ through the system firmware.
+
++ This driver is deprecated in favor of the ASUS EC Sensors driver
++ which provides fully compatible output.
++
+ This driver can also be built as a module. If so, the module
+ will be called asus_wmi_sensors_ec.
+
++config SENSORS_ASUS_EC
++ tristate "ASUS EC Sensors"
++ help
++ If you say yes here you get support for the ACPI embedded controller
++ hardware monitoring interface found in ASUS motherboards. The driver
++ currently supports B550/X570 boards, although other ASUS boards might
++ provide this monitoring interface as well.
++
++ This driver can also be built as a module. If so, the module
++ will be called asus_ec_sensors.
++
+ endif # ACPI
+
+ endif # HWMON
+diff --git a/drivers/hwmon/Makefile b/drivers/hwmon/Makefile
+index 185f946d698b..7bba3415ca3d 100644
+--- a/drivers/hwmon/Makefile
++++ b/drivers/hwmon/Makefile
+@@ -9,6 +9,7 @@ obj-$(CONFIG_HWMON_VID) += hwmon-vid.o
+ # APCI drivers
+ obj-$(CONFIG_SENSORS_ACPI_POWER) += acpi_power_meter.o
+ obj-$(CONFIG_SENSORS_ATK0110) += asus_atk0110.o
++obj-$(CONFIG_SENSORS_ASUS_EC) += asus-ec-sensors.o
+ obj-$(CONFIG_SENSORS_ASUS_WMI) += asus_wmi_sensors.o
+ obj-$(CONFIG_SENSORS_ASUS_WMI_EC) += asus_wmi_ec_sensors.o
+
+diff --git a/drivers/hwmon/asus-ec-sensors.c b/drivers/hwmon/asus-ec-sensors.c
+new file mode 100644
+index 000000000000..906597d89842
+--- /dev/null
++++ b/drivers/hwmon/asus-ec-sensors.c
+@@ -0,0 +1,694 @@
++// SPDX-License-Identifier: GPL-2.0+
++/*
++ * HWMON driver for ASUS motherboards that publish some sensor values
++ * via the embedded controller registers.
++ *
++ * Copyright (C) 2021 Eugene Shalygin <[email protected]>
++
++ * EC provides:
++ * - Chipset temperature
++ * - CPU temperature
++ * - Motherboard temperature
++ * - T_Sensor temperature
++ * - VRM temperature
++ * - Water In temperature
++ * - Water Out temperature
++ * - CPU Optional fan RPM
++ * - Chipset fan RPM
++ * - VRM Heat Sink fan RPM
++ * - Water Flow fan RPM
++ * - CPU current
++ */
++
++#include <linux/acpi.h>
++#include <linux/bitops.h>
++#include <linux/dev_printk.h>
++#include <linux/dmi.h>
++#include <linux/hwmon.h>
++#include <linux/init.h>
++#include <linux/jiffies.h>
++#include <linux/kernel.h>
++#include <linux/module.h>
++#include <linux/platform_device.h>
++#include <linux/sort.h>
++#include <linux/units.h>
++
++#include <asm/unaligned.h>
++
++static char *mutex_path_override;
++
++/* Writing to this EC register switches EC bank */
++#define ASUS_EC_BANK_REGISTER 0xff
++#define SENSOR_LABEL_LEN 16
++
++/*
++ * Arbitrary set max. allowed bank number. Required for sorting banks and
++ * currently is overkill with just 2 banks used at max, but for the sake
++ * of alignment let's set it to a higher value.
++ */
++#define ASUS_EC_MAX_BANK 3
++
++#define ACPI_LOCK_DELAY_MS 500
++
++/* ACPI mutex for locking access to the EC for the firmware */
++#define ASUS_HW_ACCESS_MUTEX_ASMX "\\AMW0.ASMX"
++
++/* There are two variants of the vendor spelling */
++#define VENDOR_ASUS_UPPER_CASE "ASUSTeK COMPUTER INC."
++
++typedef union {
++ u32 value;
++ struct {
++ u8 index;
++ u8 bank;
++ u8 size;
++ u8 dummy;
++ } components;
++} sensor_address;
++
++#define MAKE_SENSOR_ADDRESS(size, bank, index) { \
++ .value = (size << 16) + (bank << 8) + index \
++ }
++
++static u32 hwmon_attributes[hwmon_max] = {
++ [hwmon_chip] = HWMON_C_REGISTER_TZ,
++ [hwmon_temp] = HWMON_T_INPUT | HWMON_T_LABEL,
++ [hwmon_in] = HWMON_I_INPUT | HWMON_I_LABEL,
++ [hwmon_curr] = HWMON_C_INPUT | HWMON_C_LABEL,
++ [hwmon_fan] = HWMON_F_INPUT | HWMON_F_LABEL,
++};
++
++struct ec_sensor_info {
++ char label[SENSOR_LABEL_LEN];
++ enum hwmon_sensor_types type;
++ sensor_address addr;
++};
++
++#define EC_SENSOR(sensor_label, sensor_type, size, bank, index) { \
++ .label = sensor_label, .type = sensor_type, \
++ .addr = MAKE_SENSOR_ADDRESS(size, bank, index), \
++ }
++
++enum ec_sensors {
++ /* chipset temperature [℃] */
++ ec_sensor_temp_chipset,
++ /* CPU temperature [℃] */
++ ec_sensor_temp_cpu,
++ /* motherboard temperature [℃] */
++ ec_sensor_temp_mb,
++ /* "T_Sensor" temperature sensor reading [℃] */
++ ec_sensor_temp_t_sensor,
++ /* VRM temperature [℃] */
++ ec_sensor_temp_vrm,
++ /* CPU_Opt fan [RPM] */
++ ec_sensor_fan_cpu_opt,
++ /* VRM heat sink fan [RPM] */
++ ec_sensor_fan_vrm_hs,
++ /* Chipset fan [RPM] */
++ ec_sensor_fan_chipset,
++ /* Water flow sensor reading [RPM] */
++ ec_sensor_fan_water_flow,
++ /* CPU current [A] */
++ ec_sensor_curr_cpu,
++ /* "Water_In" temperature sensor reading [℃] */
++ ec_sensor_temp_water_in,
++ /* "Water_Out" temperature sensor reading [℃] */
++ ec_sensor_temp_water_out,
++};
++
++#define SENSOR_TEMP_CHIPSET BIT(ec_sensor_temp_chipset)
++#define SENSOR_TEMP_CPU BIT(ec_sensor_temp_cpu)
++#define SENSOR_TEMP_MB BIT(ec_sensor_temp_mb)
++#define SENSOR_TEMP_T_SENSOR BIT(ec_sensor_temp_t_sensor)
++#define SENSOR_TEMP_VRM BIT(ec_sensor_temp_vrm)
++#define SENSOR_FAN_CPU_OPT BIT(ec_sensor_fan_cpu_opt)
++#define SENSOR_FAN_VRM_HS BIT(ec_sensor_fan_vrm_hs)
++#define SENSOR_FAN_CHIPSET BIT(ec_sensor_fan_chipset)
++#define SENSOR_FAN_WATER_FLOW BIT(ec_sensor_fan_water_flow)
++#define SENSOR_CURR_CPU BIT(ec_sensor_curr_cpu)
++#define SENSOR_TEMP_WATER_IN BIT(ec_sensor_temp_water_in)
++#define SENSOR_TEMP_WATER_OUT BIT(ec_sensor_temp_water_out)
++
++/* All the known sensors for ASUS EC controllers */
++static const struct ec_sensor_info known_ec_sensors[] = {
++ [ec_sensor_temp_chipset] =
++ EC_SENSOR("Chipset", hwmon_temp, 1, 0x00, 0x3a),
++ [ec_sensor_temp_cpu] = EC_SENSOR("CPU", hwmon_temp, 1, 0x00, 0x3b),
++ [ec_sensor_temp_mb] =
++ EC_SENSOR("Motherboard", hwmon_temp, 1, 0x00, 0x3c),
++ [ec_sensor_temp_t_sensor] =
++ EC_SENSOR("T_Sensor", hwmon_temp, 1, 0x00, 0x3d),
++ [ec_sensor_temp_vrm] = EC_SENSOR("VRM", hwmon_temp, 1, 0x00, 0x3e),
++ [ec_sensor_fan_cpu_opt] =
++ EC_SENSOR("CPU_Opt", hwmon_fan, 2, 0x00, 0xb0),
++ [ec_sensor_fan_vrm_hs] = EC_SENSOR("VRM HS", hwmon_fan, 2, 0x00, 0xb2),
++ [ec_sensor_fan_chipset] =
++ EC_SENSOR("Chipset", hwmon_fan, 2, 0x00, 0xb4),
++ [ec_sensor_fan_water_flow] =
++ EC_SENSOR("Water_Flow", hwmon_fan, 2, 0x00, 0xbc),
++ [ec_sensor_curr_cpu] = EC_SENSOR("CPU", hwmon_curr, 1, 0x00, 0xf4),
++ [ec_sensor_temp_water_in] =
++ EC_SENSOR("Water_In", hwmon_temp, 1, 0x01, 0x00),
++ [ec_sensor_temp_water_out] =
++ EC_SENSOR("Water_Out", hwmon_temp, 1, 0x01, 0x01),
++};
++
++/* Shortcuts for common combinations */
++#define SENSOR_SET_TEMP_CHIPSET_CPU_MB \
++ (SENSOR_TEMP_CHIPSET | SENSOR_TEMP_CPU | SENSOR_TEMP_MB)
++#define SENSOR_SET_TEMP_WATER (SENSOR_TEMP_WATER_IN | SENSOR_TEMP_WATER_OUT)
++
++#define DMI_EXACT_MATCH_BOARD(vendor, name, sensors) { \
++ .matches = { \
++ DMI_EXACT_MATCH(DMI_BOARD_VENDOR, vendor), \
++ DMI_EXACT_MATCH(DMI_BOARD_NAME, name), \
++ }, \
++ .driver_data = (void *)(sensors), \
++}
++
++static const struct dmi_system_id asus_ec_dmi_table[] __initconst = {
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "PRIME X570-PRO",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM |
++ SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "Pro WS X570-ACE",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_VRM |
++ SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE,
++ "ROG CROSSHAIR VIII DARK HERO",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR |
++ SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER |
++ SENSOR_FAN_CPU_OPT | SENSOR_FAN_WATER_FLOW | SENSOR_CURR_CPU),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE,
++ "ROG CROSSHAIR VIII FORMULA",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR |
++ SENSOR_TEMP_VRM | SENSOR_FAN_CPU_OPT | SENSOR_FAN_CHIPSET |
++ SENSOR_CURR_CPU),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG CROSSHAIR VIII HERO",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR |
++ SENSOR_TEMP_VRM | SENSOR_SET_TEMP_WATER |
++ SENSOR_FAN_CPU_OPT | SENSOR_FAN_CHIPSET |
++ SENSOR_FAN_WATER_FLOW | SENSOR_CURR_CPU),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE,
++ "ROG CROSSHAIR VIII IMPACT",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB | SENSOR_TEMP_T_SENSOR |
++ SENSOR_TEMP_VRM | SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX B550-E GAMING",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB |
++ SENSOR_TEMP_T_SENSOR |
++ SENSOR_TEMP_VRM | SENSOR_FAN_CPU_OPT),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX B550-I GAMING",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB |
++ SENSOR_TEMP_T_SENSOR |
++ SENSOR_TEMP_VRM | SENSOR_FAN_VRM_HS | SENSOR_CURR_CPU),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX X570-E GAMING",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB |
++ SENSOR_TEMP_T_SENSOR |
++ SENSOR_TEMP_VRM | SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX X570-F GAMING",
++ SENSOR_SET_TEMP_CHIPSET_CPU_MB |
++ SENSOR_TEMP_T_SENSOR | SENSOR_FAN_CHIPSET
++ ),
++ DMI_EXACT_MATCH_BOARD(VENDOR_ASUS_UPPER_CASE, "ROG STRIX X570-I GAMING",
++ SENSOR_TEMP_T_SENSOR | SENSOR_FAN_VRM_HS |
++ SENSOR_FAN_CHIPSET | SENSOR_CURR_CPU),
++ {}
++};
++
++struct ec_sensor {
++ unsigned int info_index;
++ u32 cached_value;
++};
++
++struct ec_sensors_data {
++ unsigned long board_sensors;
++ struct ec_sensor *sensors;
++ /** EC registers to read from */
++ u16 *registers;
++ u8 *read_buffer;
++ /** sorted list of unique register banks */
++ u8 banks[ASUS_EC_MAX_BANK + 1];
++ /** in jiffies */
++ unsigned long last_updated;
++ acpi_handle aml_mutex;
++ /** number of board EC sensors */
++ u8 nr_sensors;
++ /** number of EC registers to read (sensor might span more than 1 register) */
++ u8 nr_registers;
++ /** number of unique register banks */
++ u8 nr_banks;
++};
++
++static u8 register_bank(u16 reg)
++{
++ return reg >> 8;
++}
++
++static u8 register_index(u16 reg)
++{
++ return reg & 0x00ff;
++}
++
++static const struct ec_sensor_info *
++get_sensor_info(const struct ec_sensors_data *state, int index)
++{
++ return &known_ec_sensors[state->sensors[index].info_index];
++}
++
++static int find_ec_sensor_index(const struct ec_sensors_data *ec,
++ enum hwmon_sensor_types type, int channel)
++{
++ unsigned int i;
++
++ for (i = 0; i < ec->nr_sensors; i++) {
++ if (get_sensor_info(ec, i)->type == type) {
++ if (channel == 0)
++ return i;
++ channel--;
++ }
++ }
++ return -ENOENT;
++}
++
++static int __init bank_compare(const void *a, const void *b)
++{
++ return *((const s8 *)a) - *((const s8 *)b);
++}
++
++static int __init board_sensors_count(unsigned long sensors)
++{
++ return hweight_long(sensors);
++}
++
++static void __init setup_sensor_data(struct ec_sensors_data *ec)
++{
++ struct ec_sensor *s = ec->sensors;
++ bool bank_found;
++ int i, j;
++ u8 bank;
++
++ ec->nr_banks = 0;
++ ec->nr_registers = 0;
++
++ for_each_set_bit(i, &ec->board_sensors,
++ BITS_PER_TYPE(ec->board_sensors)) {
++ s->info_index = i;
++ s->cached_value = 0;
++ ec->nr_registers +=
++ known_ec_sensors[s->info_index].addr.components.size;
++ bank_found = false;
++ bank = known_ec_sensors[s->info_index].addr.components.bank;
++ for (j = 0; j < ec->nr_banks; j++) {
++ if (ec->banks[j] == bank) {
++ bank_found = true;
++ break;
++ }
++ }
++ if (!bank_found) {
++ ec->banks[ec->nr_banks++] = bank;
++ }
++ s++;
++ }
++ sort(ec->banks, ec->nr_banks, 1, bank_compare, NULL);
++}
++
++static void __init fill_ec_registers(struct ec_sensors_data *ec)
++{
++ const struct ec_sensor_info *si;
++ unsigned int i, j, register_idx = 0;
++
++ for (i = 0; i < ec->nr_sensors; ++i) {
++ si = get_sensor_info(ec, i);
++ for (j = 0; j < si->addr.components.size; ++j, ++register_idx) {
++ ec->registers[register_idx] =
++ (si->addr.components.bank << 8) +
++ si->addr.components.index + j;
++ }
++ }
++}
++
++static acpi_handle __init asus_hw_access_mutex(struct device *dev)
++{
++ const char *mutex_path;
++ acpi_handle res;
++ int status;
++
++ mutex_path = mutex_path_override ?
++ mutex_path_override : ASUS_HW_ACCESS_MUTEX_ASMX;
++
++ status = acpi_get_handle(NULL, (acpi_string)mutex_path, &res);
++ if (ACPI_FAILURE(status)) {
++ dev_err(dev,
++ "Could not get hardware access guard mutex '%s': error %d",
++ mutex_path, status);
++ return NULL;
++ }
++ return res;
++}
++
++static int asus_ec_bank_switch(u8 bank, u8 *old)
++{
++ int status = 0;
++
++ if (old) {
++ status = ec_read(ASUS_EC_BANK_REGISTER, old);
++ }
++ if (status || (old && (*old == bank)))
++ return status;
++ return ec_write(ASUS_EC_BANK_REGISTER, bank);
++}
++
++static int asus_ec_block_read(const struct device *dev,
++ struct ec_sensors_data *ec)
++{
++ int ireg, ibank, status;
++ u8 bank, reg_bank, prev_bank;
++
++ bank = 0;
++ status = asus_ec_bank_switch(bank, &prev_bank);
++ if (status) {
++ dev_warn(dev, "EC bank switch failed");
++ return status;
++ }
++
++ if (prev_bank) {
++ /* oops... somebody else is working with the EC too */
++ dev_warn(dev,
++ "Concurrent access to the ACPI EC detected.\nRace condition possible.");
++ }
++
++ /*
++ * read registers minimizing bank switches.
++ */
++ for (ibank = 0; ibank < ec->nr_banks; ibank++) {
++ if (bank != ec->banks[ibank]) {
++ bank = ec->banks[ibank];
++ if (asus_ec_bank_switch(bank, NULL)) {
++ dev_warn(dev, "EC bank switch to %d failed",
++ bank);
++ break;
++ }
++ }
++ for (ireg = 0; ireg < ec->nr_registers; ireg++) {
++ reg_bank = register_bank(ec->registers[ireg]);
++ if (reg_bank < bank) {
++ continue;
++ }
++ ec_read(register_index(ec->registers[ireg]),
++ ec->read_buffer + ireg);
++ }
++ }
++
++ status = asus_ec_bank_switch(prev_bank, NULL);
++ return status;
++}
++
++static inline u32 get_sensor_value(const struct ec_sensor_info *si, u8 *data)
++{
++ switch (si->addr.components.size) {
++ case 1:
++ return *data;
++ case 2:
++ return get_unaligned_be16(data);
++ case 4:
++ return get_unaligned_be32(data);
++ default:
++ return 0;
++ }
++}
++
++static void update_sensor_values(struct ec_sensors_data *ec, u8 *data)
++{
++ const struct ec_sensor_info *si;
++ struct ec_sensor *s;
++
++ for (s = ec->sensors; s != ec->sensors + ec->nr_sensors; s++) {
++ si = &known_ec_sensors[s->info_index];
++ s->cached_value = get_sensor_value(si, data);
++ data += si->addr.components.size;
++ }
++}
++
++static int update_ec_sensors(const struct device *dev,
++ struct ec_sensors_data *ec)
++{
++ int status;
++
++ /*
++ * ASUS DSDT does not specify that access to the EC has to be guarded,
++ * but firmware does access it via ACPI
++ */
++ if (ACPI_FAILURE(acpi_acquire_mutex(ec->aml_mutex, NULL,
++ ACPI_LOCK_DELAY_MS))) {
++ dev_err(dev, "Failed to acquire AML mutex");
++ status = -EBUSY;
++ goto cleanup;
++ }
++
++ status = asus_ec_block_read(dev, ec);
++
++ if (!status) {
++ update_sensor_values(ec, ec->read_buffer);
++ }
++ if (ACPI_FAILURE(acpi_release_mutex(ec->aml_mutex, NULL))) {
++ dev_err(dev, "Failed to release AML mutex");
++ }
++cleanup:
++ return status;
++}
++
++static int scale_sensor_value(u32 value, int data_type)
++{
++ switch (data_type) {
++ case hwmon_curr:
++ case hwmon_temp:
++ case hwmon_in:
++ return value * MILLI;
++ default:
++ return value;
++ }
++}
++
++static int get_cached_value_or_update(const struct device *dev,
++ int sensor_index,
++ struct ec_sensors_data *state, u32 *value)
++{
++ if (time_after(jiffies, state->last_updated + HZ)) {
++ if (update_ec_sensors(dev, state)) {
++ dev_err(dev, "update_ec_sensors() failure\n");
++ return -EIO;
++ }
++
++ state->last_updated = jiffies;
++ }
++
++ *value = state->sensors[sensor_index].cached_value;
++ return 0;
++}
++
++/*
++ * Now follow the functions that implement the hwmon interface
++ */
++
++static int asus_ec_hwmon_read(struct device *dev, enum hwmon_sensor_types type,
++ u32 attr, int channel, long *val)
++{
++ int ret;
++ u32 value = 0;
++
++ struct ec_sensors_data *state = dev_get_drvdata(dev);
++ int sidx = find_ec_sensor_index(state, type, channel);
++
++ if (sidx < 0) {
++ return sidx;
++ }
++
++ ret = get_cached_value_or_update(dev, sidx, state, &value);
++ if (!ret) {
++ *val = scale_sensor_value(value,
++ get_sensor_info(state, sidx)->type);
++ }
++
++ return ret;
++}
++
++static int asus_ec_hwmon_read_string(struct device *dev,
++ enum hwmon_sensor_types type, u32 attr,
++ int channel, const char **str)
++{
++ struct ec_sensors_data *state = dev_get_drvdata(dev);
++ int sensor_index = find_ec_sensor_index(state, type, channel);
++ *str = get_sensor_info(state, sensor_index)->label;
++
++ return 0;
++}
++
++static umode_t asus_ec_hwmon_is_visible(const void *drvdata,
++ enum hwmon_sensor_types type, u32 attr,
++ int channel)
++{
++ const struct ec_sensors_data *state = drvdata;
++
++ return find_ec_sensor_index(state, type, channel) >= 0 ? S_IRUGO : 0;
++}
++
++static int __init
++asus_ec_hwmon_add_chan_info(struct hwmon_channel_info *asus_ec_hwmon_chan,
++ struct device *dev, int num,
++ enum hwmon_sensor_types type, u32 config)
++{
++ int i;
++ u32 *cfg = devm_kcalloc(dev, num + 1, sizeof(*cfg), GFP_KERNEL);
++
++ if (!cfg)
++ return -ENOMEM;
++
++ asus_ec_hwmon_chan->type = type;
++ asus_ec_hwmon_chan->config = cfg;
++ for (i = 0; i < num; i++, cfg++)
++ *cfg = config;
++
++ return 0;
++}
++
++static const struct hwmon_ops asus_ec_hwmon_ops = {
++ .is_visible = asus_ec_hwmon_is_visible,
++ .read = asus_ec_hwmon_read,
++ .read_string = asus_ec_hwmon_read_string,
++};
++
++static struct hwmon_chip_info asus_ec_chip_info = {
++ .ops = &asus_ec_hwmon_ops,
++};
++
++static unsigned long __init
++get_board_sensors(const struct device *dev)
++{
++ const struct dmi_system_id *dmi_entry;
++
++ dmi_entry = dmi_first_match(asus_ec_dmi_table);
++ if (!dmi_entry) {
++ dev_info(dev, "Unsupported board");
++ return 0;
++ }
++
++ return (unsigned long)dmi_entry->driver_data;
++}
++
++static int __init configure_sensor_setup(struct device *dev)
++{
++ struct ec_sensors_data *ec_data = dev_get_drvdata(dev);
++ int nr_count[hwmon_max] = { 0 }, nr_types = 0;
++ struct device *hwdev;
++ struct hwmon_channel_info *asus_ec_hwmon_chan;
++ const struct hwmon_channel_info **ptr_asus_ec_ci;
++ const struct hwmon_chip_info *chip_info;
++ const struct ec_sensor_info *si;
++ enum hwmon_sensor_types type;
++ unsigned int i;
++
++ ec_data->board_sensors = get_board_sensors(dev);
++ if (!ec_data->board_sensors) {
++ return -ENODEV;
++ }
++
++ ec_data->nr_sensors = board_sensors_count(ec_data->board_sensors);
++ ec_data->sensors = devm_kcalloc(dev, ec_data->nr_sensors,
++ sizeof(struct ec_sensor), GFP_KERNEL);
++
++ setup_sensor_data(ec_data);
++ ec_data->registers = devm_kcalloc(dev, ec_data->nr_registers,
++ sizeof(u16), GFP_KERNEL);
++ ec_data->read_buffer = devm_kcalloc(dev, ec_data->nr_registers,
++ sizeof(u8), GFP_KERNEL);
++
++ if (!ec_data->registers || !ec_data->read_buffer) {
++ return -ENOMEM;
++ }
++
++ fill_ec_registers(ec_data);
++
++ ec_data->aml_mutex = asus_hw_access_mutex(dev);
++
++ for (i = 0; i < ec_data->nr_sensors; ++i) {
++ si = get_sensor_info(ec_data, i);
++ if (!nr_count[si->type])
++ ++nr_types;
++ ++nr_count[si->type];
++ }
++
++ if (nr_count[hwmon_temp])
++ nr_count[hwmon_chip]++, nr_types++;
++
++ asus_ec_hwmon_chan = devm_kcalloc(
++ dev, nr_types, sizeof(*asus_ec_hwmon_chan), GFP_KERNEL);
++ if (!asus_ec_hwmon_chan)
++ return -ENOMEM;
++
++ ptr_asus_ec_ci = devm_kcalloc(dev, nr_types + 1,
++ sizeof(*ptr_asus_ec_ci), GFP_KERNEL);
++ if (!ptr_asus_ec_ci)
++ return -ENOMEM;
++
++ asus_ec_chip_info.info = ptr_asus_ec_ci;
++ chip_info = &asus_ec_chip_info;
++
++ for (type = 0; type < hwmon_max; ++type) {
++ if (!nr_count[type])
++ continue;
++
++ asus_ec_hwmon_add_chan_info(asus_ec_hwmon_chan, dev,
++ nr_count[type], type,
++ hwmon_attributes[type]);
++ *ptr_asus_ec_ci++ = asus_ec_hwmon_chan++;
++ }
++
++ dev_info(dev, "board has %d EC sensors that span %d registers",
++ ec_data->nr_sensors, ec_data->nr_registers);
++
++ hwdev = devm_hwmon_device_register_with_info(dev, "asus_ec_sensors",
++ ec_data, chip_info, NULL);
++
++ return PTR_ERR_OR_ZERO(hwdev);
++}
++
++static int __init asus_ec_probe(struct platform_device *pdev)
++{
++ struct asus_ec_sensors *state;
++ int status = 0;
++
++ state = devm_kzalloc(&pdev->dev, sizeof(struct ec_sensors_data),
++ GFP_KERNEL);
++
++ if (!state) {
++ return -ENOMEM;
++ }
++
++ dev_set_drvdata(&pdev->dev, state);
++ status = configure_sensor_setup(&pdev->dev);
++ return status;
++}
++
++static const struct acpi_device_id acpi_ec_ids[] = {
++ /* Embedded Controller Device */
++ { "PNP0C09", 0 },
++ {}
++};
++
++static struct platform_driver asus_ec_sensors_platform_driver = {
++ .driver = {
++ .name = "asus-ec-sensors",
++ .acpi_match_table = acpi_ec_ids,
++ },
++};
++
++MODULE_DEVICE_TABLE(dmi, asus_ec_dmi_table);
++module_platform_driver_probe(asus_ec_sensors_platform_driver, asus_ec_probe);
++
++module_param_named(mutex_path, mutex_path_override, charp, 0);
++MODULE_PARM_DESC(mutex_path,
++ "Override ACPI mutex path used to guard access to hardware");
++
++MODULE_AUTHOR("Eugene Shalygin <[email protected]>");
++MODULE_DESCRIPTION(
++ "HWMON driver for sensors accessible via ACPI EC in ASUS motherboards");
++MODULE_LICENSE("GPL");
+--
+2.35.0
+
+From ea45e4a77e054ab10a669845700399f15213628e Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sun, 23 Jan 2022 16:50:19 +0000
+Subject: [PATCH] bbr2
+
+---
+ include/linux/tcp.h | 3 +-
+ include/net/inet_connection_sock.h | 3 +-
+ include/net/tcp.h | 41 +-
+ include/uapi/linux/inet_diag.h | 33 +
+ net/ipv4/Kconfig | 22 +
+ net/ipv4/Makefile | 1 +
+ net/ipv4/bpf_tcp_ca.c | 2 +-
+ net/ipv4/tcp.c | 1 +
+ net/ipv4/tcp_bbr.c | 38 +-
+ net/ipv4/tcp_bbr2.c | 2674 ++++++++++++++++++++++++++++
+ net/ipv4/tcp_cong.c | 1 +
+ net/ipv4/tcp_input.c | 27 +-
+ net/ipv4/tcp_output.c | 26 +-
+ net/ipv4/tcp_rate.c | 30 +-
+ net/ipv4/tcp_timer.c | 1 +
+ 15 files changed, 2868 insertions(+), 35 deletions(-)
+ create mode 100644 net/ipv4/tcp_bbr2.c
+
+diff --git a/include/linux/tcp.h b/include/linux/tcp.h
+index 78b91bb92f0d..618568b01a80 100644
+--- a/include/linux/tcp.h
++++ b/include/linux/tcp.h
+@@ -225,7 +225,8 @@ struct tcp_sock {
+ u8 compressed_ack;
+ u8 dup_ack_counter:2,
+ tlp_retrans:1, /* TLP is a retransmission */
+- unused:5;
++ fast_ack_mode:2, /* which fast ack mode ? */
++ unused:3;
+ u32 chrono_start; /* Start time in jiffies of a TCP chrono */
+ u32 chrono_stat[3]; /* Time in jiffies for chrono_stat stats */
+ u8 chrono_type:2, /* current chronograph type */
+diff --git a/include/net/inet_connection_sock.h b/include/net/inet_connection_sock.h
+index 4ad47d9f9d27..7623bb1f89c0 100644
+--- a/include/net/inet_connection_sock.h
++++ b/include/net/inet_connection_sock.h
+@@ -134,7 +134,8 @@ struct inet_connection_sock {
+ u32 icsk_probes_tstamp;
+ u32 icsk_user_timeout;
+
+- u64 icsk_ca_priv[104 / sizeof(u64)];
++/* XXX inflated by temporary internal debugging info */
++ u64 icsk_ca_priv[216 / sizeof(u64)];
+ #define ICSK_CA_PRIV_SIZE sizeof_field(struct inet_connection_sock, icsk_ca_priv)
+ };
+
+diff --git a/include/net/tcp.h b/include/net/tcp.h
+index 44e442bf23f9..b49fe895506e 100644
+--- a/include/net/tcp.h
++++ b/include/net/tcp.h
+@@ -370,6 +370,7 @@ static inline void tcp_dec_quickack_mode(struct sock *sk,
+ #define TCP_ECN_QUEUE_CWR 2
+ #define TCP_ECN_DEMAND_CWR 4
+ #define TCP_ECN_SEEN 8
++#define TCP_ECN_ECT_PERMANENT 16
+
+ enum tcp_tw_status {
+ TCP_TW_SUCCESS = 0,
+@@ -808,6 +809,11 @@ static inline u32 tcp_stamp_us_delta(u64 t1, u64 t0)
+ return max_t(s64, t1 - t0, 0);
+ }
+
++static inline u32 tcp_stamp32_us_delta(u32 t1, u32 t0)
++{
++ return max_t(s32, t1 - t0, 0);
++}
++
+ static inline u32 tcp_skb_timestamp(const struct sk_buff *skb)
+ {
+ return tcp_ns_to_ts(skb->skb_mstamp_ns);
+@@ -883,9 +889,14 @@ struct tcp_skb_cb {
+ /* pkts S/ACKed so far upon tx of skb, incl retrans: */
+ __u32 delivered;
+ /* start of send pipeline phase */
+- u64 first_tx_mstamp;
++ u32 first_tx_mstamp;
+ /* when we reached the "delivered" count */
+- u64 delivered_mstamp;
++ u32 delivered_mstamp;
++#define TCPCB_IN_FLIGHT_BITS 20
++#define TCPCB_IN_FLIGHT_MAX ((1U << TCPCB_IN_FLIGHT_BITS) - 1)
++ u32 in_flight:20, /* packets in flight at transmit */
++ unused2:12;
++ u32 lost; /* packets lost so far upon tx of skb */
+ } tx; /* only used for outgoing skbs */
+ union {
+ struct inet_skb_parm h4;
+@@ -1011,7 +1022,11 @@ enum tcp_ca_ack_event_flags {
+ #define TCP_CONG_NON_RESTRICTED 0x1
+ /* Requires ECN/ECT set on all packets */
+ #define TCP_CONG_NEEDS_ECN 0x2
+-#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | TCP_CONG_NEEDS_ECN)
++/* Wants notification of CE events (CA_EVENT_ECN_IS_CE, CA_EVENT_ECN_NO_CE). */
++#define TCP_CONG_WANTS_CE_EVENTS 0x4
++#define TCP_CONG_MASK (TCP_CONG_NON_RESTRICTED | \
++ TCP_CONG_NEEDS_ECN | \
++ TCP_CONG_WANTS_CE_EVENTS)
+
+ union tcp_cc_info;
+
+@@ -1031,8 +1046,11 @@ struct ack_sample {
+ */
+ struct rate_sample {
+ u64 prior_mstamp; /* starting timestamp for interval */
++ u32 prior_lost; /* tp->lost at "prior_mstamp" */
+ u32 prior_delivered; /* tp->delivered at "prior_mstamp" */
+ u32 prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */
++ u32 tx_in_flight; /* packets in flight at starting timestamp */
++ s32 lost; /* number of packets lost over interval */
+ s32 delivered; /* number of packets delivered over interval */
+ s32 delivered_ce; /* number of packets delivered w/ CE marks*/
+ long interval_us; /* time for tp->delivered to incr "delivered" */
+@@ -1045,6 +1063,7 @@ struct rate_sample {
+ bool is_app_limited; /* is sample from packet with bubble in pipe? */
+ bool is_retrans; /* is sample from retransmission? */
+ bool is_ack_delayed; /* is this (likely) a delayed ACK? */
++ bool is_ece; /* did this ACK have ECN marked? */
+ };
+
+ struct tcp_congestion_ops {
+@@ -1068,8 +1087,11 @@ struct tcp_congestion_ops {
+ /* hook for packet ack accounting (optional) */
+ void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
+
+- /* override sysctl_tcp_min_tso_segs */
+- u32 (*min_tso_segs)(struct sock *sk);
++ /* pick target number of segments per TSO/GSO skb (optional): */
++ u32 (*tso_segs)(struct sock *sk, unsigned int mss_now);
++
++ /* react to a specific lost skb (optional) */
++ void (*skb_marked_lost)(struct sock *sk, const struct sk_buff *skb);
+
+ /* call when packets are delivered to update cwnd and pacing rate,
+ * after all the ca_state processing. (optional)
+@@ -1132,6 +1154,14 @@ static inline char *tcp_ca_get_name_by_key(u32 key, char *buffer)
+ }
+ #endif
+
++static inline bool tcp_ca_wants_ce_events(const struct sock *sk)
++{
++ const struct inet_connection_sock *icsk = inet_csk(sk);
++
++ return icsk->icsk_ca_ops->flags & (TCP_CONG_NEEDS_ECN |
++ TCP_CONG_WANTS_CE_EVENTS);
++}
++
+ static inline bool tcp_ca_needs_ecn(const struct sock *sk)
+ {
+ const struct inet_connection_sock *icsk = inet_csk(sk);
+@@ -1157,6 +1187,7 @@ static inline void tcp_ca_event(struct sock *sk, const enum tcp_ca_event event)
+ }
+
+ /* From tcp_rate.c */
++void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb);
+ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb);
+ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
+ struct rate_sample *rs);
+diff --git a/include/uapi/linux/inet_diag.h b/include/uapi/linux/inet_diag.h
+index 20ee93f0f876..96d52dd9c48a 100644
+--- a/include/uapi/linux/inet_diag.h
++++ b/include/uapi/linux/inet_diag.h
+@@ -231,9 +231,42 @@ struct tcp_bbr_info {
+ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */
+ };
+
++/* Phase as reported in netlink/ss stats. */
++enum tcp_bbr2_phase {
++ BBR2_PHASE_INVALID = 0,
++ BBR2_PHASE_STARTUP = 1,
++ BBR2_PHASE_DRAIN = 2,
++ BBR2_PHASE_PROBE_RTT = 3,
++ BBR2_PHASE_PROBE_BW_UP = 4,
++ BBR2_PHASE_PROBE_BW_DOWN = 5,
++ BBR2_PHASE_PROBE_BW_CRUISE = 6,
++ BBR2_PHASE_PROBE_BW_REFILL = 7
++};
++
++struct tcp_bbr2_info {
++ /* u64 bw: bandwidth (app throughput) estimate in Byte per sec: */
++ __u32 bbr_bw_lsb; /* lower 32 bits of bw */
++ __u32 bbr_bw_msb; /* upper 32 bits of bw */
++ __u32 bbr_min_rtt; /* min-filtered RTT in uSec */
++ __u32 bbr_pacing_gain; /* pacing gain shifted left 8 bits */
++ __u32 bbr_cwnd_gain; /* cwnd gain shifted left 8 bits */
++ __u32 bbr_bw_hi_lsb; /* lower 32 bits of bw_hi */
++ __u32 bbr_bw_hi_msb; /* upper 32 bits of bw_hi */
++ __u32 bbr_bw_lo_lsb; /* lower 32 bits of bw_lo */
++ __u32 bbr_bw_lo_msb; /* upper 32 bits of bw_lo */
++ __u8 bbr_mode; /* current bbr_mode in state machine */
++ __u8 bbr_phase; /* current state machine phase */
++ __u8 unused1; /* alignment padding; not used yet */
++ __u8 bbr_version; /* MUST be at this offset in struct */
++ __u32 bbr_inflight_lo; /* lower/short-term data volume bound */
++ __u32 bbr_inflight_hi; /* higher/long-term data volume bound */
++ __u32 bbr_extra_acked; /* max excess packets ACKed in epoch */
++};
++
+ union tcp_cc_info {
+ struct tcpvegas_info vegas;
+ struct tcp_dctcp_info dctcp;
+ struct tcp_bbr_info bbr;
++ struct tcp_bbr2_info bbr2;
+ };
+ #endif /* _UAPI_INET_DIAG_H_ */
+diff --git a/net/ipv4/Kconfig b/net/ipv4/Kconfig
+index 87983e70f03f..a833a7a67ce7 100644
+--- a/net/ipv4/Kconfig
++++ b/net/ipv4/Kconfig
+@@ -669,6 +669,24 @@ config TCP_CONG_BBR
+ AQM schemes that do not provide a delay signal. It requires the fq
+ ("Fair Queue") pacing packet scheduler.
+
++config TCP_CONG_BBR2
++ tristate "BBR2 TCP"
++ default n
++ help
++
++ BBR2 TCP congestion control is a model-based congestion control
++ algorithm that aims to maximize network utilization, keep queues and
++ retransmit rates low, and to be able to coexist with Reno/CUBIC in
++ common scenarios. It builds an explicit model of the network path. It
++ tolerates a targeted degree of random packet loss and delay that are
++ unrelated to congestion. It can operate over LAN, WAN, cellular, wifi,
++ or cable modem links, and can use DCTCP-L4S-style ECN signals. It can
++ coexist with flows that use loss-based congestion control, and can
++ operate with shallow buffers, deep buffers, bufferbloat, policers, or
++ AQM schemes that do not provide a delay signal. It requires pacing,
++ using either TCP internal pacing or the fq ("Fair Queue") pacing packet
++ scheduler.
++
+ choice
+ prompt "Default TCP congestion control"
+ default DEFAULT_CUBIC
+@@ -706,6 +724,9 @@ choice
+ config DEFAULT_BBR
+ bool "BBR" if TCP_CONG_BBR=y
+
++ config DEFAULT_BBR2
++ bool "BBR2" if TCP_CONG_BBR2=y
++
+ config DEFAULT_RENO
+ bool "Reno"
+ endchoice
+@@ -730,6 +751,7 @@ config DEFAULT_TCP_CONG
+ default "dctcp" if DEFAULT_DCTCP
+ default "cdg" if DEFAULT_CDG
+ default "bbr" if DEFAULT_BBR
++ default "bbr2" if DEFAULT_BBR2
+ default "cubic"
+
+ config TCP_MD5SIG
+diff --git a/net/ipv4/Makefile b/net/ipv4/Makefile
+index bbdd9c44f14e..8dee1547d820 100644
+--- a/net/ipv4/Makefile
++++ b/net/ipv4/Makefile
+@@ -46,6 +46,7 @@ obj-$(CONFIG_INET_TCP_DIAG) += tcp_diag.o
+ obj-$(CONFIG_INET_UDP_DIAG) += udp_diag.o
+ obj-$(CONFIG_INET_RAW_DIAG) += raw_diag.o
+ obj-$(CONFIG_TCP_CONG_BBR) += tcp_bbr.o
++obj-$(CONFIG_TCP_CONG_BBR2) += tcp_bbr2.o
+ obj-$(CONFIG_TCP_CONG_BIC) += tcp_bic.o
+ obj-$(CONFIG_TCP_CONG_CDG) += tcp_cdg.o
+ obj-$(CONFIG_TCP_CONG_CUBIC) += tcp_cubic.o
+diff --git a/net/ipv4/bpf_tcp_ca.c b/net/ipv4/bpf_tcp_ca.c
+index de610cb83694..afda34c26c62 100644
+--- a/net/ipv4/bpf_tcp_ca.c
++++ b/net/ipv4/bpf_tcp_ca.c
+@@ -20,7 +20,7 @@ static u32 optional_ops[] = {
+ offsetof(struct tcp_congestion_ops, cwnd_event),
+ offsetof(struct tcp_congestion_ops, in_ack_event),
+ offsetof(struct tcp_congestion_ops, pkts_acked),
+- offsetof(struct tcp_congestion_ops, min_tso_segs),
++ offsetof(struct tcp_congestion_ops, tso_segs),
+ offsetof(struct tcp_congestion_ops, sndbuf_expand),
+ offsetof(struct tcp_congestion_ops, cong_control),
+ };
+diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
+index 3b75836db19b..1f1ff1e18e69 100644
+--- a/net/ipv4/tcp.c
++++ b/net/ipv4/tcp.c
+@@ -3074,6 +3074,7 @@ int tcp_disconnect(struct sock *sk, int flags)
+ tp->rx_opt.dsack = 0;
+ tp->rx_opt.num_sacks = 0;
+ tp->rcv_ooopack = 0;
++ tp->fast_ack_mode = 0;
+
+
+ /* Clean up fastopen related fields */
+diff --git a/net/ipv4/tcp_bbr.c b/net/ipv4/tcp_bbr.c
+index ec5550089b4d..56e719166d4c 100644
+--- a/net/ipv4/tcp_bbr.c
++++ b/net/ipv4/tcp_bbr.c
+@@ -294,26 +294,40 @@ static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain)
+ sk->sk_pacing_rate = rate;
+ }
+
+-/* override sysctl_tcp_min_tso_segs */
+ static u32 bbr_min_tso_segs(struct sock *sk)
+ {
+ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2;
+ }
+
++/* Return the number of segments BBR would like in a TSO/GSO skb, given
++ * a particular max gso size as a constraint.
++ */
++static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now,
++ u32 gso_max_size)
++{
++ u32 segs;
++ u64 bytes;
++
++ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */
++ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift;
++
++ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER);
++ segs = max_t(u32, bytes / mss_now, bbr_min_tso_segs(sk));
++ return segs;
++}
++
++/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */
++static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now)
++{
++ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size);
++}
++
++/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */
+ static u32 bbr_tso_segs_goal(struct sock *sk)
+ {
+ struct tcp_sock *tp = tcp_sk(sk);
+- u32 segs, bytes;
+-
+- /* Sort of tcp_tso_autosize() but ignoring
+- * driver provided sk_gso_max_size.
+- */
+- bytes = min_t(unsigned long,
+- sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift),
+- GSO_MAX_SIZE - 1 - MAX_TCP_HEADER);
+- segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk));
+
+- return min(segs, 0x7FU);
++ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE);
+ }
+
+ /* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */
+@@ -1149,7 +1163,7 @@ static struct tcp_congestion_ops tcp_bbr_cong_ops __read_mostly = {
+ .undo_cwnd = bbr_undo_cwnd,
+ .cwnd_event = bbr_cwnd_event,
+ .ssthresh = bbr_ssthresh,
+- .min_tso_segs = bbr_min_tso_segs,
++ .tso_segs = bbr_tso_segs,
+ .get_info = bbr_get_info,
+ .set_state = bbr_set_state,
+ };
+diff --git a/net/ipv4/tcp_bbr2.c b/net/ipv4/tcp_bbr2.c
+new file mode 100644
+index 000000000000..1bef2336b3ff
+--- /dev/null
++++ b/net/ipv4/tcp_bbr2.c
+@@ -0,0 +1,2674 @@
++/* BBR (Bottleneck Bandwidth and RTT) congestion control, v2
++ *
++ * BBRv2 is a model-based congestion control algorithm that aims for low
++ * queues, low loss, and (bounded) Reno/CUBIC coexistence. To maintain a model
++ * of the network path, it uses measurements of bandwidth and RTT, as well as
++ * (if they occur) packet loss and/or DCTCP/L4S-style ECN signals. Note that
++ * although it can use ECN or loss signals explicitly, it does not require
++ * either; it can bound its in-flight data based on its estimate of the BDP.
++ *
++ * The model has both higher and lower bounds for the operating range:
++ * lo: bw_lo, inflight_lo: conservative short-term lower bound
++ * hi: bw_hi, inflight_hi: robust long-term upper bound
++ * The bandwidth-probing time scale is (a) extended dynamically based on
++ * estimated BDP to improve coexistence with Reno/CUBIC; (b) bounded by
++ * an interactive wall-clock time-scale to be more scalable and responsive
++ * than Reno and CUBIC.
++ *
++ * Here is a state transition diagram for BBR:
++ *
++ * |
++ * V
++ * +---> STARTUP ----+
++ * | | |
++ * | V |
++ * | DRAIN ----+
++ * | | |
++ * | V |
++ * +---> PROBE_BW ----+
++ * | ^ | |
++ * | | | |
++ * | +----+ |
++ * | |
++ * +---- PROBE_RTT <--+
++ *
++ * A BBR flow starts in STARTUP, and ramps up its sending rate quickly.
++ * When it estimates the pipe is full, it enters DRAIN to drain the queue.
++ * In steady state a BBR flow only uses PROBE_BW and PROBE_RTT.
++ * A long-lived BBR flow spends the vast majority of its time remaining
++ * (repeatedly) in PROBE_BW, fully probing and utilizing the pipe's bandwidth
++ * in a fair manner, with a small, bounded queue. *If* a flow has been
++ * continuously sending for the entire min_rtt window, and hasn't seen an RTT
++ * sample that matches or decreases its min_rtt estimate for 10 seconds, then
++ * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe
++ * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if
++ * we estimated that we reached the full bw of the pipe then we enter PROBE_BW;
++ * otherwise we enter STARTUP to try to fill the pipe.
++ *
++ * BBR is described in detail in:
++ * "BBR: Congestion-Based Congestion Control",
++ * Neal Cardwell, Yuchung Cheng, C. Stephen Gunn, Soheil Hassas Yeganeh,
++ * Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016.
++ *
++ * There is a public e-mail list for discussing BBR development and testing:
++ * https://groups.google.com/forum/#!forum/bbr-dev
++ *
++ * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled,
++ * otherwise TCP stack falls back to an internal pacing using one high
++ * resolution timer per TCP socket and may use more resources.
++ */
++#include <linux/module.h>
++#include <net/tcp.h>
++#include <linux/inet_diag.h>
++#include <linux/inet.h>
++#include <linux/random.h>
++
++#include "tcp_dctcp.h"
++
++/* Scale factor for rate in pkt/uSec unit to avoid truncation in bandwidth
++ * estimation. The rate unit ~= (1500 bytes / 1 usec / 2^24) ~= 715 bps.
++ * This handles bandwidths from 0.06pps (715bps) to 256Mpps (3Tbps) in a u32.
++ * Since the minimum window is >=4 packets, the lower bound isn't
++ * an issue. The upper bound isn't an issue with existing technologies.
++ */
++#define BW_SCALE 24
++#define BW_UNIT (1 << BW_SCALE)
++
++#define BBR_SCALE 8 /* scaling factor for fractions in BBR (e.g. gains) */
++#define BBR_UNIT (1 << BBR_SCALE)
++
++#define FLAG_DEBUG_VERBOSE 0x1 /* Verbose debugging messages */
++#define FLAG_DEBUG_LOOPBACK 0x2 /* Do NOT skip loopback addr */
++
++#define CYCLE_LEN 8 /* number of phases in a pacing gain cycle */
++
++/* BBR has the following modes for deciding how fast to send: */
++enum bbr_mode {
++ BBR_STARTUP, /* ramp up sending rate rapidly to fill pipe */
++ BBR_DRAIN, /* drain any queue created during startup */
++ BBR_PROBE_BW, /* discover, share bw: pace around estimated bw */
++ BBR_PROBE_RTT, /* cut inflight to min to probe min_rtt */
++};
++
++/* How does the incoming ACK stream relate to our bandwidth probing? */
++enum bbr_ack_phase {
++ BBR_ACKS_INIT, /* not probing; not getting probe feedback */
++ BBR_ACKS_REFILLING, /* sending at est. bw to fill pipe */
++ BBR_ACKS_PROBE_STARTING, /* inflight rising to probe bw */
++ BBR_ACKS_PROBE_FEEDBACK, /* getting feedback from bw probing */
++ BBR_ACKS_PROBE_STOPPING, /* stopped probing; still getting feedback */
++};
++
++/* BBR congestion control block */
++struct bbr {
++ u32 min_rtt_us; /* min RTT in min_rtt_win_sec window */
++ u32 min_rtt_stamp; /* timestamp of min_rtt_us */
++ u32 probe_rtt_done_stamp; /* end time for BBR_PROBE_RTT mode */
++ u32 probe_rtt_min_us; /* min RTT in bbr_probe_rtt_win_ms window */
++ u32 probe_rtt_min_stamp; /* timestamp of probe_rtt_min_us*/
++ u32 next_rtt_delivered; /* scb->tx.delivered at end of round */
++ u32 prior_rcv_nxt; /* tp->rcv_nxt when CE state last changed */
++ u64 cycle_mstamp; /* time of this cycle phase start */
++ u32 mode:3, /* current bbr_mode in state machine */
++ prev_ca_state:3, /* CA state on previous ACK */
++ packet_conservation:1, /* use packet conservation? */
++ round_start:1, /* start of packet-timed tx->ack round? */
++ ce_state:1, /* If most recent data has CE bit set */
++ bw_probe_up_rounds:5, /* cwnd-limited rounds in PROBE_UP */
++ try_fast_path:1, /* can we take fast path? */
++ unused2:11,
++ idle_restart:1, /* restarting after idle? */
++ probe_rtt_round_done:1, /* a BBR_PROBE_RTT round at 4 pkts? */
++ cycle_idx:3, /* current index in pacing_gain cycle array */
++ has_seen_rtt:1; /* have we seen an RTT sample yet? */
++ u32 pacing_gain:11, /* current gain for setting pacing rate */
++ cwnd_gain:11, /* current gain for setting cwnd */
++ full_bw_reached:1, /* reached full bw in Startup? */
++ full_bw_cnt:2, /* number of rounds without large bw gains */
++ init_cwnd:7; /* initial cwnd */
++ u32 prior_cwnd; /* prior cwnd upon entering loss recovery */
++ u32 full_bw; /* recent bw, to estimate if pipe is full */
++
++ /* For tracking ACK aggregation: */
++ u64 ack_epoch_mstamp; /* start of ACK sampling epoch */
++ u16 extra_acked[2]; /* max excess data ACKed in epoch */
++ u32 ack_epoch_acked:20, /* packets (S)ACKed in sampling epoch */
++ extra_acked_win_rtts:5, /* age of extra_acked, in round trips */
++ extra_acked_win_idx:1, /* current index in extra_acked array */
++ /* BBR v2 state: */
++ unused1:2,
++ startup_ecn_rounds:2, /* consecutive hi ECN STARTUP rounds */
++ loss_in_cycle:1, /* packet loss in this cycle? */
++ ecn_in_cycle:1; /* ECN in this cycle? */
++ u32 loss_round_delivered; /* scb->tx.delivered ending loss round */
++ u32 undo_bw_lo; /* bw_lo before latest losses */
++ u32 undo_inflight_lo; /* inflight_lo before latest losses */
++ u32 undo_inflight_hi; /* inflight_hi before latest losses */
++ u32 bw_latest; /* max delivered bw in last round trip */
++ u32 bw_lo; /* lower bound on sending bandwidth */
++ u32 bw_hi[2]; /* upper bound of sending bandwidth range*/
++ u32 inflight_latest; /* max delivered data in last round trip */
++ u32 inflight_lo; /* lower bound of inflight data range */
++ u32 inflight_hi; /* upper bound of inflight data range */
++ u32 bw_probe_up_cnt; /* packets delivered per inflight_hi incr */
++ u32 bw_probe_up_acks; /* packets (S)ACKed since inflight_hi incr */
++ u32 probe_wait_us; /* PROBE_DOWN until next clock-driven probe */
++ u32 ecn_eligible:1, /* sender can use ECN (RTT, handshake)? */
++ ecn_alpha:9, /* EWMA delivered_ce/delivered; 0..256 */
++ bw_probe_samples:1, /* rate samples reflect bw probing? */
++ prev_probe_too_high:1, /* did last PROBE_UP go too high? */
++ stopped_risky_probe:1, /* last PROBE_UP stopped due to risk? */
++ rounds_since_probe:8, /* packet-timed rounds since probed bw */
++ loss_round_start:1, /* loss_round_delivered round trip? */
++ loss_in_round:1, /* loss marked in this round trip? */
++ ecn_in_round:1, /* ECN marked in this round trip? */
++ ack_phase:3, /* bbr_ack_phase: meaning of ACKs */
++ loss_events_in_round:4,/* losses in STARTUP round */
++ initialized:1; /* has bbr_init() been called? */
++ u32 alpha_last_delivered; /* tp->delivered at alpha update */
++ u32 alpha_last_delivered_ce; /* tp->delivered_ce at alpha update */
++
++ /* Params configurable using setsockopt. Refer to correspoding
++ * module param for detailed description of params.
++ */
++ struct bbr_params {
++ u32 high_gain:11, /* max allowed value: 2047 */
++ drain_gain:10, /* max allowed value: 1023 */
++ cwnd_gain:11; /* max allowed value: 2047 */
++ u32 cwnd_min_target:4, /* max allowed value: 15 */
++ min_rtt_win_sec:5, /* max allowed value: 31 */
++ probe_rtt_mode_ms:9, /* max allowed value: 511 */
++ full_bw_cnt:3, /* max allowed value: 7 */
++ cwnd_tso_budget:1, /* allowed values: {0, 1} */
++ unused3:6,
++ drain_to_target:1, /* boolean */
++ precise_ece_ack:1, /* boolean */
++ extra_acked_in_startup:1, /* allowed values: {0, 1} */
++ fast_path:1; /* boolean */
++ u32 full_bw_thresh:10, /* max allowed value: 1023 */
++ startup_cwnd_gain:11, /* max allowed value: 2047 */
++ bw_probe_pif_gain:9, /* max allowed value: 511 */
++ usage_based_cwnd:1, /* boolean */
++ unused2:1;
++ u16 probe_rtt_win_ms:14, /* max allowed value: 16383 */
++ refill_add_inc:2; /* max allowed value: 3 */
++ u16 extra_acked_gain:11, /* max allowed value: 2047 */
++ extra_acked_win_rtts:5; /* max allowed value: 31*/
++ u16 pacing_gain[CYCLE_LEN]; /* max allowed value: 1023 */
++ /* Mostly BBR v2 parameters below here: */
++ u32 ecn_alpha_gain:8, /* max allowed value: 255 */
++ ecn_factor:8, /* max allowed value: 255 */
++ ecn_thresh:8, /* max allowed value: 255 */
++ beta:8; /* max allowed value: 255 */
++ u32 ecn_max_rtt_us:19, /* max allowed value: 524287 */
++ bw_probe_reno_gain:9, /* max allowed value: 511 */
++ full_loss_cnt:4; /* max allowed value: 15 */
++ u32 probe_rtt_cwnd_gain:8, /* max allowed value: 255 */
++ inflight_headroom:8, /* max allowed value: 255 */
++ loss_thresh:8, /* max allowed value: 255 */
++ bw_probe_max_rounds:8; /* max allowed value: 255 */
++ u32 bw_probe_rand_rounds:4, /* max allowed value: 15 */
++ bw_probe_base_us:26, /* usecs: 0..2^26-1 (67 secs) */
++ full_ecn_cnt:2; /* max allowed value: 3 */
++ u32 bw_probe_rand_us:26, /* usecs: 0..2^26-1 (67 secs) */
++ undo:1, /* boolean */
++ tso_rtt_shift:4, /* max allowed value: 15 */
++ unused5:1;
++ u32 ecn_reprobe_gain:9, /* max allowed value: 511 */
++ unused1:14,
++ ecn_alpha_init:9; /* max allowed value: 256 */
++ } params;
++
++ struct {
++ u32 snd_isn; /* Initial sequence number */
++ u32 rs_bw; /* last valid rate sample bw */
++ u32 target_cwnd; /* target cwnd, based on BDP */
++ u8 undo:1, /* Undo even happened but not yet logged */
++ unused:7;
++ char event; /* single-letter event debug codes */
++ u16 unused2;
++ } debug;
++};
++
++struct bbr_context {
++ u32 sample_bw;
++ u32 target_cwnd;
++ u32 log:1;
++};
++
++/* Window length of min_rtt filter (in sec). Max allowed value is 31 (0x1F) */
++static u32 bbr_min_rtt_win_sec = 10;
++/* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode.
++ * Max allowed value is 511 (0x1FF).
++ */
++static u32 bbr_probe_rtt_mode_ms = 200;
++/* Window length of probe_rtt_min_us filter (in ms), and consequently the
++ * typical interval between PROBE_RTT mode entries.
++ * Note that bbr_probe_rtt_win_ms must be <= bbr_min_rtt_win_sec * MSEC_PER_SEC
++ */
++static u32 bbr_probe_rtt_win_ms = 5000;
++/* Skip TSO below the following bandwidth (bits/sec): */
++static int bbr_min_tso_rate = 1200000;
++
++/* Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting
++ * in bigger TSO bursts. By default we cut the RTT-based allowance in half
++ * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance
++ * is below 1500 bytes after 6 * ~500 usec = 3ms.
++ */
++static u32 bbr_tso_rtt_shift = 9; /* halve allowance per 2^9 usecs, 512us */
++
++/* Select cwnd TSO budget approach:
++ * 0: padding
++ * 1: flooring
++ */
++static uint bbr_cwnd_tso_budget = 1;
++
++/* Pace at ~1% below estimated bw, on average, to reduce queue at bottleneck.
++ * In order to help drive the network toward lower queues and low latency while
++ * maintaining high utilization, the average pacing rate aims to be slightly
++ * lower than the estimated bandwidth. This is an important aspect of the
++ * design.
++ */
++static const int bbr_pacing_margin_percent = 1;
++
++/* We use a high_gain value of 2/ln(2) because it's the smallest pacing gain
++ * that will allow a smoothly increasing pacing rate that will double each RTT
++ * and send the same number of packets per RTT that an un-paced, slow-starting
++ * Reno or CUBIC flow would. Max allowed value is 2047 (0x7FF).
++ */
++static int bbr_high_gain = BBR_UNIT * 2885 / 1000 + 1;
++/* The gain for deriving startup cwnd. Max allowed value is 2047 (0x7FF). */
++static int bbr_startup_cwnd_gain = BBR_UNIT * 2885 / 1000 + 1;
++/* The pacing gain of 1/high_gain in BBR_DRAIN is calculated to typically drain
++ * the queue created in BBR_STARTUP in a single round. Max allowed value
++ * is 1023 (0x3FF).
++ */
++static int bbr_drain_gain = BBR_UNIT * 1000 / 2885;
++/* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs.
++ * Max allowed value is 2047 (0x7FF).
++ */
++static int bbr_cwnd_gain = BBR_UNIT * 2;
++/* The pacing_gain values for the PROBE_BW gain cycle, to discover/share bw.
++ * Max allowed value for each element is 1023 (0x3FF).
++ */
++enum bbr_pacing_gain_phase {
++ BBR_BW_PROBE_UP = 0, /* push up inflight to probe for bw/vol */
++ BBR_BW_PROBE_DOWN = 1, /* drain excess inflight from the queue */
++ BBR_BW_PROBE_CRUISE = 2, /* use pipe, w/ headroom in queue/pipe */
++ BBR_BW_PROBE_REFILL = 3, /* v2: refill the pipe again to 100% */
++};
++static int bbr_pacing_gain[] = {
++ BBR_UNIT * 5 / 4, /* probe for more available bw */
++ BBR_UNIT * 3 / 4, /* drain queue and/or yield bw to other flows */
++ BBR_UNIT, BBR_UNIT, BBR_UNIT, /* cruise at 1.0*bw to utilize pipe, */
++ BBR_UNIT, BBR_UNIT, BBR_UNIT /* without creating excess queue... */
++};
++
++/* Try to keep at least this many packets in flight, if things go smoothly. For
++ * smooth functioning, a sliding window protocol ACKing every other packet
++ * needs at least 4 packets in flight. Max allowed value is 15 (0xF).
++ */
++static u32 bbr_cwnd_min_target = 4;
++
++/* Cwnd to BDP proportion in PROBE_RTT mode scaled by BBR_UNIT. Default: 50%.
++ * Use 0 to disable. Max allowed value is 255.
++ */
++static u32 bbr_probe_rtt_cwnd_gain = BBR_UNIT * 1 / 2;
++
++/* To estimate if BBR_STARTUP mode (i.e. high_gain) has filled pipe... */
++/* If bw has increased significantly (1.25x), there may be more bw available.
++ * Max allowed value is 1023 (0x3FF).
++ */
++static u32 bbr_full_bw_thresh = BBR_UNIT * 5 / 4;
++/* But after 3 rounds w/o significant bw growth, estimate pipe is full.
++ * Max allowed value is 7 (0x7).
++ */
++static u32 bbr_full_bw_cnt = 3;
++
++static u32 bbr_flags; /* Debugging related stuff */
++
++/* Whether to debug using printk.
++ */
++static bool bbr_debug_with_printk;
++
++/* Whether to debug using ftrace event tcp:tcp_bbr_event.
++ * Ignored when bbr_debug_with_printk is set.
++ */
++static bool bbr_debug_ftrace;
++
++/* Experiment: each cycle, try to hold sub-unity gain until inflight <= BDP. */
++static bool bbr_drain_to_target = true; /* default: enabled */
++
++/* Experiment: Flags to control BBR with ECN behavior.
++ */
++static bool bbr_precise_ece_ack = true; /* default: enabled */
++
++/* The max rwin scaling shift factor is 14 (RFC 1323), so the max sane rwin is
++ * (2^(16+14) B)/(1024 B/packet) = 1M packets.
++ */
++static u32 bbr_cwnd_warn_val = 1U << 20;
++
++static u16 bbr_debug_port_mask;
++
++/* BBR module parameters. These are module parameters only in Google prod.
++ * Upstream these are intentionally not module parameters.
++ */
++static int bbr_pacing_gain_size = CYCLE_LEN;
++
++/* Gain factor for adding extra_acked to target cwnd: */
++static int bbr_extra_acked_gain = 256;
++
++/* Window length of extra_acked window. Max allowed val is 31. */
++static u32 bbr_extra_acked_win_rtts = 5;
++
++/* Max allowed val for ack_epoch_acked, after which sampling epoch is reset */
++static u32 bbr_ack_epoch_acked_reset_thresh = 1U << 20;
++
++/* Time period for clamping cwnd increment due to ack aggregation */
++static u32 bbr_extra_acked_max_us = 100 * 1000;
++
++/* Use extra acked in startup ?
++ * 0: disabled
++ * 1: use latest extra_acked value from 1-2 rtt in startup
++ */
++static int bbr_extra_acked_in_startup = 1; /* default: enabled */
++
++/* Experiment: don't grow cwnd beyond twice of what we just probed. */
++static bool bbr_usage_based_cwnd; /* default: disabled */
++
++/* For lab testing, researchers can enable BBRv2 ECN support with this flag,
++ * when they know that any ECN marks that the connections experience will be
++ * DCTCP/L4S-style ECN marks, rather than RFC3168 ECN marks.
++ * TODO(ncardwell): Production use of the BBRv2 ECN functionality depends on
++ * negotiation or configuration that is outside the scope of the BBRv2
++ * alpha release.
++ */
++static bool bbr_ecn_enable = false;
++
++module_param_named(min_tso_rate, bbr_min_tso_rate, int, 0644);
++module_param_named(tso_rtt_shift, bbr_tso_rtt_shift, int, 0644);
++module_param_named(high_gain, bbr_high_gain, int, 0644);
++module_param_named(drain_gain, bbr_drain_gain, int, 0644);
++module_param_named(startup_cwnd_gain, bbr_startup_cwnd_gain, int, 0644);
++module_param_named(cwnd_gain, bbr_cwnd_gain, int, 0644);
++module_param_array_named(pacing_gain, bbr_pacing_gain, int,
++ &bbr_pacing_gain_size, 0644);
++module_param_named(cwnd_min_target, bbr_cwnd_min_target, uint, 0644);
++module_param_named(probe_rtt_cwnd_gain,
++ bbr_probe_rtt_cwnd_gain, uint, 0664);
++module_param_named(cwnd_warn_val, bbr_cwnd_warn_val, uint, 0664);
++module_param_named(debug_port_mask, bbr_debug_port_mask, ushort, 0644);
++module_param_named(flags, bbr_flags, uint, 0644);
++module_param_named(debug_ftrace, bbr_debug_ftrace, bool, 0644);
++module_param_named(debug_with_printk, bbr_debug_with_printk, bool, 0644);
++module_param_named(min_rtt_win_sec, bbr_min_rtt_win_sec, uint, 0644);
++module_param_named(probe_rtt_mode_ms, bbr_probe_rtt_mode_ms, uint, 0644);
++module_param_named(probe_rtt_win_ms, bbr_probe_rtt_win_ms, uint, 0644);
++module_param_named(full_bw_thresh, bbr_full_bw_thresh, uint, 0644);
++module_param_named(full_bw_cnt, bbr_full_bw_cnt, uint, 0644);
++module_param_named(cwnd_tso_bduget, bbr_cwnd_tso_budget, uint, 0664);
++module_param_named(extra_acked_gain, bbr_extra_acked_gain, int, 0664);
++module_param_named(extra_acked_win_rtts,
++ bbr_extra_acked_win_rtts, uint, 0664);
++module_param_named(extra_acked_max_us,
++ bbr_extra_acked_max_us, uint, 0664);
++module_param_named(ack_epoch_acked_reset_thresh,
++ bbr_ack_epoch_acked_reset_thresh, uint, 0664);
++module_param_named(drain_to_target, bbr_drain_to_target, bool, 0664);
++module_param_named(precise_ece_ack, bbr_precise_ece_ack, bool, 0664);
++module_param_named(extra_acked_in_startup,
++ bbr_extra_acked_in_startup, int, 0664);
++module_param_named(usage_based_cwnd, bbr_usage_based_cwnd, bool, 0664);
++module_param_named(ecn_enable, bbr_ecn_enable, bool, 0664);
++
++static void bbr2_exit_probe_rtt(struct sock *sk);
++static void bbr2_reset_congestion_signals(struct sock *sk);
++
++static void bbr_check_probe_rtt_done(struct sock *sk);
++
++/* Do we estimate that STARTUP filled the pipe? */
++static bool bbr_full_bw_reached(const struct sock *sk)
++{
++ const struct bbr *bbr = inet_csk_ca(sk);
++
++ return bbr->full_bw_reached;
++}
++
++/* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */
++static u32 bbr_max_bw(const struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ return max(bbr->bw_hi[0], bbr->bw_hi[1]);
++}
++
++/* Return the estimated bandwidth of the path, in pkts/uS << BW_SCALE. */
++static u32 bbr_bw(const struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ return min(bbr_max_bw(sk), bbr->bw_lo);
++}
++
++/* Return maximum extra acked in past k-2k round trips,
++ * where k = bbr_extra_acked_win_rtts.
++ */
++static u16 bbr_extra_acked(const struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ return max(bbr->extra_acked[0], bbr->extra_acked[1]);
++}
++
++/* Return rate in bytes per second, optionally with a gain.
++ * The order here is chosen carefully to avoid overflow of u64. This should
++ * work for input rates of up to 2.9Tbit/sec and gain of 2.89x.
++ */
++static u64 bbr_rate_bytes_per_sec(struct sock *sk, u64 rate, int gain,
++ int margin)
++{
++ unsigned int mss = tcp_sk(sk)->mss_cache;
++
++ rate *= mss;
++ rate *= gain;
++ rate >>= BBR_SCALE;
++ rate *= USEC_PER_SEC / 100 * (100 - margin);
++ rate >>= BW_SCALE;
++ rate = max(rate, 1ULL);
++ return rate;
++}
++
++static u64 bbr_bw_bytes_per_sec(struct sock *sk, u64 rate)
++{
++ return bbr_rate_bytes_per_sec(sk, rate, BBR_UNIT, 0);
++}
++
++static u64 bbr_rate_kbps(struct sock *sk, u64 rate)
++{
++ rate = bbr_bw_bytes_per_sec(sk, rate);
++ rate *= 8;
++ do_div(rate, 1000);
++ return rate;
++}
++
++static u32 bbr_tso_segs_goal(struct sock *sk);
++static void bbr_debug(struct sock *sk, u32 acked,
++ const struct rate_sample *rs, struct bbr_context *ctx)
++{
++ static const char ca_states[] = {
++ [TCP_CA_Open] = 'O',
++ [TCP_CA_Disorder] = 'D',
++ [TCP_CA_CWR] = 'C',
++ [TCP_CA_Recovery] = 'R',
++ [TCP_CA_Loss] = 'L',
++ };
++ static const char mode[] = {
++ 'G', /* Growing - BBR_STARTUP */
++ 'D', /* Drain - BBR_DRAIN */
++ 'W', /* Window - BBR_PROBE_BW */
++ 'M', /* Min RTT - BBR_PROBE_RTT */
++ };
++ static const char ack_phase[] = { /* bbr_ack_phase strings */
++ 'I', /* BBR_ACKS_INIT - 'Init' */
++ 'R', /* BBR_ACKS_REFILLING - 'Refilling' */
++ 'B', /* BBR_ACKS_PROBE_STARTING - 'Before' */
++ 'F', /* BBR_ACKS_PROBE_FEEDBACK - 'Feedback' */
++ 'A', /* BBR_ACKS_PROBE_STOPPING - 'After' */
++ };
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ const u32 una = tp->snd_una - bbr->debug.snd_isn;
++ const u32 fack = tcp_highest_sack_seq(tp);
++ const u16 dport = ntohs(inet_sk(sk)->inet_dport);
++ bool is_port_match = (bbr_debug_port_mask &&
++ ((dport & bbr_debug_port_mask) == 0));
++ char debugmsg[320];
++
++ if (sk->sk_state == TCP_SYN_SENT)
++ return; /* no bbr_init() yet if SYN retransmit -> CA_Loss */
++
++ if (!tp->snd_cwnd || tp->snd_cwnd > bbr_cwnd_warn_val) {
++ char addr[INET6_ADDRSTRLEN + 10] = { 0 };
++
++ if (sk->sk_family == AF_INET)
++ snprintf(addr, sizeof(addr), "%pI4:%u",
++ &inet_sk(sk)->inet_daddr, dport);
++ else if (sk->sk_family == AF_INET6)
++ snprintf(addr, sizeof(addr), "%pI6:%u",
++ &sk->sk_daddr, dport);
++
++ WARN_ONCE(1,
++ "BBR %s cwnd alert: %u "
++ "snd_una: %u ca: %d pacing_gain: %u cwnd_gain: %u "
++ "bw: %u rtt: %u min_rtt: %u "
++ "acked: %u tso_segs: %u "
++ "bw: %d %ld %d pif: %u\n",
++ addr, tp->snd_cwnd,
++ una, inet_csk(sk)->icsk_ca_state,
++ bbr->pacing_gain, bbr->cwnd_gain,
++ bbr_max_bw(sk), (tp->srtt_us >> 3), bbr->min_rtt_us,
++ acked, bbr_tso_segs_goal(sk),
++ rs->delivered, rs->interval_us, rs->is_retrans,
++ tcp_packets_in_flight(tp));
++ }
++
++ if (likely(!bbr_debug_with_printk && !bbr_debug_ftrace))
++ return;
++
++ if (!sock_flag(sk, SOCK_DBG) && !is_port_match)
++ return;
++
++ if (!ctx->log && !tp->app_limited && !(bbr_flags & FLAG_DEBUG_VERBOSE))
++ return;
++
++ if (ipv4_is_loopback(inet_sk(sk)->inet_daddr) &&
++ !(bbr_flags & FLAG_DEBUG_LOOPBACK))
++ return;
++
++ snprintf(debugmsg, sizeof(debugmsg) - 1,
++ "BBR %pI4:%-5u %5u,%03u:%-7u %c "
++ "%c %2u br %2u cr %2d rtt %5ld d %2d i %5ld mrtt %d %cbw %llu "
++ "bw %llu lb %llu ib %llu qb %llu "
++ "a %u if %2u %c %c dl %u l %u al %u # %u t %u %c %c "
++ "lr %d er %d ea %d bwl %lld il %d ih %d c %d "
++ "v %d %c %u %c %s\n",
++ &inet_sk(sk)->inet_daddr, dport,
++ una / 1000, una % 1000, fack - tp->snd_una,
++ ca_states[inet_csk(sk)->icsk_ca_state],
++ bbr->debug.undo ? '@' : mode[bbr->mode],
++ tp->snd_cwnd,
++ bbr_extra_acked(sk), /* br (legacy): extra_acked */
++ rs->tx_in_flight, /* cr (legacy): tx_inflight */
++ rs->rtt_us,
++ rs->delivered,
++ rs->interval_us,
++ bbr->min_rtt_us,
++ rs->is_app_limited ? '_' : 'l',
++ bbr_rate_kbps(sk, ctx->sample_bw), /* lbw: latest sample bw */
++ bbr_rate_kbps(sk, bbr_max_bw(sk)), /* bw: max bw */
++ 0ULL, /* lb: [obsolete] */
++ 0ULL, /* ib: [obsolete] */
++ (u64)sk->sk_pacing_rate * 8 / 1000,
++ acked,
++ tcp_packets_in_flight(tp),
++ rs->is_ack_delayed ? 'd' : '.',
++ bbr->round_start ? '*' : '.',
++ tp->delivered, tp->lost,
++ tp->app_limited,
++ 0, /* #: [obsolete] */
++ ctx->target_cwnd,
++ tp->reord_seen ? 'r' : '.', /* r: reordering seen? */
++ ca_states[bbr->prev_ca_state],
++ (rs->lost + rs->delivered) > 0 ?
++ (1000 * rs->lost /
++ (rs->lost + rs->delivered)) : 0, /* lr: loss rate x1000 */
++ (rs->delivered) > 0 ?
++ (1000 * rs->delivered_ce /
++ (rs->delivered)) : 0, /* er: ECN rate x1000 */
++ 1000 * bbr->ecn_alpha >> BBR_SCALE, /* ea: ECN alpha x1000 */
++ bbr->bw_lo == ~0U ?
++ -1 : (s64)bbr_rate_kbps(sk, bbr->bw_lo), /* bwl */
++ bbr->inflight_lo, /* il */
++ bbr->inflight_hi, /* ih */
++ bbr->bw_probe_up_cnt, /* c */
++ 2, /* v: version */
++ bbr->debug.event,
++ bbr->cycle_idx,
++ ack_phase[bbr->ack_phase],
++ bbr->bw_probe_samples ? "Y" : "N");
++ debugmsg[sizeof(debugmsg) - 1] = 0;
++
++ /* printk takes a higher precedence. */
++ if (bbr_debug_with_printk)
++ printk(KERN_DEBUG "%s", debugmsg);
++
++ if (unlikely(bbr->debug.undo))
++ bbr->debug.undo = 0;
++}
++
++/* Convert a BBR bw and gain factor to a pacing rate in bytes per second. */
++static unsigned long bbr_bw_to_pacing_rate(struct sock *sk, u32 bw, int gain)
++{
++ u64 rate = bw;
++
++ rate = bbr_rate_bytes_per_sec(sk, rate, gain,
++ bbr_pacing_margin_percent);
++ rate = min_t(u64, rate, sk->sk_max_pacing_rate);
++ return rate;
++}
++
++/* Initialize pacing rate to: high_gain * init_cwnd / RTT. */
++static void bbr_init_pacing_rate_from_rtt(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ u64 bw;
++ u32 rtt_us;
++
++ if (tp->srtt_us) { /* any RTT sample yet? */
++ rtt_us = max(tp->srtt_us >> 3, 1U);
++ bbr->has_seen_rtt = 1;
++ } else { /* no RTT sample yet */
++ rtt_us = USEC_PER_MSEC; /* use nominal default RTT */
++ }
++ bw = (u64)tp->snd_cwnd * BW_UNIT;
++ do_div(bw, rtt_us);
++ sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr->params.high_gain);
++}
++
++/* Pace using current bw estimate and a gain factor. */
++static void bbr_set_pacing_rate(struct sock *sk, u32 bw, int gain)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ unsigned long rate = bbr_bw_to_pacing_rate(sk, bw, gain);
++
++ if (unlikely(!bbr->has_seen_rtt && tp->srtt_us))
++ bbr_init_pacing_rate_from_rtt(sk);
++ if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate)
++ sk->sk_pacing_rate = rate;
++}
++
++static u32 bbr_min_tso_segs(struct sock *sk)
++{
++ return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2;
++}
++
++/* Return the number of segments BBR would like in a TSO/GSO skb, given
++ * a particular max gso size as a constraint.
++ */
++static u32 bbr_tso_segs_generic(struct sock *sk, unsigned int mss_now,
++ u32 gso_max_size)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 segs, r;
++ u64 bytes;
++
++ /* Budget a TSO/GSO burst size allowance based on bw (pacing_rate). */
++ bytes = sk->sk_pacing_rate >> sk->sk_pacing_shift;
++
++ /* Budget a TSO/GSO burst size allowance based on min_rtt. For every
++ * K = 2^tso_rtt_shift microseconds of min_rtt, halve the burst.
++ * The min_rtt-based burst allowance is: 64 KBytes / 2^(min_rtt/K)
++ */
++ if (bbr->params.tso_rtt_shift) {
++ r = bbr->min_rtt_us >> bbr->params.tso_rtt_shift;
++ if (r < BITS_PER_TYPE(u32)) /* prevent undefined behavior */
++ bytes += GSO_MAX_SIZE >> r;
++ }
++
++ bytes = min_t(u32, bytes, gso_max_size - 1 - MAX_TCP_HEADER);
++ segs = max_t(u32, bytes / mss_now, bbr_min_tso_segs(sk));
++ return segs;
++}
++
++/* Custom tcp_tso_autosize() for BBR, used at transmit time to cap skb size. */
++static u32 bbr_tso_segs(struct sock *sk, unsigned int mss_now)
++{
++ return bbr_tso_segs_generic(sk, mss_now, sk->sk_gso_max_size);
++}
++
++/* Like bbr_tso_segs(), using mss_cache, ignoring driver's sk_gso_max_size. */
++static u32 bbr_tso_segs_goal(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++
++ return bbr_tso_segs_generic(sk, tp->mss_cache, GSO_MAX_SIZE);
++}
++
++/* Save "last known good" cwnd so we can restore it after losses or PROBE_RTT */
++static void bbr_save_cwnd(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT)
++ bbr->prior_cwnd = tp->snd_cwnd; /* this cwnd is good enough */
++ else /* loss recovery or BBR_PROBE_RTT have temporarily cut cwnd */
++ bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd);
++}
++
++static void bbr_cwnd_event(struct sock *sk, enum tcp_ca_event event)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (event == CA_EVENT_TX_START && tp->app_limited) {
++ bbr->idle_restart = 1;
++ bbr->ack_epoch_mstamp = tp->tcp_mstamp;
++ bbr->ack_epoch_acked = 0;
++ /* Avoid pointless buffer overflows: pace at est. bw if we don't
++ * need more speed (we're restarting from idle and app-limited).
++ */
++ if (bbr->mode == BBR_PROBE_BW)
++ bbr_set_pacing_rate(sk, bbr_bw(sk), BBR_UNIT);
++ else if (bbr->mode == BBR_PROBE_RTT)
++ bbr_check_probe_rtt_done(sk);
++ } else if ((event == CA_EVENT_ECN_IS_CE ||
++ event == CA_EVENT_ECN_NO_CE) &&
++ bbr_ecn_enable &&
++ bbr->params.precise_ece_ack) {
++ u32 state = bbr->ce_state;
++ dctcp_ece_ack_update(sk, event, &bbr->prior_rcv_nxt, &state);
++ bbr->ce_state = state;
++ if (tp->fast_ack_mode == 2 && event == CA_EVENT_ECN_IS_CE)
++ tcp_enter_quickack_mode(sk, TCP_MAX_QUICKACKS);
++ }
++}
++
++/* Calculate bdp based on min RTT and the estimated bottleneck bandwidth:
++ *
++ * bdp = ceil(bw * min_rtt * gain)
++ *
++ * The key factor, gain, controls the amount of queue. While a small gain
++ * builds a smaller queue, it becomes more vulnerable to noise in RTT
++ * measurements (e.g., delayed ACKs or other ACK compression effects). This
++ * noise may cause BBR to under-estimate the rate.
++ */
++static u32 bbr_bdp(struct sock *sk, u32 bw, int gain)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 bdp;
++ u64 w;
++
++ /* If we've never had a valid RTT sample, cap cwnd at the initial
++ * default. This should only happen when the connection is not using TCP
++ * timestamps and has retransmitted all of the SYN/SYNACK/data packets
++ * ACKed so far. In this case, an RTO can cut cwnd to 1, in which
++ * case we need to slow-start up toward something safe: initial cwnd.
++ */
++ if (unlikely(bbr->min_rtt_us == ~0U)) /* no valid RTT samples yet? */
++ return bbr->init_cwnd; /* be safe: cap at initial cwnd */
++
++ w = (u64)bw * bbr->min_rtt_us;
++
++ /* Apply a gain to the given value, remove the BW_SCALE shift, and
++ * round the value up to avoid a negative feedback loop.
++ */
++ bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT;
++
++ return bdp;
++}
++
++/* To achieve full performance in high-speed paths, we budget enough cwnd to
++ * fit full-sized skbs in-flight on both end hosts to fully utilize the path:
++ * - one skb in sending host Qdisc,
++ * - one skb in sending host TSO/GSO engine
++ * - one skb being received by receiver host LRO/GRO/delayed-ACK engine
++ * Don't worry, at low rates (bbr_min_tso_rate) this won't bloat cwnd because
++ * in such cases tso_segs_goal is 1. The minimum cwnd is 4 packets,
++ * which allows 2 outstanding 2-packet sequences, to try to keep pipe
++ * full even with ACK-every-other-packet delayed ACKs.
++ */
++static u32 bbr_quantization_budget(struct sock *sk, u32 cwnd)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 tso_segs_goal;
++
++ tso_segs_goal = 3 * bbr_tso_segs_goal(sk);
++
++ /* Allow enough full-sized skbs in flight to utilize end systems. */
++ if (bbr->params.cwnd_tso_budget == 1) {
++ cwnd = max_t(u32, cwnd, tso_segs_goal);
++ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target);
++ } else {
++ cwnd += tso_segs_goal;
++ cwnd = (cwnd + 1) & ~1U;
++ }
++ /* Ensure gain cycling gets inflight above BDP even for small BDPs. */
++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP)
++ cwnd += 2;
++
++ return cwnd;
++}
++
++/* Find inflight based on min RTT and the estimated bottleneck bandwidth. */
++static u32 bbr_inflight(struct sock *sk, u32 bw, int gain)
++{
++ u32 inflight;
++
++ inflight = bbr_bdp(sk, bw, gain);
++ inflight = bbr_quantization_budget(sk, inflight);
++
++ return inflight;
++}
++
++/* With pacing at lower layers, there's often less data "in the network" than
++ * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq),
++ * we often have several skbs queued in the pacing layer with a pre-scheduled
++ * earliest departure time (EDT). BBR adapts its pacing rate based on the
++ * inflight level that it estimates has already been "baked in" by previous
++ * departure time decisions. We calculate a rough estimate of the number of our
++ * packets that might be in the network at the earliest departure time for the
++ * next skb scheduled:
++ * in_network_at_edt = inflight_at_edt - (EDT - now) * bw
++ * If we're increasing inflight, then we want to know if the transmit of the
++ * EDT skb will push inflight above the target, so inflight_at_edt includes
++ * bbr_tso_segs_goal() from the skb departing at EDT. If decreasing inflight,
++ * then estimate if inflight will sink too low just before the EDT transmit.
++ */
++static u32 bbr_packets_in_net_at_edt(struct sock *sk, u32 inflight_now)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ u64 now_ns, edt_ns, interval_us;
++ u32 interval_delivered, inflight_at_edt;
++
++ now_ns = tp->tcp_clock_cache;
++ edt_ns = max(tp->tcp_wstamp_ns, now_ns);
++ interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC);
++ interval_delivered = (u64)bbr_bw(sk) * interval_us >> BW_SCALE;
++ inflight_at_edt = inflight_now;
++ if (bbr->pacing_gain > BBR_UNIT) /* increasing inflight */
++ inflight_at_edt += bbr_tso_segs_goal(sk); /* include EDT skb */
++ if (interval_delivered >= inflight_at_edt)
++ return 0;
++ return inflight_at_edt - interval_delivered;
++}
++
++/* Find the cwnd increment based on estimate of ack aggregation */
++static u32 bbr_ack_aggregation_cwnd(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 max_aggr_cwnd, aggr_cwnd = 0;
++
++ if (bbr->params.extra_acked_gain &&
++ (bbr_full_bw_reached(sk) || bbr->params.extra_acked_in_startup)) {
++ max_aggr_cwnd = ((u64)bbr_bw(sk) * bbr_extra_acked_max_us)
++ / BW_UNIT;
++ aggr_cwnd = (bbr->params.extra_acked_gain * bbr_extra_acked(sk))
++ >> BBR_SCALE;
++ aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd);
++ }
++
++ return aggr_cwnd;
++}
++
++/* Returns the cwnd for PROBE_RTT mode. */
++static u32 bbr_probe_rtt_cwnd(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (bbr->params.probe_rtt_cwnd_gain == 0)
++ return bbr->params.cwnd_min_target;
++ return max_t(u32, bbr->params.cwnd_min_target,
++ bbr_bdp(sk, bbr_bw(sk), bbr->params.probe_rtt_cwnd_gain));
++}
++
++/* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss
++ * has drawn us down below target), or snap down to target if we're above it.
++ */
++static void bbr_set_cwnd(struct sock *sk, const struct rate_sample *rs,
++ u32 acked, u32 bw, int gain, u32 cwnd,
++ struct bbr_context *ctx)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 target_cwnd = 0, prev_cwnd = tp->snd_cwnd, max_probe;
++
++ if (!acked)
++ goto done; /* no packet fully ACKed; just apply caps */
++
++ target_cwnd = bbr_bdp(sk, bw, gain);
++
++ /* Increment the cwnd to account for excess ACKed data that seems
++ * due to aggregation (of data and/or ACKs) visible in the ACK stream.
++ */
++ target_cwnd += bbr_ack_aggregation_cwnd(sk);
++ target_cwnd = bbr_quantization_budget(sk, target_cwnd);
++
++ /* If we're below target cwnd, slow start cwnd toward target cwnd. */
++ bbr->debug.target_cwnd = target_cwnd;
++
++ /* Update cwnd and enable fast path if cwnd reaches target_cwnd. */
++ bbr->try_fast_path = 0;
++ if (bbr_full_bw_reached(sk)) { /* only cut cwnd if we filled the pipe */
++ cwnd += acked;
++ if (cwnd >= target_cwnd) {
++ cwnd = target_cwnd;
++ bbr->try_fast_path = 1;
++ }
++ } else if (cwnd < target_cwnd || cwnd < 2 * bbr->init_cwnd) {
++ cwnd += acked;
++ } else {
++ bbr->try_fast_path = 1;
++ }
++
++ /* When growing cwnd, don't grow beyond twice what we just probed. */
++ if (bbr->params.usage_based_cwnd) {
++ max_probe = max(2 * tp->max_packets_out, tp->snd_cwnd);
++ cwnd = min(cwnd, max_probe);
++ }
++
++ cwnd = max_t(u32, cwnd, bbr->params.cwnd_min_target);
++done:
++ tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp); /* apply global cap */
++ if (bbr->mode == BBR_PROBE_RTT) /* drain queue, refresh min_rtt */
++ tp->snd_cwnd = min_t(u32, tp->snd_cwnd, bbr_probe_rtt_cwnd(sk));
++
++ ctx->target_cwnd = target_cwnd;
++ ctx->log = (tp->snd_cwnd != prev_cwnd);
++}
++
++/* See if we have reached next round trip */
++static void bbr_update_round_start(struct sock *sk,
++ const struct rate_sample *rs, struct bbr_context *ctx)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr->round_start = 0;
++
++ /* See if we've reached the next RTT */
++ if (rs->interval_us > 0 &&
++ !before(rs->prior_delivered, bbr->next_rtt_delivered)) {
++ bbr->next_rtt_delivered = tp->delivered;
++ bbr->round_start = 1;
++ }
++}
++
++/* Calculate the bandwidth based on how fast packets are delivered */
++static void bbr_calculate_bw_sample(struct sock *sk,
++ const struct rate_sample *rs, struct bbr_context *ctx)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u64 bw = 0;
++
++ /* Divide delivered by the interval to find a (lower bound) bottleneck
++ * bandwidth sample. Delivered is in packets and interval_us in uS and
++ * ratio will be <<1 for most connections. So delivered is first scaled.
++ * Round up to allow growth at low rates, even with integer division.
++ */
++ if (rs->interval_us > 0) {
++ if (WARN_ONCE(rs->delivered < 0,
++ "negative delivered: %d interval_us: %ld\n",
++ rs->delivered, rs->interval_us))
++ return;
++
++ bw = DIV_ROUND_UP_ULL((u64)rs->delivered * BW_UNIT, rs->interval_us);
++ }
++
++ ctx->sample_bw = bw;
++ bbr->debug.rs_bw = bw;
++}
++
++/* Estimates the windowed max degree of ack aggregation.
++ * This is used to provision extra in-flight data to keep sending during
++ * inter-ACK silences.
++ *
++ * Degree of ack aggregation is estimated as extra data acked beyond expected.
++ *
++ * max_extra_acked = "maximum recent excess data ACKed beyond max_bw * interval"
++ * cwnd += max_extra_acked
++ *
++ * Max extra_acked is clamped by cwnd and bw * bbr_extra_acked_max_us (100 ms).
++ * Max filter is an approximate sliding window of 5-10 (packet timed) round
++ * trips for non-startup phase, and 1-2 round trips for startup.
++ */
++static void bbr_update_ack_aggregation(struct sock *sk,
++ const struct rate_sample *rs)
++{
++ u32 epoch_us, expected_acked, extra_acked;
++ struct bbr *bbr = inet_csk_ca(sk);
++ struct tcp_sock *tp = tcp_sk(sk);
++ u32 extra_acked_win_rtts_thresh = bbr->params.extra_acked_win_rtts;
++
++ if (!bbr->params.extra_acked_gain || rs->acked_sacked <= 0 ||
++ rs->delivered < 0 || rs->interval_us <= 0)
++ return;
++
++ if (bbr->round_start) {
++ bbr->extra_acked_win_rtts = min(0x1F,
++ bbr->extra_acked_win_rtts + 1);
++ if (bbr->params.extra_acked_in_startup &&
++ !bbr_full_bw_reached(sk))
++ extra_acked_win_rtts_thresh = 1;
++ if (bbr->extra_acked_win_rtts >=
++ extra_acked_win_rtts_thresh) {
++ bbr->extra_acked_win_rtts = 0;
++ bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ?
++ 0 : 1;
++ bbr->extra_acked[bbr->extra_acked_win_idx] = 0;
++ }
++ }
++
++ /* Compute how many packets we expected to be delivered over epoch. */
++ epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp,
++ bbr->ack_epoch_mstamp);
++ expected_acked = ((u64)bbr_bw(sk) * epoch_us) / BW_UNIT;
++
++ /* Reset the aggregation epoch if ACK rate is below expected rate or
++ * significantly large no. of ack received since epoch (potentially
++ * quite old epoch).
++ */
++ if (bbr->ack_epoch_acked <= expected_acked ||
++ (bbr->ack_epoch_acked + rs->acked_sacked >=
++ bbr_ack_epoch_acked_reset_thresh)) {
++ bbr->ack_epoch_acked = 0;
++ bbr->ack_epoch_mstamp = tp->delivered_mstamp;
++ expected_acked = 0;
++ }
++
++ /* Compute excess data delivered, beyond what was expected. */
++ bbr->ack_epoch_acked = min_t(u32, 0xFFFFF,
++ bbr->ack_epoch_acked + rs->acked_sacked);
++ extra_acked = bbr->ack_epoch_acked - expected_acked;
++ extra_acked = min(extra_acked, tp->snd_cwnd);
++ if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx])
++ bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked;
++}
++
++/* Estimate when the pipe is full, using the change in delivery rate: BBR
++ * estimates that STARTUP filled the pipe if the estimated bw hasn't changed by
++ * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited
++ * rounds. Why 3 rounds: 1: rwin autotuning grows the rwin, 2: we fill the
++ * higher rwin, 3: we get higher delivery rate samples. Or transient
++ * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar
++ * design goal, but uses delay and inter-ACK spacing instead of bandwidth.
++ */
++static void bbr_check_full_bw_reached(struct sock *sk,
++ const struct rate_sample *rs)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 bw_thresh;
++
++ if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited)
++ return;
++
++ bw_thresh = (u64)bbr->full_bw * bbr->params.full_bw_thresh >> BBR_SCALE;
++ if (bbr_max_bw(sk) >= bw_thresh) {
++ bbr->full_bw = bbr_max_bw(sk);
++ bbr->full_bw_cnt = 0;
++ return;
++ }
++ ++bbr->full_bw_cnt;
++ bbr->full_bw_reached = bbr->full_bw_cnt >= bbr->params.full_bw_cnt;
++}
++
++/* If pipe is probably full, drain the queue and then enter steady-state. */
++static bool bbr_check_drain(struct sock *sk, const struct rate_sample *rs,
++ struct bbr_context *ctx)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) {
++ bbr->mode = BBR_DRAIN; /* drain queue we created */
++ tcp_sk(sk)->snd_ssthresh =
++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT);
++ bbr2_reset_congestion_signals(sk);
++ } /* fall through to check if in-flight is already small: */
++ if (bbr->mode == BBR_DRAIN &&
++ bbr_packets_in_net_at_edt(sk, tcp_packets_in_flight(tcp_sk(sk))) <=
++ bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT))
++ return true; /* exiting DRAIN now */
++ return false;
++}
++
++static void bbr_check_probe_rtt_done(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (!(bbr->probe_rtt_done_stamp &&
++ after(tcp_jiffies32, bbr->probe_rtt_done_stamp)))
++ return;
++
++ bbr->probe_rtt_min_stamp = tcp_jiffies32; /* schedule next PROBE_RTT */
++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd);
++ bbr2_exit_probe_rtt(sk);
++}
++
++/* The goal of PROBE_RTT mode is to have BBR flows cooperatively and
++ * periodically drain the bottleneck queue, to converge to measure the true
++ * min_rtt (unloaded propagation delay). This allows the flows to keep queues
++ * small (reducing queuing delay and packet loss) and achieve fairness among
++ * BBR flows.
++ *
++ * The min_rtt filter window is 10 seconds. When the min_rtt estimate expires,
++ * we enter PROBE_RTT mode and cap the cwnd at bbr_cwnd_min_target=4 packets.
++ * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed
++ * round trip elapsed with that flight size <= 4, we leave PROBE_RTT mode and
++ * re-enter the previous mode. BBR uses 200ms to approximately bound the
++ * performance penalty of PROBE_RTT's cwnd capping to roughly 2% (200ms/10s).
++ *
++ * Note that flows need only pay 2% if they are busy sending over the last 10
++ * seconds. Interactive applications (e.g., Web, RPCs, video chunks) often have
++ * natural silences or low-rate periods within 10 seconds where the rate is low
++ * enough for long enough to drain its queue in the bottleneck. We pick up
++ * these min RTT measurements opportunistically with our min_rtt filter. :-)
++ */
++static void bbr_update_min_rtt(struct sock *sk, const struct rate_sample *rs)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ bool probe_rtt_expired, min_rtt_expired;
++ u32 expire;
++
++ /* Track min RTT in probe_rtt_win_ms to time next PROBE_RTT state. */
++ expire = bbr->probe_rtt_min_stamp +
++ msecs_to_jiffies(bbr->params.probe_rtt_win_ms);
++ probe_rtt_expired = after(tcp_jiffies32, expire);
++ if (rs->rtt_us >= 0 &&
++ (rs->rtt_us <= bbr->probe_rtt_min_us ||
++ (probe_rtt_expired && !rs->is_ack_delayed))) {
++ bbr->probe_rtt_min_us = rs->rtt_us;
++ bbr->probe_rtt_min_stamp = tcp_jiffies32;
++ }
++ /* Track min RTT seen in the min_rtt_win_sec filter window: */
++ expire = bbr->min_rtt_stamp + bbr->params.min_rtt_win_sec * HZ;
++ min_rtt_expired = after(tcp_jiffies32, expire);
++ if (bbr->probe_rtt_min_us <= bbr->min_rtt_us ||
++ min_rtt_expired) {
++ bbr->min_rtt_us = bbr->probe_rtt_min_us;
++ bbr->min_rtt_stamp = bbr->probe_rtt_min_stamp;
++ }
++
++ if (bbr->params.probe_rtt_mode_ms > 0 && probe_rtt_expired &&
++ !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) {
++ bbr->mode = BBR_PROBE_RTT; /* dip, drain queue */
++ bbr_save_cwnd(sk); /* note cwnd so we can restore it */
++ bbr->probe_rtt_done_stamp = 0;
++ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING;
++ bbr->next_rtt_delivered = tp->delivered;
++ }
++
++ if (bbr->mode == BBR_PROBE_RTT) {
++ /* Ignore low rate samples during this mode. */
++ tp->app_limited =
++ (tp->delivered + tcp_packets_in_flight(tp)) ? : 1;
++ /* Maintain min packets in flight for max(200 ms, 1 round). */
++ if (!bbr->probe_rtt_done_stamp &&
++ tcp_packets_in_flight(tp) <= bbr_probe_rtt_cwnd(sk)) {
++ bbr->probe_rtt_done_stamp = tcp_jiffies32 +
++ msecs_to_jiffies(bbr->params.probe_rtt_mode_ms);
++ bbr->probe_rtt_round_done = 0;
++ bbr->next_rtt_delivered = tp->delivered;
++ } else if (bbr->probe_rtt_done_stamp) {
++ if (bbr->round_start)
++ bbr->probe_rtt_round_done = 1;
++ if (bbr->probe_rtt_round_done)
++ bbr_check_probe_rtt_done(sk);
++ }
++ }
++ /* Restart after idle ends only once we process a new S/ACK for data */
++ if (rs->delivered > 0)
++ bbr->idle_restart = 0;
++}
++
++static void bbr_update_gains(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ switch (bbr->mode) {
++ case BBR_STARTUP:
++ bbr->pacing_gain = bbr->params.high_gain;
++ bbr->cwnd_gain = bbr->params.startup_cwnd_gain;
++ break;
++ case BBR_DRAIN:
++ bbr->pacing_gain = bbr->params.drain_gain; /* slow, to drain */
++ bbr->cwnd_gain = bbr->params.startup_cwnd_gain; /* keep cwnd */
++ break;
++ case BBR_PROBE_BW:
++ bbr->pacing_gain = bbr->params.pacing_gain[bbr->cycle_idx];
++ bbr->cwnd_gain = bbr->params.cwnd_gain;
++ break;
++ case BBR_PROBE_RTT:
++ bbr->pacing_gain = BBR_UNIT;
++ bbr->cwnd_gain = BBR_UNIT;
++ break;
++ default:
++ WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode);
++ break;
++ }
++}
++
++static void bbr_init(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ int i;
++
++ WARN_ON_ONCE(tp->snd_cwnd >= bbr_cwnd_warn_val);
++
++ bbr->initialized = 1;
++ bbr->params.high_gain = min(0x7FF, bbr_high_gain);
++ bbr->params.drain_gain = min(0x3FF, bbr_drain_gain);
++ bbr->params.startup_cwnd_gain = min(0x7FF, bbr_startup_cwnd_gain);
++ bbr->params.cwnd_gain = min(0x7FF, bbr_cwnd_gain);
++ bbr->params.cwnd_tso_budget = min(0x1U, bbr_cwnd_tso_budget);
++ bbr->params.cwnd_min_target = min(0xFU, bbr_cwnd_min_target);
++ bbr->params.min_rtt_win_sec = min(0x1FU, bbr_min_rtt_win_sec);
++ bbr->params.probe_rtt_mode_ms = min(0x1FFU, bbr_probe_rtt_mode_ms);
++ bbr->params.full_bw_cnt = min(0x7U, bbr_full_bw_cnt);
++ bbr->params.full_bw_thresh = min(0x3FFU, bbr_full_bw_thresh);
++ bbr->params.extra_acked_gain = min(0x7FF, bbr_extra_acked_gain);
++ bbr->params.extra_acked_win_rtts = min(0x1FU, bbr_extra_acked_win_rtts);
++ bbr->params.drain_to_target = bbr_drain_to_target ? 1 : 0;
++ bbr->params.precise_ece_ack = bbr_precise_ece_ack ? 1 : 0;
++ bbr->params.extra_acked_in_startup = bbr_extra_acked_in_startup ? 1 : 0;
++ bbr->params.probe_rtt_cwnd_gain = min(0xFFU, bbr_probe_rtt_cwnd_gain);
++ bbr->params.probe_rtt_win_ms =
++ min(0x3FFFU,
++ min_t(u32, bbr_probe_rtt_win_ms,
++ bbr->params.min_rtt_win_sec * MSEC_PER_SEC));
++ for (i = 0; i < CYCLE_LEN; i++)
++ bbr->params.pacing_gain[i] = min(0x3FF, bbr_pacing_gain[i]);
++ bbr->params.usage_based_cwnd = bbr_usage_based_cwnd ? 1 : 0;
++ bbr->params.tso_rtt_shift = min(0xFU, bbr_tso_rtt_shift);
++
++ bbr->debug.snd_isn = tp->snd_una;
++ bbr->debug.target_cwnd = 0;
++ bbr->debug.undo = 0;
++
++ bbr->init_cwnd = min(0x7FU, tp->snd_cwnd);
++ bbr->prior_cwnd = tp->prior_cwnd;
++ tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
++ bbr->next_rtt_delivered = 0;
++ bbr->prev_ca_state = TCP_CA_Open;
++ bbr->packet_conservation = 0;
++
++ bbr->probe_rtt_done_stamp = 0;
++ bbr->probe_rtt_round_done = 0;
++ bbr->probe_rtt_min_us = tcp_min_rtt(tp);
++ bbr->probe_rtt_min_stamp = tcp_jiffies32;
++ bbr->min_rtt_us = tcp_min_rtt(tp);
++ bbr->min_rtt_stamp = tcp_jiffies32;
++
++ bbr->has_seen_rtt = 0;
++ bbr_init_pacing_rate_from_rtt(sk);
++
++ bbr->round_start = 0;
++ bbr->idle_restart = 0;
++ bbr->full_bw_reached = 0;
++ bbr->full_bw = 0;
++ bbr->full_bw_cnt = 0;
++ bbr->cycle_mstamp = 0;
++ bbr->cycle_idx = 0;
++ bbr->mode = BBR_STARTUP;
++ bbr->debug.rs_bw = 0;
++
++ bbr->ack_epoch_mstamp = tp->tcp_mstamp;
++ bbr->ack_epoch_acked = 0;
++ bbr->extra_acked_win_rtts = 0;
++ bbr->extra_acked_win_idx = 0;
++ bbr->extra_acked[0] = 0;
++ bbr->extra_acked[1] = 0;
++
++ bbr->ce_state = 0;
++ bbr->prior_rcv_nxt = tp->rcv_nxt;
++ bbr->try_fast_path = 0;
++
++ cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED);
++}
++
++static u32 bbr_sndbuf_expand(struct sock *sk)
++{
++ /* Provision 3 * cwnd since BBR may slow-start even during recovery. */
++ return 3;
++}
++
++/* __________________________________________________________________________
++ *
++ * Functions new to BBR v2 ("bbr") congestion control are below here.
++ * __________________________________________________________________________
++ */
++
++/* Incorporate a new bw sample into the current window of our max filter. */
++static void bbr2_take_bw_hi_sample(struct sock *sk, u32 bw)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr->bw_hi[1] = max(bw, bbr->bw_hi[1]);
++}
++
++/* Keep max of last 1-2 cycles. Each PROBE_BW cycle, flip filter window. */
++static void bbr2_advance_bw_hi_filter(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (!bbr->bw_hi[1])
++ return; /* no samples in this window; remember old window */
++ bbr->bw_hi[0] = bbr->bw_hi[1];
++ bbr->bw_hi[1] = 0;
++}
++
++/* How much do we want in flight? Our BDP, unless congestion cut cwnd. */
++static u32 bbr2_target_inflight(struct sock *sk)
++{
++ u32 bdp = bbr_inflight(sk, bbr_bw(sk), BBR_UNIT);
++
++ return min(bdp, tcp_sk(sk)->snd_cwnd);
++}
++
++static bool bbr2_is_probing_bandwidth(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ return (bbr->mode == BBR_STARTUP) ||
++ (bbr->mode == BBR_PROBE_BW &&
++ (bbr->cycle_idx == BBR_BW_PROBE_REFILL ||
++ bbr->cycle_idx == BBR_BW_PROBE_UP));
++}
++
++/* Has the given amount of time elapsed since we marked the phase start? */
++static bool bbr2_has_elapsed_in_phase(const struct sock *sk, u32 interval_us)
++{
++ const struct tcp_sock *tp = tcp_sk(sk);
++ const struct bbr *bbr = inet_csk_ca(sk);
++
++ return tcp_stamp_us_delta(tp->tcp_mstamp,
++ bbr->cycle_mstamp + interval_us) > 0;
++}
++
++static void bbr2_handle_queue_too_high_in_startup(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr->full_bw_reached = 1;
++ bbr->inflight_hi = bbr_inflight(sk, bbr_max_bw(sk), BBR_UNIT);
++}
++
++/* Exit STARTUP upon N consecutive rounds with ECN mark rate > ecn_thresh. */
++static void bbr2_check_ecn_too_high_in_startup(struct sock *sk, u32 ce_ratio)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (bbr_full_bw_reached(sk) || !bbr->ecn_eligible ||
++ !bbr->params.full_ecn_cnt || !bbr->params.ecn_thresh)
++ return;
++
++ if (ce_ratio >= bbr->params.ecn_thresh)
++ bbr->startup_ecn_rounds++;
++ else
++ bbr->startup_ecn_rounds = 0;
++
++ if (bbr->startup_ecn_rounds >= bbr->params.full_ecn_cnt) {
++ bbr->debug.event = 'E'; /* ECN caused STARTUP exit */
++ bbr2_handle_queue_too_high_in_startup(sk);
++ return;
++ }
++}
++
++static void bbr2_update_ecn_alpha(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ s32 delivered, delivered_ce;
++ u64 alpha, ce_ratio;
++ u32 gain;
++
++ if (bbr->params.ecn_factor == 0)
++ return;
++
++ delivered = tp->delivered - bbr->alpha_last_delivered;
++ delivered_ce = tp->delivered_ce - bbr->alpha_last_delivered_ce;
++
++ if (delivered == 0 || /* avoid divide by zero */
++ WARN_ON_ONCE(delivered < 0 || delivered_ce < 0)) /* backwards? */
++ return;
++
++ /* See if we should use ECN sender logic for this connection. */
++ if (!bbr->ecn_eligible && bbr_ecn_enable &&
++ (bbr->min_rtt_us <= bbr->params.ecn_max_rtt_us ||
++ !bbr->params.ecn_max_rtt_us))
++ bbr->ecn_eligible = 1;
++
++ ce_ratio = (u64)delivered_ce << BBR_SCALE;
++ do_div(ce_ratio, delivered);
++ gain = bbr->params.ecn_alpha_gain;
++ alpha = ((BBR_UNIT - gain) * bbr->ecn_alpha) >> BBR_SCALE;
++ alpha += (gain * ce_ratio) >> BBR_SCALE;
++ bbr->ecn_alpha = min_t(u32, alpha, BBR_UNIT);
++
++ bbr->alpha_last_delivered = tp->delivered;
++ bbr->alpha_last_delivered_ce = tp->delivered_ce;
++
++ bbr2_check_ecn_too_high_in_startup(sk, ce_ratio);
++}
++
++/* Each round trip of BBR_BW_PROBE_UP, double volume of probing data. */
++static void bbr2_raise_inflight_hi_slope(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 growth_this_round, cnt;
++
++ /* Calculate "slope": packets S/Acked per inflight_hi increment. */
++ growth_this_round = 1 << bbr->bw_probe_up_rounds;
++ bbr->bw_probe_up_rounds = min(bbr->bw_probe_up_rounds + 1, 30);
++ cnt = tp->snd_cwnd / growth_this_round;
++ cnt = max(cnt, 1U);
++ bbr->bw_probe_up_cnt = cnt;
++ bbr->debug.event = 'G'; /* Grow inflight_hi slope */
++}
++
++/* In BBR_BW_PROBE_UP, not seeing high loss/ECN/queue, so raise inflight_hi. */
++static void bbr2_probe_inflight_hi_upward(struct sock *sk,
++ const struct rate_sample *rs)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 delta;
++
++ if (!tp->is_cwnd_limited || tp->snd_cwnd < bbr->inflight_hi) {
++ bbr->bw_probe_up_acks = 0; /* don't accmulate unused credits */
++ return; /* not fully using inflight_hi, so don't grow it */
++ }
++
++ /* For each bw_probe_up_cnt packets ACKed, increase inflight_hi by 1. */
++ bbr->bw_probe_up_acks += rs->acked_sacked;
++ if (bbr->bw_probe_up_acks >= bbr->bw_probe_up_cnt) {
++ delta = bbr->bw_probe_up_acks / bbr->bw_probe_up_cnt;
++ bbr->bw_probe_up_acks -= delta * bbr->bw_probe_up_cnt;
++ bbr->inflight_hi += delta;
++ bbr->debug.event = 'I'; /* Increment inflight_hi */
++ }
++
++ if (bbr->round_start)
++ bbr2_raise_inflight_hi_slope(sk);
++}
++
++/* Does loss/ECN rate for this sample say inflight is "too high"?
++ * This is used by both the bbr_check_loss_too_high_in_startup() function,
++ * which can be used in either v1 or v2, and the PROBE_UP phase of v2, which
++ * uses it to notice when loss/ECN rates suggest inflight is too high.
++ */
++static bool bbr2_is_inflight_too_high(const struct sock *sk,
++ const struct rate_sample *rs)
++{
++ const struct bbr *bbr = inet_csk_ca(sk);
++ u32 loss_thresh, ecn_thresh;
++
++ if (rs->lost > 0 && rs->tx_in_flight) {
++ loss_thresh = (u64)rs->tx_in_flight * bbr->params.loss_thresh >>
++ BBR_SCALE;
++ if (rs->lost > loss_thresh)
++ return true;
++ }
++
++ if (rs->delivered_ce > 0 && rs->delivered > 0 &&
++ bbr->ecn_eligible && bbr->params.ecn_thresh) {
++ ecn_thresh = (u64)rs->delivered * bbr->params.ecn_thresh >>
++ BBR_SCALE;
++ if (rs->delivered_ce >= ecn_thresh)
++ return true;
++ }
++
++ return false;
++}
++
++/* Calculate the tx_in_flight level that corresponded to excessive loss.
++ * We find "lost_prefix" segs of the skb where loss rate went too high,
++ * by solving for "lost_prefix" in the following equation:
++ * lost / inflight >= loss_thresh
++ * (lost_prev + lost_prefix) / (inflight_prev + lost_prefix) >= loss_thresh
++ * Then we take that equation, convert it to fixed point, and
++ * round up to the nearest packet.
++ */
++static u32 bbr2_inflight_hi_from_lost_skb(const struct sock *sk,
++ const struct rate_sample *rs,
++ const struct sk_buff *skb)
++{
++ const struct bbr *bbr = inet_csk_ca(sk);
++ u32 loss_thresh = bbr->params.loss_thresh;
++ u32 pcount, divisor, inflight_hi;
++ s32 inflight_prev, lost_prev;
++ u64 loss_budget, lost_prefix;
++
++ pcount = tcp_skb_pcount(skb);
++
++ /* How much data was in flight before this skb? */
++ inflight_prev = rs->tx_in_flight - pcount;
++ if (WARN_ONCE(inflight_prev < 0,
++ "tx_in_flight: %u pcount: %u reneg: %u",
++ rs->tx_in_flight, pcount, tcp_sk(sk)->is_sack_reneg))
++ return ~0U;
++
++ /* How much inflight data was marked lost before this skb? */
++ lost_prev = rs->lost - pcount;
++ if (WARN_ON_ONCE(lost_prev < 0))
++ return ~0U;
++
++ /* At what prefix of this lost skb did losss rate exceed loss_thresh? */
++ loss_budget = (u64)inflight_prev * loss_thresh + BBR_UNIT - 1;
++ loss_budget >>= BBR_SCALE;
++ if (lost_prev >= loss_budget) {
++ lost_prefix = 0; /* previous losses crossed loss_thresh */
++ } else {
++ lost_prefix = loss_budget - lost_prev;
++ lost_prefix <<= BBR_SCALE;
++ divisor = BBR_UNIT - loss_thresh;
++ if (WARN_ON_ONCE(!divisor)) /* loss_thresh is 8 bits */
++ return ~0U;
++ do_div(lost_prefix, divisor);
++ }
++
++ inflight_hi = inflight_prev + lost_prefix;
++ return inflight_hi;
++}
++
++/* If loss/ECN rates during probing indicated we may have overfilled a
++ * buffer, return an operating point that tries to leave unutilized headroom in
++ * the path for other flows, for fairness convergence and lower RTTs and loss.
++ */
++static u32 bbr2_inflight_with_headroom(const struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 headroom, headroom_fraction;
++
++ if (bbr->inflight_hi == ~0U)
++ return ~0U;
++
++ headroom_fraction = bbr->params.inflight_headroom;
++ headroom = ((u64)bbr->inflight_hi * headroom_fraction) >> BBR_SCALE;
++ headroom = max(headroom, 1U);
++ return max_t(s32, bbr->inflight_hi - headroom,
++ bbr->params.cwnd_min_target);
++}
++
++/* Bound cwnd to a sensible level, based on our current probing state
++ * machine phase and model of a good inflight level (inflight_lo, inflight_hi).
++ */
++static void bbr2_bound_cwnd_for_inflight_model(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 cap;
++
++ /* tcp_rcv_synsent_state_process() currently calls tcp_ack()
++ * and thus cong_control() without first initializing us(!).
++ */
++ if (!bbr->initialized)
++ return;
++
++ cap = ~0U;
++ if (bbr->mode == BBR_PROBE_BW &&
++ bbr->cycle_idx != BBR_BW_PROBE_CRUISE) {
++ /* Probe to see if more packets fit in the path. */
++ cap = bbr->inflight_hi;
++ } else {
++ if (bbr->mode == BBR_PROBE_RTT ||
++ (bbr->mode == BBR_PROBE_BW &&
++ bbr->cycle_idx == BBR_BW_PROBE_CRUISE))
++ cap = bbr2_inflight_with_headroom(sk);
++ }
++ /* Adapt to any loss/ECN since our last bw probe. */
++ cap = min(cap, bbr->inflight_lo);
++
++ cap = max_t(u32, cap, bbr->params.cwnd_min_target);
++ tp->snd_cwnd = min(cap, tp->snd_cwnd);
++}
++
++/* Estimate a short-term lower bound on the capacity available now, based
++ * on measurements of the current delivery process and recent history. When we
++ * are seeing loss/ECN at times when we are not probing bw, then conservatively
++ * move toward flow balance by multiplicatively cutting our short-term
++ * estimated safe rate and volume of data (bw_lo and inflight_lo). We use a
++ * multiplicative decrease in order to converge to a lower capacity in time
++ * logarithmic in the magnitude of the decrease.
++ *
++ * However, we do not cut our short-term estimates lower than the current rate
++ * and volume of delivered data from this round trip, since from the current
++ * delivery process we can estimate the measured capacity available now.
++ *
++ * Anything faster than that approach would knowingly risk high loss, which can
++ * cause low bw for Reno/CUBIC and high loss recovery latency for
++ * request/response flows using any congestion control.
++ */
++static void bbr2_adapt_lower_bounds(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 ecn_cut, ecn_inflight_lo, beta;
++
++ /* We only use lower-bound estimates when not probing bw.
++ * When probing we need to push inflight higher to probe bw.
++ */
++ if (bbr2_is_probing_bandwidth(sk))
++ return;
++
++ /* ECN response. */
++ if (bbr->ecn_in_round && bbr->ecn_eligible && bbr->params.ecn_factor) {
++ /* Reduce inflight to (1 - alpha*ecn_factor). */
++ ecn_cut = (BBR_UNIT -
++ ((bbr->ecn_alpha * bbr->params.ecn_factor) >>
++ BBR_SCALE));
++ if (bbr->inflight_lo == ~0U)
++ bbr->inflight_lo = tp->snd_cwnd;
++ ecn_inflight_lo = (u64)bbr->inflight_lo * ecn_cut >> BBR_SCALE;
++ } else {
++ ecn_inflight_lo = ~0U;
++ }
++
++ /* Loss response. */
++ if (bbr->loss_in_round) {
++ /* Reduce bw and inflight to (1 - beta). */
++ if (bbr->bw_lo == ~0U)
++ bbr->bw_lo = bbr_max_bw(sk);
++ if (bbr->inflight_lo == ~0U)
++ bbr->inflight_lo = tp->snd_cwnd;
++ beta = bbr->params.beta;
++ bbr->bw_lo =
++ max_t(u32, bbr->bw_latest,
++ (u64)bbr->bw_lo *
++ (BBR_UNIT - beta) >> BBR_SCALE);
++ bbr->inflight_lo =
++ max_t(u32, bbr->inflight_latest,
++ (u64)bbr->inflight_lo *
++ (BBR_UNIT - beta) >> BBR_SCALE);
++ }
++
++ /* Adjust to the lower of the levels implied by loss or ECN. */
++ bbr->inflight_lo = min(bbr->inflight_lo, ecn_inflight_lo);
++}
++
++/* Reset any short-term lower-bound adaptation to congestion, so that we can
++ * push our inflight up.
++ */
++static void bbr2_reset_lower_bounds(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr->bw_lo = ~0U;
++ bbr->inflight_lo = ~0U;
++}
++
++/* After bw probing (STARTUP/PROBE_UP), reset signals before entering a state
++ * machine phase where we adapt our lower bound based on congestion signals.
++ */
++static void bbr2_reset_congestion_signals(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr->loss_in_round = 0;
++ bbr->ecn_in_round = 0;
++ bbr->loss_in_cycle = 0;
++ bbr->ecn_in_cycle = 0;
++ bbr->bw_latest = 0;
++ bbr->inflight_latest = 0;
++}
++
++/* Update (most of) our congestion signals: track the recent rate and volume of
++ * delivered data, presence of loss, and EWMA degree of ECN marking.
++ */
++static void bbr2_update_congestion_signals(
++ struct sock *sk, const struct rate_sample *rs, struct bbr_context *ctx)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ u64 bw;
++
++ bbr->loss_round_start = 0;
++ if (rs->interval_us <= 0 || !rs->acked_sacked)
++ return; /* Not a valid observation */
++ bw = ctx->sample_bw;
++
++ if (!rs->is_app_limited || bw >= bbr_max_bw(sk))
++ bbr2_take_bw_hi_sample(sk, bw);
++
++ bbr->loss_in_round |= (rs->losses > 0);
++
++ /* Update rate and volume of delivered data from latest round trip: */
++ bbr->bw_latest = max_t(u32, bbr->bw_latest, ctx->sample_bw);
++ bbr->inflight_latest = max_t(u32, bbr->inflight_latest, rs->delivered);
++
++ if (before(rs->prior_delivered, bbr->loss_round_delivered))
++ return; /* skip the per-round-trip updates */
++ /* Now do per-round-trip updates. */
++ bbr->loss_round_delivered = tp->delivered; /* mark round trip */
++ bbr->loss_round_start = 1;
++ bbr2_adapt_lower_bounds(sk);
++
++ /* Update windowed "latest" (single-round-trip) filters. */
++ bbr->loss_in_round = 0;
++ bbr->ecn_in_round = 0;
++ bbr->bw_latest = ctx->sample_bw;
++ bbr->inflight_latest = rs->delivered;
++}
++
++/* Bandwidth probing can cause loss. To help coexistence with loss-based
++ * congestion control we spread out our probing in a Reno-conscious way. Due to
++ * the shape of the Reno sawtooth, the time required between loss epochs for an
++ * idealized Reno flow is a number of round trips that is the BDP of that
++ * flow. We count packet-timed round trips directly, since measured RTT can
++ * vary widely, and Reno is driven by packet-timed round trips.
++ */
++static bool bbr2_is_reno_coexistence_probe_time(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 inflight, rounds, reno_gain, reno_rounds;
++
++ /* Random loss can shave some small percentage off of our inflight
++ * in each round. To survive this, flows need robust periodic probes.
++ */
++ rounds = bbr->params.bw_probe_max_rounds;
++
++ reno_gain = bbr->params.bw_probe_reno_gain;
++ if (reno_gain) {
++ inflight = bbr2_target_inflight(sk);
++ reno_rounds = ((u64)inflight * reno_gain) >> BBR_SCALE;
++ rounds = min(rounds, reno_rounds);
++ }
++ return bbr->rounds_since_probe >= rounds;
++}
++
++/* How long do we want to wait before probing for bandwidth (and risking
++ * loss)? We randomize the wait, for better mixing and fairness convergence.
++ *
++ * We bound the Reno-coexistence inter-bw-probe time to be 62-63 round trips.
++ * This is calculated to allow fairness with a 25Mbps, 30ms Reno flow,
++ * (eg 4K video to a broadband user):
++ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets
++ *
++ * We bound the BBR-native inter-bw-probe wall clock time to be:
++ * (a) higher than 2 sec: to try to avoid causing loss for a long enough time
++ * to allow Reno at 30ms to get 4K video bw, the inter-bw-probe time must
++ * be at least: 25Mbps * .030sec / (1514bytes) * 0.030sec = 1.9secs
++ * (b) lower than 3 sec: to ensure flows can start probing in a reasonable
++ * amount of time to discover unutilized bw on human-scale interactive
++ * time-scales (e.g. perhaps traffic from a web page download that we
++ * were competing with is now complete).
++ */
++static void bbr2_pick_probe_wait(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ /* Decide the random round-trip bound for wait until probe: */
++ bbr->rounds_since_probe =
++ prandom_u32_max(bbr->params.bw_probe_rand_rounds);
++ /* Decide the random wall clock bound for wait until probe: */
++ bbr->probe_wait_us = bbr->params.bw_probe_base_us +
++ prandom_u32_max(bbr->params.bw_probe_rand_us);
++}
++
++static void bbr2_set_cycle_idx(struct sock *sk, int cycle_idx)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr->cycle_idx = cycle_idx;
++ /* New phase, so need to update cwnd and pacing rate. */
++ bbr->try_fast_path = 0;
++}
++
++/* Send at estimated bw to fill the pipe, but not queue. We need this phase
++ * before PROBE_UP, because as soon as we send faster than the available bw
++ * we will start building a queue, and if the buffer is shallow we can cause
++ * loss. If we do not fill the pipe before we cause this loss, our bw_hi and
++ * inflight_hi estimates will underestimate.
++ */
++static void bbr2_start_bw_probe_refill(struct sock *sk, u32 bw_probe_up_rounds)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr2_reset_lower_bounds(sk);
++ if (bbr->inflight_hi != ~0U)
++ bbr->inflight_hi += bbr->params.refill_add_inc;
++ bbr->bw_probe_up_rounds = bw_probe_up_rounds;
++ bbr->bw_probe_up_acks = 0;
++ bbr->stopped_risky_probe = 0;
++ bbr->ack_phase = BBR_ACKS_REFILLING;
++ bbr->next_rtt_delivered = tp->delivered;
++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_REFILL);
++}
++
++/* Now probe max deliverable data rate and volume. */
++static void bbr2_start_bw_probe_up(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr->ack_phase = BBR_ACKS_PROBE_STARTING;
++ bbr->next_rtt_delivered = tp->delivered;
++ bbr->cycle_mstamp = tp->tcp_mstamp;
++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_UP);
++ bbr2_raise_inflight_hi_slope(sk);
++}
++
++/* Start a new PROBE_BW probing cycle of some wall clock length. Pick a wall
++ * clock time at which to probe beyond an inflight that we think to be
++ * safe. This will knowingly risk packet loss, so we want to do this rarely, to
++ * keep packet loss rates low. Also start a round-trip counter, to probe faster
++ * if we estimate a Reno flow at our BDP would probe faster.
++ */
++static void bbr2_start_bw_probe_down(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr2_reset_congestion_signals(sk);
++ bbr->bw_probe_up_cnt = ~0U; /* not growing inflight_hi any more */
++ bbr2_pick_probe_wait(sk);
++ bbr->cycle_mstamp = tp->tcp_mstamp; /* start wall clock */
++ bbr->ack_phase = BBR_ACKS_PROBE_STOPPING;
++ bbr->next_rtt_delivered = tp->delivered;
++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_DOWN);
++}
++
++/* Cruise: maintain what we estimate to be a neutral, conservative
++ * operating point, without attempting to probe up for bandwidth or down for
++ * RTT, and only reducing inflight in response to loss/ECN signals.
++ */
++static void bbr2_start_bw_probe_cruise(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (bbr->inflight_lo != ~0U)
++ bbr->inflight_lo = min(bbr->inflight_lo, bbr->inflight_hi);
++
++ bbr2_set_cycle_idx(sk, BBR_BW_PROBE_CRUISE);
++}
++
++/* Loss and/or ECN rate is too high while probing.
++ * Adapt (once per bw probe) by cutting inflight_hi and then restarting cycle.
++ */
++static void bbr2_handle_inflight_too_high(struct sock *sk,
++ const struct rate_sample *rs)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ const u32 beta = bbr->params.beta;
++
++ bbr->prev_probe_too_high = 1;
++ bbr->bw_probe_samples = 0; /* only react once per probe */
++ bbr->debug.event = 'L'; /* Loss/ECN too high */
++ /* If we are app-limited then we are not robustly
++ * probing the max volume of inflight data we think
++ * might be safe (analogous to how app-limited bw
++ * samples are not known to be robustly probing bw).
++ */
++ if (!rs->is_app_limited)
++ bbr->inflight_hi = max_t(u32, rs->tx_in_flight,
++ (u64)bbr2_target_inflight(sk) *
++ (BBR_UNIT - beta) >> BBR_SCALE);
++ if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == BBR_BW_PROBE_UP)
++ bbr2_start_bw_probe_down(sk);
++}
++
++/* If we're seeing bw and loss samples reflecting our bw probing, adapt
++ * using the signals we see. If loss or ECN mark rate gets too high, then adapt
++ * inflight_hi downward. If we're able to push inflight higher without such
++ * signals, push higher: adapt inflight_hi upward.
++ */
++static bool bbr2_adapt_upper_bounds(struct sock *sk,
++ const struct rate_sample *rs)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ /* Track when we'll see bw/loss samples resulting from our bw probes. */
++ if (bbr->ack_phase == BBR_ACKS_PROBE_STARTING && bbr->round_start)
++ bbr->ack_phase = BBR_ACKS_PROBE_FEEDBACK;
++ if (bbr->ack_phase == BBR_ACKS_PROBE_STOPPING && bbr->round_start) {
++ /* End of samples from bw probing phase. */
++ bbr->bw_probe_samples = 0;
++ bbr->ack_phase = BBR_ACKS_INIT;
++ /* At this point in the cycle, our current bw sample is also
++ * our best recent chance at finding the highest available bw
++ * for this flow. So now is the best time to forget the bw
++ * samples from the previous cycle, by advancing the window.
++ */
++ if (bbr->mode == BBR_PROBE_BW && !rs->is_app_limited)
++ bbr2_advance_bw_hi_filter(sk);
++ /* If we had an inflight_hi, then probed and pushed inflight all
++ * the way up to hit that inflight_hi without seeing any
++ * high loss/ECN in all the resulting ACKs from that probing,
++ * then probe up again, this time letting inflight persist at
++ * inflight_hi for a round trip, then accelerating beyond.
++ */
++ if (bbr->mode == BBR_PROBE_BW &&
++ bbr->stopped_risky_probe && !bbr->prev_probe_too_high) {
++ bbr->debug.event = 'R'; /* reprobe */
++ bbr2_start_bw_probe_refill(sk, 0);
++ return true; /* yes, decided state transition */
++ }
++ }
++
++ if (bbr2_is_inflight_too_high(sk, rs)) {
++ if (bbr->bw_probe_samples) /* sample is from bw probing? */
++ bbr2_handle_inflight_too_high(sk, rs);
++ } else {
++ /* Loss/ECN rate is declared safe. Adjust upper bound upward. */
++ if (bbr->inflight_hi == ~0U) /* no excess queue signals yet? */
++ return false;
++
++ /* To be resilient to random loss, we must raise inflight_hi
++ * if we observe in any phase that a higher level is safe.
++ */
++ if (rs->tx_in_flight > bbr->inflight_hi) {
++ bbr->inflight_hi = rs->tx_in_flight;
++ bbr->debug.event = 'U'; /* raise up inflight_hi */
++ }
++
++ if (bbr->mode == BBR_PROBE_BW &&
++ bbr->cycle_idx == BBR_BW_PROBE_UP)
++ bbr2_probe_inflight_hi_upward(sk, rs);
++ }
++
++ return false;
++}
++
++/* Check if it's time to probe for bandwidth now, and if so, kick it off. */
++static bool bbr2_check_time_to_probe_bw(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 n;
++
++ /* If we seem to be at an operating point where we are not seeing loss
++ * but we are seeing ECN marks, then when the ECN marks cease we reprobe
++ * quickly (in case a burst of cross-traffic has ceased and freed up bw,
++ * or in case we are sharing with multiplicatively probing traffic).
++ */
++ if (bbr->params.ecn_reprobe_gain && bbr->ecn_eligible &&
++ bbr->ecn_in_cycle && !bbr->loss_in_cycle &&
++ inet_csk(sk)->icsk_ca_state == TCP_CA_Open) {
++ bbr->debug.event = 'A'; /* *A*ll clear to probe *A*gain */
++ /* Calculate n so that when bbr2_raise_inflight_hi_slope()
++ * computes growth_this_round as 2^n it will be roughly the
++ * desired volume of data (inflight_hi*ecn_reprobe_gain).
++ */
++ n = ilog2((((u64)bbr->inflight_hi *
++ bbr->params.ecn_reprobe_gain) >> BBR_SCALE));
++ bbr2_start_bw_probe_refill(sk, n);
++ return true;
++ }
++
++ if (bbr2_has_elapsed_in_phase(sk, bbr->probe_wait_us) ||
++ bbr2_is_reno_coexistence_probe_time(sk)) {
++ bbr2_start_bw_probe_refill(sk, 0);
++ return true;
++ }
++ return false;
++}
++
++/* Is it time to transition from PROBE_DOWN to PROBE_CRUISE? */
++static bool bbr2_check_time_to_cruise(struct sock *sk, u32 inflight, u32 bw)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ bool is_under_bdp, is_long_enough;
++
++ /* Always need to pull inflight down to leave headroom in queue. */
++ if (inflight > bbr2_inflight_with_headroom(sk))
++ return false;
++
++ is_under_bdp = inflight <= bbr_inflight(sk, bw, BBR_UNIT);
++ if (bbr->params.drain_to_target)
++ return is_under_bdp;
++
++ is_long_enough = bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us);
++ return is_under_bdp || is_long_enough;
++}
++
++/* PROBE_BW state machine: cruise, refill, probe for bw, or drain? */
++static void bbr2_update_cycle_phase(struct sock *sk,
++ const struct rate_sample *rs)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ bool is_risky = false, is_queuing = false;
++ u32 inflight, bw;
++
++ if (!bbr_full_bw_reached(sk))
++ return;
++
++ /* In DRAIN, PROBE_BW, or PROBE_RTT, adjust upper bounds. */
++ if (bbr2_adapt_upper_bounds(sk, rs))
++ return; /* already decided state transition */
++
++ if (bbr->mode != BBR_PROBE_BW)
++ return;
++
++ inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight);
++ bw = bbr_max_bw(sk);
++
++ switch (bbr->cycle_idx) {
++ /* First we spend most of our time cruising with a pacing_gain of 1.0,
++ * which paces at the estimated bw, to try to fully use the pipe
++ * without building queue. If we encounter loss/ECN marks, we adapt
++ * by slowing down.
++ */
++ case BBR_BW_PROBE_CRUISE:
++ if (bbr2_check_time_to_probe_bw(sk))
++ return; /* already decided state transition */
++ break;
++
++ /* After cruising, when it's time to probe, we first "refill": we send
++ * at the estimated bw to fill the pipe, before probing higher and
++ * knowingly risking overflowing the bottleneck buffer (causing loss).
++ */
++ case BBR_BW_PROBE_REFILL:
++ if (bbr->round_start) {
++ /* After one full round trip of sending in REFILL, we
++ * start to see bw samples reflecting our REFILL, which
++ * may be putting too much data in flight.
++ */
++ bbr->bw_probe_samples = 1;
++ bbr2_start_bw_probe_up(sk);
++ }
++ break;
++
++ /* After we refill the pipe, we probe by using a pacing_gain > 1.0, to
++ * probe for bw. If we have not seen loss/ECN, we try to raise inflight
++ * to at least pacing_gain*BDP; note that this may take more than
++ * min_rtt if min_rtt is small (e.g. on a LAN).
++ *
++ * We terminate PROBE_UP bandwidth probing upon any of the following:
++ *
++ * (1) We've pushed inflight up to hit the inflight_hi target set in the
++ * most recent previous bw probe phase. Thus we want to start
++ * draining the queue immediately because it's very likely the most
++ * recently sent packets will fill the queue and cause drops.
++ * (checked here)
++ * (2) We have probed for at least 1*min_rtt_us, and the
++ * estimated queue is high enough (inflight > 1.25 * estimated_bdp).
++ * (checked here)
++ * (3) Loss filter says loss rate is "too high".
++ * (checked in bbr_is_inflight_too_high())
++ * (4) ECN filter says ECN mark rate is "too high".
++ * (checked in bbr_is_inflight_too_high())
++ */
++ case BBR_BW_PROBE_UP:
++ if (bbr->prev_probe_too_high &&
++ inflight >= bbr->inflight_hi) {
++ bbr->stopped_risky_probe = 1;
++ is_risky = true;
++ bbr->debug.event = 'D'; /* D for danger */
++ } else if (bbr2_has_elapsed_in_phase(sk, bbr->min_rtt_us) &&
++ inflight >=
++ bbr_inflight(sk, bw,
++ bbr->params.bw_probe_pif_gain)) {
++ is_queuing = true;
++ bbr->debug.event = 'Q'; /* building Queue */
++ }
++ if (is_risky || is_queuing) {
++ bbr->prev_probe_too_high = 0; /* no loss/ECN (yet) */
++ bbr2_start_bw_probe_down(sk); /* restart w/ down */
++ }
++ break;
++
++ /* After probing in PROBE_UP, we have usually accumulated some data in
++ * the bottleneck buffer (if bw probing didn't find more bw). We next
++ * enter PROBE_DOWN to try to drain any excess data from the queue. To
++ * do this, we use a pacing_gain < 1.0. We hold this pacing gain until
++ * our inflight is less then that target cruising point, which is the
++ * minimum of (a) the amount needed to leave headroom, and (b) the
++ * estimated BDP. Once inflight falls to match the target, we estimate
++ * the queue is drained; persisting would underutilize the pipe.
++ */
++ case BBR_BW_PROBE_DOWN:
++ if (bbr2_check_time_to_probe_bw(sk))
++ return; /* already decided state transition */
++ if (bbr2_check_time_to_cruise(sk, inflight, bw))
++ bbr2_start_bw_probe_cruise(sk);
++ break;
++
++ default:
++ WARN_ONCE(1, "BBR invalid cycle index %u\n", bbr->cycle_idx);
++ }
++}
++
++/* Exiting PROBE_RTT, so return to bandwidth probing in STARTUP or PROBE_BW. */
++static void bbr2_exit_probe_rtt(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr2_reset_lower_bounds(sk);
++ if (bbr_full_bw_reached(sk)) {
++ bbr->mode = BBR_PROBE_BW;
++ /* Raising inflight after PROBE_RTT may cause loss, so reset
++ * the PROBE_BW clock and schedule the next bandwidth probe for
++ * a friendly and randomized future point in time.
++ */
++ bbr2_start_bw_probe_down(sk);
++ /* Since we are exiting PROBE_RTT, we know inflight is
++ * below our estimated BDP, so it is reasonable to cruise.
++ */
++ bbr2_start_bw_probe_cruise(sk);
++ } else {
++ bbr->mode = BBR_STARTUP;
++ }
++}
++
++/* Exit STARTUP based on loss rate > 1% and loss gaps in round >= N. Wait until
++ * the end of the round in recovery to get a good estimate of how many packets
++ * have been lost, and how many we need to drain with a low pacing rate.
++ */
++static void bbr2_check_loss_too_high_in_startup(struct sock *sk,
++ const struct rate_sample *rs)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (bbr_full_bw_reached(sk))
++ return;
++
++ /* For STARTUP exit, check the loss rate at the end of each round trip
++ * of Recovery episodes in STARTUP. We check the loss rate at the end
++ * of the round trip to filter out noisy/low loss and have a better
++ * sense of inflight (extent of loss), so we can drain more accurately.
++ */
++ if (rs->losses && bbr->loss_events_in_round < 0xf)
++ bbr->loss_events_in_round++; /* update saturating counter */
++ if (bbr->params.full_loss_cnt && bbr->loss_round_start &&
++ inet_csk(sk)->icsk_ca_state == TCP_CA_Recovery &&
++ bbr->loss_events_in_round >= bbr->params.full_loss_cnt &&
++ bbr2_is_inflight_too_high(sk, rs)) {
++ bbr->debug.event = 'P'; /* Packet loss caused STARTUP exit */
++ bbr2_handle_queue_too_high_in_startup(sk);
++ return;
++ }
++ if (bbr->loss_round_start)
++ bbr->loss_events_in_round = 0;
++}
++
++/* If we are done draining, advance into steady state operation in PROBE_BW. */
++static void bbr2_check_drain(struct sock *sk, const struct rate_sample *rs,
++ struct bbr_context *ctx)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (bbr_check_drain(sk, rs, ctx)) {
++ bbr->mode = BBR_PROBE_BW;
++ bbr2_start_bw_probe_down(sk);
++ }
++}
++
++static void bbr2_update_model(struct sock *sk, const struct rate_sample *rs,
++ struct bbr_context *ctx)
++{
++ bbr2_update_congestion_signals(sk, rs, ctx);
++ bbr_update_ack_aggregation(sk, rs);
++ bbr2_check_loss_too_high_in_startup(sk, rs);
++ bbr_check_full_bw_reached(sk, rs);
++ bbr2_check_drain(sk, rs, ctx);
++ bbr2_update_cycle_phase(sk, rs);
++ bbr_update_min_rtt(sk, rs);
++}
++
++/* Fast path for app-limited case.
++ *
++ * On each ack, we execute bbr state machine, which primarily consists of:
++ * 1) update model based on new rate sample, and
++ * 2) update control based on updated model or state change.
++ *
++ * There are certain workload/scenarios, e.g. app-limited case, where
++ * either we can skip updating model or we can skip update of both model
++ * as well as control. This provides signifcant softirq cpu savings for
++ * processing incoming acks.
++ *
++ * In case of app-limited, if there is no congestion (loss/ecn) and
++ * if observed bw sample is less than current estimated bw, then we can
++ * skip some of the computation in bbr state processing:
++ *
++ * - if there is no rtt/mode/phase change: In this case, since all the
++ * parameters of the network model are constant, we can skip model
++ * as well control update.
++ *
++ * - else we can skip rest of the model update. But we still need to
++ * update the control to account for the new rtt/mode/phase.
++ *
++ * Returns whether we can take fast path or not.
++ */
++static bool bbr2_fast_path(struct sock *sk, bool *update_model,
++ const struct rate_sample *rs, struct bbr_context *ctx)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++ u32 prev_min_rtt_us, prev_mode;
++
++ if (bbr->params.fast_path && bbr->try_fast_path &&
++ rs->is_app_limited && ctx->sample_bw < bbr_max_bw(sk) &&
++ !bbr->loss_in_round && !bbr->ecn_in_round) {
++ prev_mode = bbr->mode;
++ prev_min_rtt_us = bbr->min_rtt_us;
++ bbr2_check_drain(sk, rs, ctx);
++ bbr2_update_cycle_phase(sk, rs);
++ bbr_update_min_rtt(sk, rs);
++
++ if (bbr->mode == prev_mode &&
++ bbr->min_rtt_us == prev_min_rtt_us &&
++ bbr->try_fast_path)
++ return true;
++
++ /* Skip model update, but control still needs to be updated */
++ *update_model = false;
++ }
++ return false;
++}
++
++static void bbr2_main(struct sock *sk, const struct rate_sample *rs)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ struct bbr_context ctx = { 0 };
++ bool update_model = true;
++ u32 bw;
++
++ bbr->debug.event = '.'; /* init to default NOP (no event yet) */
++
++ bbr_update_round_start(sk, rs, &ctx);
++ if (bbr->round_start) {
++ bbr->rounds_since_probe =
++ min_t(s32, bbr->rounds_since_probe + 1, 0xFF);
++ bbr2_update_ecn_alpha(sk);
++ }
++
++ bbr->ecn_in_round |= rs->is_ece;
++ bbr_calculate_bw_sample(sk, rs, &ctx);
++
++ if (bbr2_fast_path(sk, &update_model, rs, &ctx))
++ goto out;
++
++ if (update_model)
++ bbr2_update_model(sk, rs, &ctx);
++
++ bbr_update_gains(sk);
++ bw = bbr_bw(sk);
++ bbr_set_pacing_rate(sk, bw, bbr->pacing_gain);
++ bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain,
++ tp->snd_cwnd, &ctx);
++ bbr2_bound_cwnd_for_inflight_model(sk);
++
++out:
++ bbr->prev_ca_state = inet_csk(sk)->icsk_ca_state;
++ bbr->loss_in_cycle |= rs->lost > 0;
++ bbr->ecn_in_cycle |= rs->delivered_ce > 0;
++
++ bbr_debug(sk, rs->acked_sacked, rs, &ctx);
++}
++
++/* Module parameters that are settable by TCP_CONGESTION_PARAMS are declared
++ * down here, so that the algorithm functions that use the parameters must use
++ * the per-socket parameters; if they accidentally use the global version
++ * then there will be a compile error.
++ * TODO(ncardwell): move all per-socket parameters down to this section.
++ */
++
++/* On losses, scale down inflight and pacing rate by beta scaled by BBR_SCALE.
++ * No loss response when 0. Max allwed value is 255.
++ */
++static u32 bbr_beta = BBR_UNIT * 30 / 100;
++
++/* Gain factor for ECN mark ratio samples, scaled by BBR_SCALE.
++ * Max allowed value is 255.
++ */
++static u32 bbr_ecn_alpha_gain = BBR_UNIT * 1 / 16; /* 1/16 = 6.25% */
++
++/* The initial value for the ecn_alpha state variable. Default and max
++ * BBR_UNIT (256), representing 1.0. This allows a flow to respond quickly
++ * to congestion if the bottleneck is congested when the flow starts up.
++ */
++static u32 bbr_ecn_alpha_init = BBR_UNIT; /* 1.0, to respond quickly */
++
++/* On ECN, cut inflight_lo to (1 - ecn_factor * ecn_alpha) scaled by BBR_SCALE.
++ * No ECN based bounding when 0. Max allwed value is 255.
++ */
++static u32 bbr_ecn_factor = BBR_UNIT * 1 / 3; /* 1/3 = 33% */
++
++/* Estimate bw probing has gone too far if CE ratio exceeds this threshold.
++ * Scaled by BBR_SCALE. Disabled when 0. Max allowed is 255.
++ */
++static u32 bbr_ecn_thresh = BBR_UNIT * 1 / 2; /* 1/2 = 50% */
++
++/* Max RTT (in usec) at which to use sender-side ECN logic.
++ * Disabled when 0 (ECN allowed at any RTT).
++ * Max allowed for the parameter is 524287 (0x7ffff) us, ~524 ms.
++ */
++static u32 bbr_ecn_max_rtt_us = 5000;
++
++/* If non-zero, if in a cycle with no losses but some ECN marks, after ECN
++ * clears then use a multiplicative increase to quickly reprobe bw by
++ * starting inflight probing at the given multiple of inflight_hi.
++ * Default for this experimental knob is 0 (disabled).
++ * Planned value for experiments: BBR_UNIT * 1 / 2 = 128, representing 0.5.
++ */
++static u32 bbr_ecn_reprobe_gain;
++
++/* Estimate bw probing has gone too far if loss rate exceeds this level. */
++static u32 bbr_loss_thresh = BBR_UNIT * 2 / 100; /* 2% loss */
++
++/* Exit STARTUP if number of loss marking events in a Recovery round is >= N,
++ * and loss rate is higher than bbr_loss_thresh.
++ * Disabled if 0. Max allowed value is 15 (0xF).
++ */
++static u32 bbr_full_loss_cnt = 8;
++
++/* Exit STARTUP if number of round trips with ECN mark rate above ecn_thresh
++ * meets this count. Max allowed value is 3.
++ */
++static u32 bbr_full_ecn_cnt = 2;
++
++/* Fraction of unutilized headroom to try to leave in path upon high loss. */
++static u32 bbr_inflight_headroom = BBR_UNIT * 15 / 100;
++
++/* Multiplier to get target inflight (as multiple of BDP) for PROBE_UP phase.
++ * Default is 1.25x, as in BBR v1. Max allowed is 511.
++ */
++static u32 bbr_bw_probe_pif_gain = BBR_UNIT * 5 / 4;
++
++/* Multiplier to get Reno-style probe epoch duration as: k * BDP round trips.
++ * If zero, disables this BBR v2 Reno-style BDP-scaled coexistence mechanism.
++ * Max allowed is 511.
++ */
++static u32 bbr_bw_probe_reno_gain = BBR_UNIT;
++
++/* Max number of packet-timed rounds to wait before probing for bandwidth. If
++ * we want to tolerate 1% random loss per round, and not have this cut our
++ * inflight too much, we must probe for bw periodically on roughly this scale.
++ * If low, limits Reno/CUBIC coexistence; if high, limits loss tolerance.
++ * We aim to be fair with Reno/CUBIC up to a BDP of at least:
++ * BDP = 25Mbps * .030sec /(1514bytes) = 61.9 packets
++ */
++static u32 bbr_bw_probe_max_rounds = 63;
++
++/* Max amount of randomness to inject in round counting for Reno-coexistence.
++ * Max value is 15.
++ */
++static u32 bbr_bw_probe_rand_rounds = 2;
++
++/* Use BBR-native probe time scale starting at this many usec.
++ * We aim to be fair with Reno/CUBIC up to an inter-loss time epoch of at least:
++ * BDP*RTT = 25Mbps * .030sec /(1514bytes) * 0.030sec = 1.9 secs
++ */
++static u32 bbr_bw_probe_base_us = 2 * USEC_PER_SEC; /* 2 secs */
++
++/* Use BBR-native probes spread over this many usec: */
++static u32 bbr_bw_probe_rand_us = 1 * USEC_PER_SEC; /* 1 secs */
++
++/* Undo the model changes made in loss recovery if recovery was spurious? */
++static bool bbr_undo = true;
++
++/* Use fast path if app-limited, no loss/ECN, and target cwnd was reached? */
++static bool bbr_fast_path = true; /* default: enabled */
++
++/* Use fast ack mode ? */
++static int bbr_fast_ack_mode = 1; /* default: rwnd check off */
++
++/* How much to additively increase inflight_hi when entering REFILL? */
++static u32 bbr_refill_add_inc; /* default: disabled */
++
++module_param_named(beta, bbr_beta, uint, 0644);
++module_param_named(ecn_alpha_gain, bbr_ecn_alpha_gain, uint, 0644);
++module_param_named(ecn_alpha_init, bbr_ecn_alpha_init, uint, 0644);
++module_param_named(ecn_factor, bbr_ecn_factor, uint, 0644);
++module_param_named(ecn_thresh, bbr_ecn_thresh, uint, 0644);
++module_param_named(ecn_max_rtt_us, bbr_ecn_max_rtt_us, uint, 0644);
++module_param_named(ecn_reprobe_gain, bbr_ecn_reprobe_gain, uint, 0644);
++module_param_named(loss_thresh, bbr_loss_thresh, uint, 0664);
++module_param_named(full_loss_cnt, bbr_full_loss_cnt, uint, 0664);
++module_param_named(full_ecn_cnt, bbr_full_ecn_cnt, uint, 0664);
++module_param_named(inflight_headroom, bbr_inflight_headroom, uint, 0664);
++module_param_named(bw_probe_pif_gain, bbr_bw_probe_pif_gain, uint, 0664);
++module_param_named(bw_probe_reno_gain, bbr_bw_probe_reno_gain, uint, 0664);
++module_param_named(bw_probe_max_rounds, bbr_bw_probe_max_rounds, uint, 0664);
++module_param_named(bw_probe_rand_rounds, bbr_bw_probe_rand_rounds, uint, 0664);
++module_param_named(bw_probe_base_us, bbr_bw_probe_base_us, uint, 0664);
++module_param_named(bw_probe_rand_us, bbr_bw_probe_rand_us, uint, 0664);
++module_param_named(undo, bbr_undo, bool, 0664);
++module_param_named(fast_path, bbr_fast_path, bool, 0664);
++module_param_named(fast_ack_mode, bbr_fast_ack_mode, uint, 0664);
++module_param_named(refill_add_inc, bbr_refill_add_inc, uint, 0664);
++
++static void bbr2_init(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr_init(sk); /* run shared init code for v1 and v2 */
++
++ /* BBR v2 parameters: */
++ bbr->params.beta = min_t(u32, 0xFFU, bbr_beta);
++ bbr->params.ecn_alpha_gain = min_t(u32, 0xFFU, bbr_ecn_alpha_gain);
++ bbr->params.ecn_alpha_init = min_t(u32, BBR_UNIT, bbr_ecn_alpha_init);
++ bbr->params.ecn_factor = min_t(u32, 0xFFU, bbr_ecn_factor);
++ bbr->params.ecn_thresh = min_t(u32, 0xFFU, bbr_ecn_thresh);
++ bbr->params.ecn_max_rtt_us = min_t(u32, 0x7ffffU, bbr_ecn_max_rtt_us);
++ bbr->params.ecn_reprobe_gain = min_t(u32, 0x1FF, bbr_ecn_reprobe_gain);
++ bbr->params.loss_thresh = min_t(u32, 0xFFU, bbr_loss_thresh);
++ bbr->params.full_loss_cnt = min_t(u32, 0xFU, bbr_full_loss_cnt);
++ bbr->params.full_ecn_cnt = min_t(u32, 0x3U, bbr_full_ecn_cnt);
++ bbr->params.inflight_headroom =
++ min_t(u32, 0xFFU, bbr_inflight_headroom);
++ bbr->params.bw_probe_pif_gain =
++ min_t(u32, 0x1FFU, bbr_bw_probe_pif_gain);
++ bbr->params.bw_probe_reno_gain =
++ min_t(u32, 0x1FFU, bbr_bw_probe_reno_gain);
++ bbr->params.bw_probe_max_rounds =
++ min_t(u32, 0xFFU, bbr_bw_probe_max_rounds);
++ bbr->params.bw_probe_rand_rounds =
++ min_t(u32, 0xFU, bbr_bw_probe_rand_rounds);
++ bbr->params.bw_probe_base_us =
++ min_t(u32, (1 << 26) - 1, bbr_bw_probe_base_us);
++ bbr->params.bw_probe_rand_us =
++ min_t(u32, (1 << 26) - 1, bbr_bw_probe_rand_us);
++ bbr->params.undo = bbr_undo;
++ bbr->params.fast_path = bbr_fast_path ? 1 : 0;
++ bbr->params.refill_add_inc = min_t(u32, 0x3U, bbr_refill_add_inc);
++
++ /* BBR v2 state: */
++ bbr->initialized = 1;
++ /* Start sampling ECN mark rate after first full flight is ACKed: */
++ bbr->loss_round_delivered = tp->delivered + 1;
++ bbr->loss_round_start = 0;
++ bbr->undo_bw_lo = 0;
++ bbr->undo_inflight_lo = 0;
++ bbr->undo_inflight_hi = 0;
++ bbr->loss_events_in_round = 0;
++ bbr->startup_ecn_rounds = 0;
++ bbr2_reset_congestion_signals(sk);
++ bbr->bw_lo = ~0U;
++ bbr->bw_hi[0] = 0;
++ bbr->bw_hi[1] = 0;
++ bbr->inflight_lo = ~0U;
++ bbr->inflight_hi = ~0U;
++ bbr->bw_probe_up_cnt = ~0U;
++ bbr->bw_probe_up_acks = 0;
++ bbr->bw_probe_up_rounds = 0;
++ bbr->probe_wait_us = 0;
++ bbr->stopped_risky_probe = 0;
++ bbr->ack_phase = BBR_ACKS_INIT;
++ bbr->rounds_since_probe = 0;
++ bbr->bw_probe_samples = 0;
++ bbr->prev_probe_too_high = 0;
++ bbr->ecn_eligible = 0;
++ bbr->ecn_alpha = bbr->params.ecn_alpha_init;
++ bbr->alpha_last_delivered = 0;
++ bbr->alpha_last_delivered_ce = 0;
++
++ tp->fast_ack_mode = min_t(u32, 0x2U, bbr_fast_ack_mode);
++
++ if ((tp->ecn_flags & TCP_ECN_OK) && bbr_ecn_enable)
++ tp->ecn_flags |= TCP_ECN_ECT_PERMANENT;
++}
++
++/* Core TCP stack informs us that the given skb was just marked lost. */
++static void bbr2_skb_marked_lost(struct sock *sk, const struct sk_buff *skb)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++ struct tcp_skb_cb *scb = TCP_SKB_CB(skb);
++ struct rate_sample rs;
++
++ /* Capture "current" data over the full round trip of loss,
++ * to have a better chance to see the full capacity of the path.
++ */
++ if (!bbr->loss_in_round) /* first loss in this round trip? */
++ bbr->loss_round_delivered = tp->delivered; /* set round trip */
++ bbr->loss_in_round = 1;
++ bbr->loss_in_cycle = 1;
++
++ if (!bbr->bw_probe_samples)
++ return; /* not an skb sent while probing for bandwidth */
++ if (unlikely(!scb->tx.delivered_mstamp))
++ return; /* skb was SACKed, reneged, marked lost; ignore it */
++ /* We are probing for bandwidth. Construct a rate sample that
++ * estimates what happened in the flight leading up to this lost skb,
++ * then see if the loss rate went too high, and if so at which packet.
++ */
++ memset(&rs, 0, sizeof(rs));
++ rs.tx_in_flight = scb->tx.in_flight;
++ rs.lost = tp->lost - scb->tx.lost;
++ rs.is_app_limited = scb->tx.is_app_limited;
++ if (bbr2_is_inflight_too_high(sk, &rs)) {
++ rs.tx_in_flight = bbr2_inflight_hi_from_lost_skb(sk, &rs, skb);
++ bbr2_handle_inflight_too_high(sk, &rs);
++ }
++}
++
++/* Revert short-term model if current loss recovery event was spurious. */
++static u32 bbr2_undo_cwnd(struct sock *sk)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr->debug.undo = 1;
++ bbr->full_bw = 0; /* spurious slow-down; reset full pipe detection */
++ bbr->full_bw_cnt = 0;
++ bbr->loss_in_round = 0;
++
++ if (!bbr->params.undo)
++ return tp->snd_cwnd;
++
++ /* Revert to cwnd and other state saved before loss episode. */
++ bbr->bw_lo = max(bbr->bw_lo, bbr->undo_bw_lo);
++ bbr->inflight_lo = max(bbr->inflight_lo, bbr->undo_inflight_lo);
++ bbr->inflight_hi = max(bbr->inflight_hi, bbr->undo_inflight_hi);
++ return bbr->prior_cwnd;
++}
++
++/* Entering loss recovery, so save state for when we undo recovery. */
++static u32 bbr2_ssthresh(struct sock *sk)
++{
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ bbr_save_cwnd(sk);
++ /* For undo, save state that adapts based on loss signal. */
++ bbr->undo_bw_lo = bbr->bw_lo;
++ bbr->undo_inflight_lo = bbr->inflight_lo;
++ bbr->undo_inflight_hi = bbr->inflight_hi;
++ return tcp_sk(sk)->snd_ssthresh;
++}
++
++static enum tcp_bbr2_phase bbr2_get_phase(struct bbr *bbr)
++{
++ switch (bbr->mode) {
++ case BBR_STARTUP:
++ return BBR2_PHASE_STARTUP;
++ case BBR_DRAIN:
++ return BBR2_PHASE_DRAIN;
++ case BBR_PROBE_BW:
++ break;
++ case BBR_PROBE_RTT:
++ return BBR2_PHASE_PROBE_RTT;
++ default:
++ return BBR2_PHASE_INVALID;
++ }
++ switch (bbr->cycle_idx) {
++ case BBR_BW_PROBE_UP:
++ return BBR2_PHASE_PROBE_BW_UP;
++ case BBR_BW_PROBE_DOWN:
++ return BBR2_PHASE_PROBE_BW_DOWN;
++ case BBR_BW_PROBE_CRUISE:
++ return BBR2_PHASE_PROBE_BW_CRUISE;
++ case BBR_BW_PROBE_REFILL:
++ return BBR2_PHASE_PROBE_BW_REFILL;
++ default:
++ return BBR2_PHASE_INVALID;
++ }
++}
++
++static size_t bbr2_get_info(struct sock *sk, u32 ext, int *attr,
++ union tcp_cc_info *info)
++{
++ if (ext & (1 << (INET_DIAG_BBRINFO - 1)) ||
++ ext & (1 << (INET_DIAG_VEGASINFO - 1))) {
++ struct bbr *bbr = inet_csk_ca(sk);
++ u64 bw = bbr_bw_bytes_per_sec(sk, bbr_bw(sk));
++ u64 bw_hi = bbr_bw_bytes_per_sec(sk, bbr_max_bw(sk));
++ u64 bw_lo = bbr->bw_lo == ~0U ?
++ ~0ULL : bbr_bw_bytes_per_sec(sk, bbr->bw_lo);
++
++ memset(&info->bbr2, 0, sizeof(info->bbr2));
++ info->bbr2.bbr_bw_lsb = (u32)bw;
++ info->bbr2.bbr_bw_msb = (u32)(bw >> 32);
++ info->bbr2.bbr_min_rtt = bbr->min_rtt_us;
++ info->bbr2.bbr_pacing_gain = bbr->pacing_gain;
++ info->bbr2.bbr_cwnd_gain = bbr->cwnd_gain;
++ info->bbr2.bbr_bw_hi_lsb = (u32)bw_hi;
++ info->bbr2.bbr_bw_hi_msb = (u32)(bw_hi >> 32);
++ info->bbr2.bbr_bw_lo_lsb = (u32)bw_lo;
++ info->bbr2.bbr_bw_lo_msb = (u32)(bw_lo >> 32);
++ info->bbr2.bbr_mode = bbr->mode;
++ info->bbr2.bbr_phase = (__u8)bbr2_get_phase(bbr);
++ info->bbr2.bbr_version = (__u8)2;
++ info->bbr2.bbr_inflight_lo = bbr->inflight_lo;
++ info->bbr2.bbr_inflight_hi = bbr->inflight_hi;
++ info->bbr2.bbr_extra_acked = bbr_extra_acked(sk);
++ *attr = INET_DIAG_BBRINFO;
++ return sizeof(info->bbr2);
++ }
++ return 0;
++}
++
++static void bbr2_set_state(struct sock *sk, u8 new_state)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ struct bbr *bbr = inet_csk_ca(sk);
++
++ if (new_state == TCP_CA_Loss) {
++ struct rate_sample rs = { .losses = 1 };
++ struct bbr_context ctx = { 0 };
++
++ bbr->prev_ca_state = TCP_CA_Loss;
++ bbr->full_bw = 0;
++ if (!bbr2_is_probing_bandwidth(sk) && bbr->inflight_lo == ~0U) {
++ /* bbr_adapt_lower_bounds() needs cwnd before
++ * we suffered an RTO, to update inflight_lo:
++ */
++ bbr->inflight_lo =
++ max(tp->snd_cwnd, bbr->prior_cwnd);
++ }
++ bbr_debug(sk, 0, &rs, &ctx);
++ } else if (bbr->prev_ca_state == TCP_CA_Loss &&
++ new_state != TCP_CA_Loss) {
++ tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd);
++ bbr->try_fast_path = 0; /* bound cwnd using latest model */
++ }
++}
++
++static struct tcp_congestion_ops tcp_bbr2_cong_ops __read_mostly = {
++ .flags = TCP_CONG_NON_RESTRICTED | TCP_CONG_WANTS_CE_EVENTS,
++ .name = "bbr2",
++ .owner = THIS_MODULE,
++ .init = bbr2_init,
++ .cong_control = bbr2_main,
++ .sndbuf_expand = bbr_sndbuf_expand,
++ .skb_marked_lost = bbr2_skb_marked_lost,
++ .undo_cwnd = bbr2_undo_cwnd,
++ .cwnd_event = bbr_cwnd_event,
++ .ssthresh = bbr2_ssthresh,
++ .tso_segs = bbr_tso_segs,
++ .get_info = bbr2_get_info,
++ .set_state = bbr2_set_state,
++};
++
++static int __init bbr_register(void)
++{
++ BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE);
++ return tcp_register_congestion_control(&tcp_bbr2_cong_ops);
++}
++
++static void __exit bbr_unregister(void)
++{
++ tcp_unregister_congestion_control(&tcp_bbr2_cong_ops);
++}
++
++module_init(bbr_register);
++module_exit(bbr_unregister);
++
++MODULE_AUTHOR("Van Jacobson <[email protected]>");
++MODULE_AUTHOR("Neal Cardwell <[email protected]>");
++MODULE_AUTHOR("Yuchung Cheng <[email protected]>");
++MODULE_AUTHOR("Soheil Hassas Yeganeh <[email protected]>");
++MODULE_AUTHOR("Priyaranjan Jha <[email protected]>");
++MODULE_AUTHOR("Yousuk Seung <[email protected]>");
++MODULE_AUTHOR("Kevin Yang <[email protected]>");
++MODULE_AUTHOR("Arjun Roy <[email protected]>");
++
++MODULE_LICENSE("Dual BSD/GPL");
++MODULE_DESCRIPTION("TCP BBR (Bottleneck Bandwidth and RTT)");
+diff --git a/net/ipv4/tcp_cong.c b/net/ipv4/tcp_cong.c
+index db5831e6c136..153ed9010c0c 100644
+--- a/net/ipv4/tcp_cong.c
++++ b/net/ipv4/tcp_cong.c
+@@ -179,6 +179,7 @@ void tcp_init_congestion_control(struct sock *sk)
+ struct inet_connection_sock *icsk = inet_csk(sk);
+
+ tcp_sk(sk)->prior_ssthresh = 0;
++ tcp_sk(sk)->fast_ack_mode = 0;
+ if (icsk->icsk_ca_ops->init)
+ icsk->icsk_ca_ops->init(sk);
+ if (tcp_ca_needs_ecn(sk))
+diff --git a/net/ipv4/tcp_input.c b/net/ipv4/tcp_input.c
+index dc49a3d551eb..a403e21c702e 100644
+--- a/net/ipv4/tcp_input.c
++++ b/net/ipv4/tcp_input.c
+@@ -349,7 +349,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
+ tcp_enter_quickack_mode(sk, 2);
+ break;
+ case INET_ECN_CE:
+- if (tcp_ca_needs_ecn(sk))
++ if (tcp_ca_wants_ce_events(sk))
+ tcp_ca_event(sk, CA_EVENT_ECN_IS_CE);
+
+ if (!(tp->ecn_flags & TCP_ECN_DEMAND_CWR)) {
+@@ -360,7 +360,7 @@ static void __tcp_ecn_check_ce(struct sock *sk, const struct sk_buff *skb)
+ tp->ecn_flags |= TCP_ECN_SEEN;
+ break;
+ default:
+- if (tcp_ca_needs_ecn(sk))
++ if (tcp_ca_wants_ce_events(sk))
+ tcp_ca_event(sk, CA_EVENT_ECN_NO_CE);
+ tp->ecn_flags |= TCP_ECN_SEEN;
+ break;
+@@ -1079,7 +1079,12 @@ static void tcp_verify_retransmit_hint(struct tcp_sock *tp, struct sk_buff *skb)
+ */
+ static void tcp_notify_skb_loss_event(struct tcp_sock *tp, const struct sk_buff *skb)
+ {
++ struct sock *sk = (struct sock *)tp;
++ const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
++
+ tp->lost += tcp_skb_pcount(skb);
++ if (ca_ops->skb_marked_lost)
++ ca_ops->skb_marked_lost(sk, skb);
+ }
+
+ void tcp_mark_skb_lost(struct sock *sk, struct sk_buff *skb)
+@@ -1460,6 +1465,17 @@ static bool tcp_shifted_skb(struct sock *sk, struct sk_buff *prev,
+ WARN_ON_ONCE(tcp_skb_pcount(skb) < pcount);
+ tcp_skb_pcount_add(skb, -pcount);
+
++ /* Adjust tx.in_flight as pcount is shifted from skb to prev. */
++ if (WARN_ONCE(TCP_SKB_CB(skb)->tx.in_flight < pcount,
++ "prev in_flight: %u skb in_flight: %u pcount: %u",
++ TCP_SKB_CB(prev)->tx.in_flight,
++ TCP_SKB_CB(skb)->tx.in_flight,
++ pcount))
++ TCP_SKB_CB(skb)->tx.in_flight = 0;
++ else
++ TCP_SKB_CB(skb)->tx.in_flight -= pcount;
++ TCP_SKB_CB(prev)->tx.in_flight += pcount;
++
+ /* When we're adding to gso_segs == 1, gso_size will be zero,
+ * in theory this shouldn't be necessary but as long as DSACK
+ * code can come after this skb later on it's better to keep
+@@ -3788,6 +3804,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
+
+ prior_fack = tcp_is_sack(tp) ? tcp_highest_sack_seq(tp) : tp->snd_una;
+ rs.prior_in_flight = tcp_packets_in_flight(tp);
++ tcp_rate_check_app_limited(sk);
+
+ /* ts_recent update must be made after we are sure that the packet
+ * is in window.
+@@ -3885,6 +3902,7 @@ static int tcp_ack(struct sock *sk, const struct sk_buff *skb, int flag)
+ delivered = tcp_newly_delivered(sk, delivered, flag);
+ lost = tp->lost - lost; /* freshly marked lost */
+ rs.is_ack_delayed = !!(flag & FLAG_ACK_MAYBE_DELAYED);
++ rs.is_ece = !!(flag & FLAG_ECE);
+ tcp_rate_gen(sk, delivered, lost, is_sack_reneg, sack_state.rate);
+ tcp_cong_control(sk, ack, delivered, flag, sack_state.rate);
+ tcp_xmit_recovery(sk, rexmit);
+@@ -5463,13 +5481,14 @@ static void __tcp_ack_snd_check(struct sock *sk, int ofo_possible)
+
+ /* More than one full frame received... */
+ if (((tp->rcv_nxt - tp->rcv_wup) > inet_csk(sk)->icsk_ack.rcv_mss &&
++ (tp->fast_ack_mode == 1 ||
+ /* ... and right edge of window advances far enough.
+ * (tcp_recvmsg() will send ACK otherwise).
+ * If application uses SO_RCVLOWAT, we want send ack now if
+ * we have not received enough bytes to satisfy the condition.
+ */
+- (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat ||
+- __tcp_select_window(sk) >= tp->rcv_wnd)) ||
++ (tp->rcv_nxt - tp->copied_seq < sk->sk_rcvlowat ||
++ __tcp_select_window(sk) >= tp->rcv_wnd))) ||
+ /* We ACK each frame or... */
+ tcp_in_quickack_mode(sk) ||
+ /* Protocol state mandates a one-time immediate ACK */
+diff --git a/net/ipv4/tcp_output.c b/net/ipv4/tcp_output.c
+index 5079832af5c1..baa20918f11a 100644
+--- a/net/ipv4/tcp_output.c
++++ b/net/ipv4/tcp_output.c
+@@ -377,7 +377,8 @@ static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
+ th->cwr = 1;
+ skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
+ }
+- } else if (!tcp_ca_needs_ecn(sk)) {
++ } else if (!(tp->ecn_flags & TCP_ECN_ECT_PERMANENT) &&
++ !tcp_ca_needs_ecn(sk)) {
+ /* ACK or retransmitted segment: clear ECT|CE */
+ INET_ECN_dontxmit(sk);
+ }
+@@ -1533,7 +1534,7 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
+ {
+ struct tcp_sock *tp = tcp_sk(sk);
+ struct sk_buff *buff;
+- int nsize, old_factor;
++ int nsize, old_factor, inflight_prev;
+ long limit;
+ int nlen;
+ u8 flags;
+@@ -1610,6 +1611,15 @@ int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
+
+ if (diff)
+ tcp_adjust_pcount(sk, skb, diff);
++
++ /* Set buff tx.in_flight as if buff were sent by itself. */
++ inflight_prev = TCP_SKB_CB(skb)->tx.in_flight - old_factor;
++ if (WARN_ONCE(inflight_prev < 0,
++ "inconsistent: tx.in_flight: %u old_factor: %d",
++ TCP_SKB_CB(skb)->tx.in_flight, old_factor))
++ inflight_prev = 0;
++ TCP_SKB_CB(buff)->tx.in_flight = inflight_prev +
++ tcp_skb_pcount(buff);
+ }
+
+ /* Link BUFF into the send queue. */
+@@ -1978,13 +1988,12 @@ static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
+ static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)
+ {
+ const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
+- u32 min_tso, tso_segs;
+-
+- min_tso = ca_ops->min_tso_segs ?
+- ca_ops->min_tso_segs(sk) :
+- sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs;
++ u32 tso_segs;
+
+- tso_segs = tcp_tso_autosize(sk, mss_now, min_tso);
++ tso_segs = ca_ops->tso_segs ?
++ ca_ops->tso_segs(sk, mss_now) :
++ tcp_tso_autosize(sk, mss_now,
++ sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs);
+ return min_t(u32, tso_segs, sk->sk_gso_max_segs);
+ }
+
+@@ -2619,6 +2628,7 @@ static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
+ skb->skb_mstamp_ns = tp->tcp_wstamp_ns = tp->tcp_clock_cache;
+ list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
+ tcp_init_tso_segs(skb, mss_now);
++ tcp_set_tx_in_flight(sk, skb);
+ goto repair; /* Skip network transmission */
+ }
+
+diff --git a/net/ipv4/tcp_rate.c b/net/ipv4/tcp_rate.c
+index fbab921670cc..796fa6e5310c 100644
+--- a/net/ipv4/tcp_rate.c
++++ b/net/ipv4/tcp_rate.c
+@@ -34,6 +34,24 @@
+ * ready to send in the write queue.
+ */
+
++void tcp_set_tx_in_flight(struct sock *sk, struct sk_buff *skb)
++{
++ struct tcp_sock *tp = tcp_sk(sk);
++ u32 in_flight;
++
++ /* Check, sanitize, and record packets in flight after skb was sent. */
++ in_flight = tcp_packets_in_flight(tp) + tcp_skb_pcount(skb);
++ if (WARN_ONCE(in_flight > TCPCB_IN_FLIGHT_MAX,
++ "insane in_flight %u cc %s mss %u "
++ "cwnd %u pif %u %u %u %u\n",
++ in_flight, inet_csk(sk)->icsk_ca_ops->name,
++ tp->mss_cache, tp->snd_cwnd,
++ tp->packets_out, tp->retrans_out,
++ tp->sacked_out, tp->lost_out))
++ in_flight = TCPCB_IN_FLIGHT_MAX;
++ TCP_SKB_CB(skb)->tx.in_flight = in_flight;
++}
++
+ /* Snapshot the current delivery information in the skb, to generate
+ * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered().
+ */
+@@ -66,7 +84,9 @@ void tcp_rate_skb_sent(struct sock *sk, struct sk_buff *skb)
+ TCP_SKB_CB(skb)->tx.delivered_mstamp = tp->delivered_mstamp;
+ TCP_SKB_CB(skb)->tx.delivered = tp->delivered;
+ TCP_SKB_CB(skb)->tx.delivered_ce = tp->delivered_ce;
++ TCP_SKB_CB(skb)->tx.lost = tp->lost;
+ TCP_SKB_CB(skb)->tx.is_app_limited = tp->app_limited ? 1 : 0;
++ tcp_set_tx_in_flight(sk, skb);
+ }
+
+ /* When an skb is sacked or acked, we fill in the rate sample with the (prior)
+@@ -87,17 +107,20 @@ void tcp_rate_skb_delivered(struct sock *sk, struct sk_buff *skb,
+
+ if (!rs->prior_delivered ||
+ after(scb->tx.delivered, rs->prior_delivered)) {
++ rs->prior_lost = scb->tx.lost;
+ rs->prior_delivered_ce = scb->tx.delivered_ce;
+ rs->prior_delivered = scb->tx.delivered;
+ rs->prior_mstamp = scb->tx.delivered_mstamp;
+ rs->is_app_limited = scb->tx.is_app_limited;
+ rs->is_retrans = scb->sacked & TCPCB_RETRANS;
++ rs->tx_in_flight = scb->tx.in_flight;
+
+ /* Record send time of most recently ACKed packet: */
+ tp->first_tx_mstamp = tcp_skb_timestamp_us(skb);
+ /* Find the duration of the "send phase" of this window: */
+- rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp,
+- scb->tx.first_tx_mstamp);
++ rs->interval_us = tcp_stamp32_us_delta(
++ tp->first_tx_mstamp,
++ scb->tx.first_tx_mstamp);
+
+ }
+ /* Mark off the skb delivered once it's sacked to avoid being
+@@ -139,6 +162,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
+ return;
+ }
+ rs->delivered = tp->delivered - rs->prior_delivered;
++ rs->lost = tp->lost - rs->prior_lost;
+
+ rs->delivered_ce = tp->delivered_ce - rs->prior_delivered_ce;
+ /* delivered_ce occupies less than 32 bits in the skb control block */
+@@ -150,7 +174,7 @@ void tcp_rate_gen(struct sock *sk, u32 delivered, u32 lost,
+ * longer phase.
+ */
+ snd_us = rs->interval_us; /* send phase */
+- ack_us = tcp_stamp_us_delta(tp->tcp_mstamp,
++ ack_us = tcp_stamp32_us_delta(tp->tcp_mstamp,
+ rs->prior_mstamp); /* ack phase */
+ rs->interval_us = max(snd_us, ack_us);
+
+diff --git a/net/ipv4/tcp_timer.c b/net/ipv4/tcp_timer.c
+index 20cf4a98c69d..b5f7e49a003a 100644
+--- a/net/ipv4/tcp_timer.c
++++ b/net/ipv4/tcp_timer.c
+@@ -607,6 +607,7 @@ void tcp_write_timer_handler(struct sock *sk)
+ goto out;
+ }
+
++ tcp_rate_check_app_limited(sk);
+ tcp_mstamp_refresh(tcp_sk(sk));
+ event = icsk->icsk_pending;
+
+--
+2.34.1
+
+From f7f9bfc59cd69338d35b85f688cc94edc86bdba9 Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sat, 29 Jan 2022 07:49:56 +0000
+Subject: [PATCH] cfi
+
+---
+ arch/x86/Kconfig | 1 +
+ arch/x86/include/asm/ftrace.h | 2 +-
+ arch/x86/include/asm/idtentry.h | 10 +++---
+ arch/x86/include/asm/page_64.h | 7 ++--
+ arch/x86/include/asm/paravirt_types.h | 3 +-
+ arch/x86/include/asm/processor.h | 2 +-
+ arch/x86/include/asm/proto.h | 25 ++++++-------
+ arch/x86/include/asm/static_call.h | 1 -
+ arch/x86/include/asm/uaccess_64.h | 9 ++---
+ arch/x86/kernel/alternative.c | 2 +-
+ arch/x86/kernel/ftrace.c | 2 +-
+ arch/x86/kernel/kprobes/opt.c | 4 +--
+ arch/x86/kernel/module.c | 4 +++
+ arch/x86/kernel/paravirt.c | 4 +--
+ arch/x86/kernel/static_call.c | 14 +++-----
+ arch/x86/kvm/emulate.c | 4 +--
+ arch/x86/kvm/kvm_emulate.h | 9 ++---
+ arch/x86/power/Makefile | 2 ++
+ arch/x86/purgatory/Makefile | 4 +++
+ arch/x86/tools/relocs.c | 7 ++++
+ arch/x86/xen/enlighten_pv.c | 6 ++--
+ arch/x86/xen/xen-ops.h | 10 +++---
+ drivers/misc/lkdtm/bugs.c | 2 +-
+ drivers/misc/lkdtm/lkdtm.h | 2 +-
+ drivers/misc/lkdtm/perms.c | 2 +-
+ drivers/misc/lkdtm/rodata.c | 2 +-
+ include/asm-generic/vmlinux.lds.h | 11 ++++++
+ include/linux/cfi.h | 13 +++++++
+ include/linux/ftrace.h | 7 ++--
+ include/linux/linkage.h | 13 +++++++
+ include/linux/objtool.h | 6 ++++
+ kernel/cfi.c | 24 ++++++++++++-
+ kernel/tracepoint.c | 6 ++--
+ tools/include/linux/objtool.h | 6 ++++
+ tools/objtool/arch/x86/decode.c | 17 +++++++++
+ tools/objtool/check.c | 3 --
+ tools/objtool/elf.c | 51 +++++++++++++++++++++++++++
+ tools/objtool/include/objtool/arch.h | 3 ++
+ tools/objtool/include/objtool/elf.h | 2 +-
+ 39 files changed, 222 insertions(+), 80 deletions(-)
+
+diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
+index ebe8fc76949a..fb7e51bb0210 100644
+--- a/arch/x86/Kconfig
++++ b/arch/x86/Kconfig
+@@ -109,6 +109,7 @@ config X86
+ select ARCH_SUPPORTS_KMAP_LOCAL_FORCE_MAP if NR_CPUS <= 4096
+ select ARCH_SUPPORTS_LTO_CLANG
+ select ARCH_SUPPORTS_LTO_CLANG_THIN
++ select ARCH_SUPPORTS_CFI_CLANG if X86_64 && CLANG_VERSION >= 130000
+ select ARCH_USE_BUILTIN_BSWAP
+ select ARCH_USE_MEMTEST
+ select ARCH_USE_QUEUED_RWLOCKS
+diff --git a/arch/x86/include/asm/ftrace.h b/arch/x86/include/asm/ftrace.h
+index 024d9797646e..6fa07b9015e5 100644
+--- a/arch/x86/include/asm/ftrace.h
++++ b/arch/x86/include/asm/ftrace.h
+@@ -17,7 +17,7 @@
+
+ #ifndef __ASSEMBLY__
+ extern atomic_t modifying_ftrace_code;
+-extern void __fentry__(void);
++DECLARE_NOT_CALLED_FROM_C(__fentry__);
+
+ static inline unsigned long ftrace_call_adjust(unsigned long addr)
+ {
+diff --git a/arch/x86/include/asm/idtentry.h b/arch/x86/include/asm/idtentry.h
+index 1345088e9902..6538bf5a47d6 100644
+--- a/arch/x86/include/asm/idtentry.h
++++ b/arch/x86/include/asm/idtentry.h
+@@ -27,8 +27,8 @@
+ * as well which is used to emit the entry stubs in entry_32/64.S.
+ */
+ #define DECLARE_IDTENTRY(vector, func) \
+- asmlinkage void asm_##func(void); \
+- asmlinkage void xen_asm_##func(void); \
++ DECLARE_NOT_CALLED_FROM_C(asm_##func); \
++ DECLARE_NOT_CALLED_FROM_C(xen_asm_##func); \
+ __visible void func(struct pt_regs *regs)
+
+ /**
+@@ -78,8 +78,8 @@ static __always_inline void __##func(struct pt_regs *regs)
+ * C-handler.
+ */
+ #define DECLARE_IDTENTRY_ERRORCODE(vector, func) \
+- asmlinkage void asm_##func(void); \
+- asmlinkage void xen_asm_##func(void); \
++ DECLARE_NOT_CALLED_FROM_C(asm_##func); \
++ DECLARE_NOT_CALLED_FROM_C(xen_asm_##func); \
+ __visible void func(struct pt_regs *regs, unsigned long error_code)
+
+ /**
+@@ -386,7 +386,7 @@ static __always_inline void __##func(struct pt_regs *regs)
+ * - The C handler called from the C shim
+ */
+ #define DECLARE_IDTENTRY_DF(vector, func) \
+- asmlinkage void asm_##func(void); \
++ DECLARE_NOT_CALLED_FROM_C(asm_##func); \
+ __visible void func(struct pt_regs *regs, \
+ unsigned long error_code, \
+ unsigned long address)
+diff --git a/arch/x86/include/asm/page_64.h b/arch/x86/include/asm/page_64.h
+index e9c86299b835..7f314bcc78e8 100644
+--- a/arch/x86/include/asm/page_64.h
++++ b/arch/x86/include/asm/page_64.h
+@@ -6,6 +6,7 @@
+
+ #ifndef __ASSEMBLY__
+ #include <asm/cpufeatures.h>
++#include <linux/linkage.h>
+ #include <asm/alternative.h>
+
+ /* duplicated to the one in bootmem.h */
+@@ -41,9 +42,9 @@ extern unsigned long __phys_addr_symbol(unsigned long);
+ #define pfn_valid(pfn) ((pfn) < max_pfn)
+ #endif
+
+-void clear_page_orig(void *page);
+-void clear_page_rep(void *page);
+-void clear_page_erms(void *page);
++DECLARE_NOT_CALLED_FROM_C(clear_page_orig);
++DECLARE_NOT_CALLED_FROM_C(clear_page_rep);
++DECLARE_NOT_CALLED_FROM_C(clear_page_erms);
+
+ static inline void clear_page(void *page)
+ {
+diff --git a/arch/x86/include/asm/paravirt_types.h b/arch/x86/include/asm/paravirt_types.h
+index a69012e1903f..c3b00efd8658 100644
+--- a/arch/x86/include/asm/paravirt_types.h
++++ b/arch/x86/include/asm/paravirt_types.h
+@@ -38,6 +38,7 @@
+ #include <asm/desc_defs.h>
+ #include <asm/pgtable_types.h>
+ #include <asm/nospec-branch.h>
++#include <asm/proto.h>
+
+ struct page;
+ struct thread_struct;
+@@ -272,7 +273,7 @@ struct paravirt_patch_template {
+
+ extern struct pv_info pv_info;
+ extern struct paravirt_patch_template pv_ops;
+-extern void (*paravirt_iret)(void);
++extern asm_func_ptr paravirt_iret;
+
+ #define PARAVIRT_PATCH(x) \
+ (offsetof(struct paravirt_patch_template, x) / sizeof(void *))
+diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
+index 2c5f12ae7d04..4fe53e114fe0 100644
+--- a/arch/x86/include/asm/processor.h
++++ b/arch/x86/include/asm/processor.h
+@@ -450,7 +450,7 @@ static inline unsigned long cpu_kernelmode_gs_base(int cpu)
+
+ DECLARE_PER_CPU(void *, hardirq_stack_ptr);
+ DECLARE_PER_CPU(bool, hardirq_stack_inuse);
+-extern asmlinkage void ignore_sysret(void);
++DECLARE_NOT_CALLED_FROM_C(ignore_sysret);
+
+ /* Save actual FS/GS selectors and bases to current->thread */
+ void current_save_fsgs(void);
+diff --git a/arch/x86/include/asm/proto.h b/arch/x86/include/asm/proto.h
+index feed36d44d04..31ea9aeaf68e 100644
+--- a/arch/x86/include/asm/proto.h
++++ b/arch/x86/include/asm/proto.h
+@@ -2,6 +2,7 @@
+ #ifndef _ASM_X86_PROTO_H
+ #define _ASM_X86_PROTO_H
+
++#include <linux/linkage.h>
+ #include <asm/ldt.h>
+
+ struct task_struct;
+@@ -11,26 +12,26 @@ struct task_struct;
+ void syscall_init(void);
+
+ #ifdef CONFIG_X86_64
+-void entry_SYSCALL_64(void);
+-void entry_SYSCALL_64_safe_stack(void);
++DECLARE_NOT_CALLED_FROM_C(entry_SYSCALL_64);
++DECLARE_NOT_CALLED_FROM_C(entry_SYSCALL_64_safe_stack);
+ long do_arch_prctl_64(struct task_struct *task, int option, unsigned long arg2);
+ #endif
+
+ #ifdef CONFIG_X86_32
+-void entry_INT80_32(void);
+-void entry_SYSENTER_32(void);
+-void __begin_SYSENTER_singlestep_region(void);
+-void __end_SYSENTER_singlestep_region(void);
++DECLARE_NOT_CALLED_FROM_C(entry_INT80_32);
++DECLARE_NOT_CALLED_FROM_C(entry_SYSENTER_32);
++DECLARE_NOT_CALLED_FROM_C(__begin_SYSENTER_singlestep_region);
++DECLARE_NOT_CALLED_FROM_C(__end_SYSENTER_singlestep_region);
+ #endif
+
+ #ifdef CONFIG_IA32_EMULATION
+-void entry_SYSENTER_compat(void);
+-void __end_entry_SYSENTER_compat(void);
+-void entry_SYSCALL_compat(void);
+-void entry_SYSCALL_compat_safe_stack(void);
+-void entry_INT80_compat(void);
++DECLARE_NOT_CALLED_FROM_C(entry_SYSENTER_compat);
++DECLARE_NOT_CALLED_FROM_C(__end_entry_SYSENTER_compat);
++DECLARE_NOT_CALLED_FROM_C(entry_SYSCALL_compat);
++DECLARE_NOT_CALLED_FROM_C(entry_SYSCALL_compat_safe_stack);
++DECLARE_NOT_CALLED_FROM_C(entry_INT80_compat);
+ #ifdef CONFIG_XEN_PV
+-void xen_entry_INT80_compat(void);
++DECLARE_NOT_CALLED_FROM_C(xen_entry_INT80_compat);
+ #endif
+ #endif
+
+diff --git a/arch/x86/include/asm/static_call.h b/arch/x86/include/asm/static_call.h
+index ed4f8bb6c2d9..343234569392 100644
+--- a/arch/x86/include/asm/static_call.h
++++ b/arch/x86/include/asm/static_call.h
+@@ -27,7 +27,6 @@
+ ".globl " STATIC_CALL_TRAMP_STR(name) " \n" \
+ STATIC_CALL_TRAMP_STR(name) ": \n" \
+ insns " \n" \
+- ".byte 0x53, 0x43, 0x54 \n" \
+ ".type " STATIC_CALL_TRAMP_STR(name) ", @function \n" \
+ ".size " STATIC_CALL_TRAMP_STR(name) ", . - " STATIC_CALL_TRAMP_STR(name) " \n" \
+ ".popsection \n")
+diff --git a/arch/x86/include/asm/uaccess_64.h b/arch/x86/include/asm/uaccess_64.h
+index 45697e04d771..96cf72d6b75c 100644
+--- a/arch/x86/include/asm/uaccess_64.h
++++ b/arch/x86/include/asm/uaccess_64.h
+@@ -17,12 +17,9 @@
+ */
+
+ /* Handles exceptions in both to and from, but doesn't do access_ok */
+-__must_check unsigned long
+-copy_user_enhanced_fast_string(void *to, const void *from, unsigned len);
+-__must_check unsigned long
+-copy_user_generic_string(void *to, const void *from, unsigned len);
+-__must_check unsigned long
+-copy_user_generic_unrolled(void *to, const void *from, unsigned len);
++DECLARE_NOT_CALLED_FROM_C(copy_user_enhanced_fast_string);
++DECLARE_NOT_CALLED_FROM_C(copy_user_generic_string);
++DECLARE_NOT_CALLED_FROM_C(copy_user_generic_unrolled);
+
+ static __always_inline __must_check unsigned long
+ copy_user_generic(void *to, const void *from, unsigned len)
+diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
+index 5007c3ffe96f..60347ce35871 100644
+--- a/arch/x86/kernel/alternative.c
++++ b/arch/x86/kernel/alternative.c
+@@ -707,7 +707,7 @@ extern struct paravirt_patch_site __start_parainstructions[],
+ * convention such that we can 'call' it from assembly.
+ */
+
+-extern void int3_magic(unsigned int *ptr); /* defined in asm */
++DECLARE_NOT_CALLED_FROM_C(int3_magic);
+
+ asm (
+ " .pushsection .init.text, \"ax\", @progbits\n"
+diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
+index 7cc540e6de0c..d5a5df914e16 100644
+--- a/arch/x86/kernel/ftrace.c
++++ b/arch/x86/kernel/ftrace.c
+@@ -587,7 +587,7 @@ void arch_ftrace_trampoline_free(struct ftrace_ops *ops)
+ #ifdef CONFIG_DYNAMIC_FTRACE
+
+ #ifndef CONFIG_HAVE_DYNAMIC_FTRACE_WITH_ARGS
+-extern void ftrace_graph_call(void);
++DECLARE_NOT_CALLED_FROM_C(ftrace_graph_call);
+ static const char *ftrace_jmp_replace(unsigned long ip, unsigned long addr)
+ {
+ return text_gen_insn(JMP32_INSN_OPCODE, (void *)ip, (void *)addr);
+diff --git a/arch/x86/kernel/kprobes/opt.c b/arch/x86/kernel/kprobes/opt.c
+index b4a54a52aa59..48d302cd7448 100644
+--- a/arch/x86/kernel/kprobes/opt.c
++++ b/arch/x86/kernel/kprobes/opt.c
+@@ -103,6 +103,7 @@ static void synthesize_set_arg1(kprobe_opcode_t *addr, unsigned long val)
+ asm (
+ ".pushsection .rodata\n"
+ "optprobe_template_func:\n"
++ ASM_STACK_FRAME_NON_STANDARD(optprobe_template_func)
+ ".global optprobe_template_entry\n"
+ "optprobe_template_entry:\n"
+ #ifdef CONFIG_X86_64
+@@ -154,9 +155,6 @@ asm (
+ "optprobe_template_end:\n"
+ ".popsection\n");
+
+-void optprobe_template_func(void);
+-STACK_FRAME_NON_STANDARD(optprobe_template_func);
+-
+ #define TMPL_CLAC_IDX \
+ ((long)optprobe_template_clac - (long)optprobe_template_entry)
+ #define TMPL_MOVE_IDX \
+diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
+index 95fa745e310a..35baee327976 100644
+--- a/arch/x86/kernel/module.c
++++ b/arch/x86/kernel/module.c
+@@ -198,6 +198,10 @@ static int __apply_relocate_add(Elf64_Shdr *sechdrs,
+ val -= (u64)loc;
+ write(loc, &val, 8);
+ break;
++ case R_X86_64_8:
++ if (!strncmp(strtab + sym->st_name, "__typeid__", 10))
++ break;
++ fallthrough;
+ default:
+ pr_err("%s: Unknown rela relocation: %llu\n",
+ me->name, ELF64_R_TYPE(rel[i].r_info));
+diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c
+index 4420499f7bb4..3dec0dd45d92 100644
+--- a/arch/x86/kernel/paravirt.c
++++ b/arch/x86/kernel/paravirt.c
+@@ -149,7 +149,7 @@ void paravirt_set_sched_clock(u64 (*func)(void))
+ }
+
+ /* These are in entry.S */
+-extern void native_iret(void);
++DECLARE_NOT_CALLED_FROM_C(native_iret);
+
+ static struct resource reserve_ioports = {
+ .start = 0,
+@@ -415,7 +415,7 @@ struct paravirt_patch_template pv_ops = {
+ #ifdef CONFIG_PARAVIRT_XXL
+ NOKPROBE_SYMBOL(native_load_idt);
+
+-void (*paravirt_iret)(void) = native_iret;
++asm_func_ptr paravirt_iret = native_iret;
+ #endif
+
+ EXPORT_SYMBOL(pv_ops);
+diff --git a/arch/x86/kernel/static_call.c b/arch/x86/kernel/static_call.c
+index 531fb4cbb63f..81c9052033c7 100644
+--- a/arch/x86/kernel/static_call.c
++++ b/arch/x86/kernel/static_call.c
+@@ -57,15 +57,10 @@ static void __ref __static_call_transform(void *insn, enum insn_type type, void
+ text_poke_bp(insn, code, size, emulate);
+ }
+
+-static void __static_call_validate(void *insn, bool tail, bool tramp)
++static void __static_call_validate(void *insn, bool tail)
+ {
+ u8 opcode = *(u8 *)insn;
+
+- if (tramp && memcmp(insn+5, "SCT", 3)) {
+- pr_err("trampoline signature fail");
+- BUG();
+- }
+-
+ if (tail) {
+ if (opcode == JMP32_INSN_OPCODE ||
+ opcode == RET_INSN_OPCODE)
+@@ -80,8 +75,7 @@ static void __static_call_validate(void *insn, bool tail, bool tramp)
+ /*
+ * If we ever trigger this, our text is corrupt, we'll probably not live long.
+ */
+- pr_err("unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn);
+- BUG();
++ WARN_ONCE(1, "unexpected static_call insn opcode 0x%x at %pS\n", opcode, insn);
+ }
+
+ static inline enum insn_type __sc_insn(bool null, bool tail)
+@@ -104,12 +98,12 @@ void arch_static_call_transform(void *site, void *tramp, void *func, bool tail)
+ mutex_lock(&text_mutex);
+
+ if (tramp) {
+- __static_call_validate(tramp, true, true);
++ __static_call_validate(tramp, true);
+ __static_call_transform(tramp, __sc_insn(!func, true), func);
+ }
+
+ if (IS_ENABLED(CONFIG_HAVE_STATIC_CALL_INLINE) && site) {
+- __static_call_validate(site, tail, false);
++ __static_call_validate(site, tail);
+ __static_call_transform(site, __sc_insn(!func, tail), func);
+ }
+
+diff --git a/arch/x86/kvm/emulate.c b/arch/x86/kvm/emulate.c
+index 5719d8cfdbd9..978e6fffcc69 100644
+--- a/arch/x86/kvm/emulate.c
++++ b/arch/x86/kvm/emulate.c
+@@ -203,7 +203,7 @@ struct opcode {
+ const struct escape *esc;
+ const struct instr_dual *idual;
+ const struct mode_dual *mdual;
+- void (*fastop)(struct fastop *fake);
++ fastop_t fastop;
+ } u;
+ int (*check_perm)(struct x86_emulate_ctxt *ctxt);
+ };
+@@ -324,7 +324,7 @@ static int fastop(struct x86_emulate_ctxt *ctxt, fastop_t fop);
+ __FOP_RET(#name)
+
+ #define FOP_START(op) \
+- extern void em_##op(struct fastop *fake); \
++ DECLARE_NOT_CALLED_FROM_C(em_##op); \
+ asm(".pushsection .text, \"ax\" \n\t" \
+ ".global em_" #op " \n\t" \
+ ".align " __stringify(FASTOP_SIZE) " \n\t" \
+diff --git a/arch/x86/kvm/kvm_emulate.h b/arch/x86/kvm/kvm_emulate.h
+index 39eded2426ff..49ca866e478d 100644
+--- a/arch/x86/kvm/kvm_emulate.h
++++ b/arch/x86/kvm/kvm_emulate.h
+@@ -290,13 +290,8 @@ enum x86emul_mode {
+ #define X86EMUL_SMM_MASK (1 << 6)
+ #define X86EMUL_SMM_INSIDE_NMI_MASK (1 << 7)
+
+-/*
+- * fastop functions are declared as taking a never-defined fastop parameter,
+- * so they can't be called from C directly.
+- */
+-struct fastop;
+-
+-typedef void (*fastop_t)(struct fastop *);
++/* fastop functions cannot be called from C directly. */
++typedef asm_func_ptr fastop_t;
+
+ struct x86_emulate_ctxt {
+ void *vcpu;
+diff --git a/arch/x86/power/Makefile b/arch/x86/power/Makefile
+index 379777572bc9..a0532851fed7 100644
+--- a/arch/x86/power/Makefile
++++ b/arch/x86/power/Makefile
+@@ -4,9 +4,11 @@
+ # itself be stack-protected
+ CFLAGS_cpu.o := -fno-stack-protector
+
++ifndef CONFIG_CFI_CLANG
+ # Clang may incorrectly inline functions with stack protector enabled into
+ # __restore_processor_state(): https://bugs.llvm.org/show_bug.cgi?id=47479
+ CFLAGS_REMOVE_cpu.o := $(CC_FLAGS_LTO)
++endif
+
+ obj-$(CONFIG_PM_SLEEP) += cpu.o
+ obj-$(CONFIG_HIBERNATION) += hibernate_$(BITS).o hibernate_asm_$(BITS).o hibernate.o
+diff --git a/arch/x86/purgatory/Makefile b/arch/x86/purgatory/Makefile
+index ae53d54d7959..b3fa947fa38b 100644
+--- a/arch/x86/purgatory/Makefile
++++ b/arch/x86/purgatory/Makefile
+@@ -55,6 +55,10 @@ ifdef CONFIG_RETPOLINE
+ PURGATORY_CFLAGS_REMOVE += $(RETPOLINE_CFLAGS)
+ endif
+
++ifdef CONFIG_CFI_CLANG
++PURGATORY_CFLAGS_REMOVE += $(CC_FLAGS_CFI)
++endif
++
+ CFLAGS_REMOVE_purgatory.o += $(PURGATORY_CFLAGS_REMOVE)
+ CFLAGS_purgatory.o += $(PURGATORY_CFLAGS)
+
+diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
+index e2c5b296120d..e1d261ed4da8 100644
+--- a/arch/x86/tools/relocs.c
++++ b/arch/x86/tools/relocs.c
+@@ -56,6 +56,7 @@ static const char * const sym_regex_kernel[S_NSYMTYPES] = {
+ "^(xen_irq_disable_direct_reloc$|"
+ "xen_save_fl_direct_reloc$|"
+ "VDSO|"
++ "__typeid__|"
+ "__crc_)",
+
+ /*
+@@ -866,6 +867,12 @@ static int do_reloc64(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym,
+ symname);
+ break;
+
++ case R_X86_64_8:
++ if (!shn_abs || !is_reloc(S_ABS, symname))
++ die("Non-whitelisted %s relocation: %s\n",
++ rel_type(r_type), symname);
++ break;
++
+ case R_X86_64_32:
+ case R_X86_64_32S:
+ case R_X86_64_64:
+diff --git a/arch/x86/xen/enlighten_pv.c b/arch/x86/xen/enlighten_pv.c
+index 5004feb16783..20c7fc1a660a 100644
+--- a/arch/x86/xen/enlighten_pv.c
++++ b/arch/x86/xen/enlighten_pv.c
+@@ -583,8 +583,8 @@ DEFINE_IDTENTRY_RAW(xenpv_exc_machine_check)
+ #endif
+
+ struct trap_array_entry {
+- void (*orig)(void);
+- void (*xen)(void);
++ asm_func_ptr orig;
++ asm_func_ptr xen;
+ bool ist_okay;
+ };
+
+@@ -643,7 +643,7 @@ static bool __ref get_trap_addr(void **addr, unsigned int ist)
+ struct trap_array_entry *entry = trap_array + nr;
+
+ if (*addr == entry->orig) {
+- *addr = entry->xen;
++ *addr = (void *)entry->xen;
+ ist_okay = entry->ist_okay;
+ found = true;
+ break;
+diff --git a/arch/x86/xen/xen-ops.h b/arch/x86/xen/xen-ops.h
+index fd0fec6e92f4..fc1afa5cd935 100644
+--- a/arch/x86/xen/xen-ops.h
++++ b/arch/x86/xen/xen-ops.h
+@@ -8,12 +8,12 @@
+ #include <xen/xen-ops.h>
+
+ /* These are code, but not functions. Defined in entry.S */
+-extern const char xen_failsafe_callback[];
++DECLARE_NOT_CALLED_FROM_C(xen_failsafe_callback);
+
+-void xen_sysenter_target(void);
++DECLARE_NOT_CALLED_FROM_C(xen_sysenter_target);
+ #ifdef CONFIG_X86_64
+-void xen_syscall_target(void);
+-void xen_syscall32_target(void);
++DECLARE_NOT_CALLED_FROM_C(xen_syscall_target);
++DECLARE_NOT_CALLED_FROM_C(xen_syscall32_target);
+ #endif
+
+ extern void *xen_initial_gdt;
+@@ -137,7 +137,7 @@ __visible unsigned long xen_read_cr2(void);
+ __visible unsigned long xen_read_cr2_direct(void);
+
+ /* These are not functions, and cannot be called normally */
+-__visible void xen_iret(void);
++DECLARE_NOT_CALLED_FROM_C(xen_iret);
+
+ extern int xen_panic_handler_init(void);
+
+diff --git a/drivers/misc/lkdtm/bugs.c b/drivers/misc/lkdtm/bugs.c
+index f21854ac5cc2..7b4181720848 100644
+--- a/drivers/misc/lkdtm/bugs.c
++++ b/drivers/misc/lkdtm/bugs.c
+@@ -446,7 +446,7 @@ void lkdtm_STACK_GUARD_PAGE_TRAILING(void)
+
+ void lkdtm_UNSET_SMEP(void)
+ {
+-#if IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_UML)
++#if IS_ENABLED(CONFIG_X86_64) && !IS_ENABLED(CONFIG_UML) && !IS_ENABLED(CONFIG_CFI_CLANG)
+ #define MOV_CR4_DEPTH 64
+ void (*direct_write_cr4)(unsigned long val);
+ unsigned char *insn;
+diff --git a/drivers/misc/lkdtm/lkdtm.h b/drivers/misc/lkdtm/lkdtm.h
+index d6137c70ebbe..867d2fcb1abe 100644
+--- a/drivers/misc/lkdtm/lkdtm.h
++++ b/drivers/misc/lkdtm/lkdtm.h
+@@ -138,7 +138,7 @@ void lkdtm_REFCOUNT_TIMING(void);
+ void lkdtm_ATOMIC_TIMING(void);
+
+ /* rodata.c */
+-void lkdtm_rodata_do_nothing(void);
++DECLARE_NOT_CALLED_FROM_C(lkdtm_rodata_do_nothing);
+
+ /* usercopy.c */
+ void __init lkdtm_usercopy_init(void);
+diff --git a/drivers/misc/lkdtm/perms.c b/drivers/misc/lkdtm/perms.c
+index 2dede2ef658f..fa2bd90bd8ee 100644
+--- a/drivers/misc/lkdtm/perms.c
++++ b/drivers/misc/lkdtm/perms.c
+@@ -151,7 +151,7 @@ void lkdtm_EXEC_VMALLOC(void)
+
+ void lkdtm_EXEC_RODATA(void)
+ {
+- execute_location(lkdtm_rodata_do_nothing, CODE_AS_IS);
++ execute_location((void *)lkdtm_rodata_do_nothing, CODE_AS_IS);
+ }
+
+ void lkdtm_EXEC_USERSPACE(void)
+diff --git a/drivers/misc/lkdtm/rodata.c b/drivers/misc/lkdtm/rodata.c
+index baacb876d1d9..17ed0ad4e6ae 100644
+--- a/drivers/misc/lkdtm/rodata.c
++++ b/drivers/misc/lkdtm/rodata.c
+@@ -3,7 +3,7 @@
+ * This includes functions that are meant to live entirely in .rodata
+ * (via objcopy tricks), to validate the non-executability of .rodata.
+ */
+-#include "lkdtm.h"
++void lkdtm_rodata_do_nothing(void);
+
+ void noinstr lkdtm_rodata_do_nothing(void)
+ {
+diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h
+index 42f3866bca69..b4d0be59c8d5 100644
+--- a/include/asm-generic/vmlinux.lds.h
++++ b/include/asm-generic/vmlinux.lds.h
+@@ -413,6 +413,16 @@
+ KEEP(*(.static_call_tramp_key)) \
+ __stop_static_call_tramp_key = .;
+
++#ifdef CONFIG_CFI_CLANG
++#define CFI_EXCLUDED_DATA \
++ . = ALIGN(8); \
++ __start_cfi_excluded = .; \
++ KEEP(*(.cfi_excluded_stubs)) \
++ __stop_cfi_excluded = .;
++#else
++#define CFI_EXCLUDED_DATA
++#endif
++
+ /*
+ * Allow architectures to handle ro_after_init data on their
+ * own by defining an empty RO_AFTER_INIT_DATA.
+@@ -436,6 +446,7 @@
+ __start_rodata = .; \
+ *(.rodata) *(.rodata.*) \
+ SCHED_DATA \
++ CFI_EXCLUDED_DATA \
+ RO_AFTER_INIT_DATA /* Read only after init */ \
+ . = ALIGN(8); \
+ __start___tracepoints_ptrs = .; \
+diff --git a/include/linux/cfi.h b/include/linux/cfi.h
+index 879744aaa6e0..19f74af8eac2 100644
+--- a/include/linux/cfi.h
++++ b/include/linux/cfi.h
+@@ -20,6 +20,17 @@ extern void __cfi_check(uint64_t id, void *ptr, void *diag);
+ #define __CFI_ADDRESSABLE(fn, __attr) \
+ const void *__cfi_jt_ ## fn __visible __attr = (void *)&fn
+
++/*
++ * Defines a stub function that returns immediately, and when defined and
++ * referenced in the core kernel, always passes CFI checking. This should
++ * be used only for stubs that cannot be called using the correct function
++ * pointer type, which should be rare.
++ */
++#define DEFINE_CFI_IMMEDIATE_RETURN_STUB(fn) \
++ void fn(void) { return; } \
++ const void *__cfi_excl_ ## fn __visible \
++ __section(".cfi_excluded_stubs") = (void *)&fn
++
+ #ifdef CONFIG_CFI_CLANG_SHADOW
+
+ extern void cfi_module_add(struct module *mod, unsigned long base_addr);
+@@ -35,6 +46,8 @@ static inline void cfi_module_remove(struct module *mod, unsigned long base_addr
+ #else /* !CONFIG_CFI_CLANG */
+
+ #define __CFI_ADDRESSABLE(fn, __attr)
++#define DEFINE_CFI_IMMEDIATE_RETURN_STUB(fn) \
++ void fn(void) { return; }
+
+ #endif /* CONFIG_CFI_CLANG */
+
+diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
+index 9999e29187de..d36aac8caeef 100644
+--- a/include/linux/ftrace.h
++++ b/include/linux/ftrace.h
+@@ -603,9 +603,10 @@ extern void ftrace_replace_code(int enable);
+ extern int ftrace_update_ftrace_func(ftrace_func_t func);
+ extern void ftrace_caller(void);
+ extern void ftrace_regs_caller(void);
+-extern void ftrace_call(void);
+-extern void ftrace_regs_call(void);
+-extern void mcount_call(void);
++
++DECLARE_NOT_CALLED_FROM_C(ftrace_call);
++DECLARE_NOT_CALLED_FROM_C(ftrace_regs_call);
++DECLARE_NOT_CALLED_FROM_C(mcount_call);
+
+ void ftrace_modify_all_code(int command);
+
+diff --git a/include/linux/linkage.h b/include/linux/linkage.h
+index dbf8506decca..f982d5f550ac 100644
+--- a/include/linux/linkage.h
++++ b/include/linux/linkage.h
+@@ -48,6 +48,19 @@
+ #define __PAGE_ALIGNED_DATA .section ".data..page_aligned", "aw"
+ #define __PAGE_ALIGNED_BSS .section ".bss..page_aligned", "aw"
+
++/*
++ * Declares a function not callable from C using an opaque type. Defined as
++ * an array to allow the address of the symbol to be taken without '&'.
++ */
++#ifndef DECLARE_NOT_CALLED_FROM_C
++#define DECLARE_NOT_CALLED_FROM_C(sym) \
++ extern const u8 sym[]
++#endif
++
++#ifndef __ASSEMBLY__
++typedef const u8 *asm_func_ptr;
++#endif
++
+ /*
+ * This is used by architectures to keep arguments on the stack
+ * untouched by the compiler by keeping them live until the end.
+diff --git a/include/linux/objtool.h b/include/linux/objtool.h
+index aca52db2f3f3..f3374fc3ff01 100644
+--- a/include/linux/objtool.h
++++ b/include/linux/objtool.h
+@@ -66,6 +66,11 @@ struct unwind_hint {
+ static void __used __section(".discard.func_stack_frame_non_standard") \
+ *__func_stack_frame_non_standard_##func = func
+
++#define ASM_STACK_FRAME_NON_STANDARD(func) \
++ ".pushsection .discard.func_stack_frame_non_standard, \"aw\"\n" \
++ ".long " __stringify(func) " - .\n" \
++ ".popsection\n"
++
+ /*
+ * STACK_FRAME_NON_STANDARD_FP() is a frame-pointer-specific function ignore
+ * for the case where a function is intentionally missing frame pointer setup,
+@@ -138,6 +143,7 @@ struct unwind_hint {
+ #define UNWIND_HINT(sp_reg, sp_offset, type, end) \
+ "\n\t"
+ #define STACK_FRAME_NON_STANDARD(func)
++#define ASM_STACK_FRAME_NON_STANDARD(func)
+ #define STACK_FRAME_NON_STANDARD_FP(func)
+ #else
+ #define ANNOTATE_INTRA_FUNCTION_CALL
+diff --git a/kernel/cfi.c b/kernel/cfi.c
+index 9594cfd1cf2c..8d931089141b 100644
+--- a/kernel/cfi.c
++++ b/kernel/cfi.c
+@@ -278,12 +278,34 @@ static inline cfi_check_fn find_module_check_fn(unsigned long ptr)
+ return fn;
+ }
+
++extern unsigned long __start_cfi_excluded[];
++extern unsigned long __stop_cfi_excluded[];
++
++static inline bool is_cfi_excluded(unsigned long ptr)
++{
++ unsigned long *p = __start_cfi_excluded;
++
++ for ( ; p < __stop_cfi_excluded; ++p)
++ if (*p == ptr)
++ return true;
++
++ return false;
++}
++
++static void __cfi_pass(uint64_t id, void *ptr, void *diag)
++{
++}
++
+ static inline cfi_check_fn find_check_fn(unsigned long ptr)
+ {
+ cfi_check_fn fn = NULL;
+
+- if (is_kernel_text(ptr))
++ if (is_kernel_text(ptr)) {
++ if (unlikely(is_cfi_excluded(ptr)))
++ return __cfi_pass;
++
+ return __cfi_check;
++ }
+
+ /*
+ * Indirect call checks can happen when RCU is not watching. Both
+diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
+index 64ea283f2f86..8a0d463c8507 100644
+--- a/kernel/tracepoint.c
++++ b/kernel/tracepoint.c
+@@ -9,6 +9,7 @@
+ #include <linux/list.h>
+ #include <linux/rcupdate.h>
+ #include <linux/tracepoint.h>
++#include <linux/cfi.h>
+ #include <linux/err.h>
+ #include <linux/slab.h>
+ #include <linux/sched/signal.h>
+@@ -99,10 +100,7 @@ struct tp_probes {
+ };
+
+ /* Called in removal of a func but failed to allocate a new tp_funcs */
+-static void tp_stub_func(void)
+-{
+- return;
+-}
++static DEFINE_CFI_IMMEDIATE_RETURN_STUB(tp_stub_func);
+
+ static inline void *allocate_probes(int count)
+ {
+diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h
+index aca52db2f3f3..f3374fc3ff01 100644
+--- a/tools/include/linux/objtool.h
++++ b/tools/include/linux/objtool.h
+@@ -66,6 +66,11 @@ struct unwind_hint {
+ static void __used __section(".discard.func_stack_frame_non_standard") \
+ *__func_stack_frame_non_standard_##func = func
+
++#define ASM_STACK_FRAME_NON_STANDARD(func) \
++ ".pushsection .discard.func_stack_frame_non_standard, \"aw\"\n" \
++ ".long " __stringify(func) " - .\n" \
++ ".popsection\n"
++
+ /*
+ * STACK_FRAME_NON_STANDARD_FP() is a frame-pointer-specific function ignore
+ * for the case where a function is intentionally missing frame pointer setup,
+@@ -138,6 +143,7 @@ struct unwind_hint {
+ #define UNWIND_HINT(sp_reg, sp_offset, type, end) \
+ "\n\t"
+ #define STACK_FRAME_NON_STANDARD(func)
++#define ASM_STACK_FRAME_NON_STANDARD(func)
+ #define STACK_FRAME_NON_STANDARD_FP(func)
+ #else
+ #define ANNOTATE_INTRA_FUNCTION_CALL
+diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c
+index c10ef78df050..52436b0ae0d7 100644
+--- a/tools/objtool/arch/x86/decode.c
++++ b/tools/objtool/arch/x86/decode.c
+@@ -63,6 +63,23 @@ bool arch_callee_saved_reg(unsigned char reg)
+ }
+ }
+
++unsigned long arch_cfi_section_reloc_offset(struct reloc *reloc)
++{
++ if (!reloc->addend)
++ return 0;
++
++ if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)
++ return reloc->addend + 4;
++
++ return reloc->addend;
++}
++
++unsigned long arch_cfi_jump_reloc_offset(unsigned long offset)
++{
++ /* offset to the relocation in a jmp instruction */
++ return offset + 1;
++}
++
+ unsigned long arch_dest_reloc_offset(int addend)
+ {
+ return addend + 4;
+diff --git a/tools/objtool/check.c b/tools/objtool/check.c
+index c2d2ab9a2861..de98b9704ae4 100644
+--- a/tools/objtool/check.c
++++ b/tools/objtool/check.c
+@@ -3351,9 +3351,6 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio
+ if (!insn->func)
+ return false;
+
+- if (insn->func->static_call_tramp)
+- return true;
+-
+ /*
+ * CONFIG_UBSAN_TRAP inserts a UD2 when it sees
+ * __builtin_unreachable(). The BUG() macro has an unreachable() after
+diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c
+index 4b384c907027..177f53007788 100644
+--- a/tools/objtool/elf.c
++++ b/tools/objtool/elf.c
+@@ -18,6 +18,7 @@
+ #include <errno.h>
+ #include <objtool/builtin.h>
+
++#include <objtool/arch.h>
+ #include <objtool/elf.h>
+ #include <objtool/warn.h>
+
+@@ -290,6 +291,10 @@ static int read_sections(struct elf *elf)
+ if (sec->sh.sh_flags & SHF_EXECINSTR)
+ elf->text_size += sec->sh.sh_size;
+
++ /* Detect -fsanitize=cfi jump table sections */
++ if (!strncmp(sec->name, ".text..L.cfi.jumptable", 22))
++ sec->cfi_jt = true;
++
+ list_add_tail(&sec->list, &elf->sections);
+ elf_hash_add(section, &sec->hash, sec->idx);
+ elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name));
+@@ -577,6 +582,49 @@ static int read_rela_reloc(struct section *sec, int i, struct reloc *reloc, unsi
+ return 0;
+ }
+
++/*
++ * CONFIG_CFI_CLANG replaces function relocations to refer to an intermediate
++ * jump table. Undo the conversion so objtool can make sense of things.
++ */
++static int fix_cfi_relocs(const struct elf *elf)
++{
++ struct section *sec;
++ struct reloc *reloc;
++
++ list_for_each_entry(sec, &elf->sections, list) {
++ list_for_each_entry(reloc, &sec->reloc_list, list) {
++ struct reloc *cfi_reloc;
++ unsigned long offset;
++
++ if (!reloc->sym->sec->cfi_jt)
++ continue;
++
++ if (reloc->sym->type == STT_SECTION)
++ offset = arch_cfi_section_reloc_offset(reloc);
++ else
++ offset = reloc->sym->offset;
++
++ /*
++ * The jump table immediately jumps to the actual function,
++ * so look up the relocation there.
++ */
++ offset = arch_cfi_jump_reloc_offset(offset);
++ cfi_reloc = find_reloc_by_dest(elf, reloc->sym->sec, offset);
++
++ if (!cfi_reloc || !cfi_reloc->sym) {
++ WARN("can't find a CFI jump table relocation at %s+0x%lx",
++ reloc->sym->sec->name, offset);
++ return -1;
++ }
++
++ reloc->sym = cfi_reloc->sym;
++ reloc->addend = 0;
++ }
++ }
++
++ return 0;
++}
++
+ static int read_relocs(struct elf *elf)
+ {
+ struct section *sec;
+@@ -640,6 +688,9 @@ static int read_relocs(struct elf *elf)
+ tot_reloc += nr_reloc;
+ }
+
++ if (fix_cfi_relocs(elf))
++ return -1;
++
+ if (stats) {
+ printf("max_reloc: %lu\n", max_reloc);
+ printf("tot_reloc: %lu\n", tot_reloc);
+diff --git a/tools/objtool/include/objtool/arch.h b/tools/objtool/include/objtool/arch.h
+index 76bae3078286..86fbf9d84033 100644
+--- a/tools/objtool/include/objtool/arch.h
++++ b/tools/objtool/include/objtool/arch.h
+@@ -82,6 +82,9 @@ unsigned long arch_jump_destination(struct instruction *insn);
+
+ unsigned long arch_dest_reloc_offset(int addend);
+
++unsigned long arch_cfi_section_reloc_offset(struct reloc *reloc);
++unsigned long arch_cfi_jump_reloc_offset(unsigned long offset);
++
+ const char *arch_nop_insn(int len);
+ const char *arch_ret_insn(int len);
+
+diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h
+index d22336781401..31c9be55b10e 100644
+--- a/tools/objtool/include/objtool/elf.h
++++ b/tools/objtool/include/objtool/elf.h
+@@ -38,7 +38,7 @@ struct section {
+ Elf_Data *data;
+ char *name;
+ int idx;
+- bool changed, text, rodata, noinstr;
++ bool changed, text, rodata, noinstr, cfi_jt;
+ };
+
+ struct symbol {
+--
+2.35.0
+
+From 44aeb477991bdddef576e7eb93c5fb433b9ba48a Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sun, 23 Jan 2022 16:52:45 +0000
+Subject: [PATCH] clearlinux
+
+---
+ arch/x86/include/asm/topology.h | 1 +
+ arch/x86/include/asm/vdso/processor.h | 2 +-
+ arch/x86/kernel/alternative.c | 2 +
+ arch/x86/kernel/cpu/intel_epb.c | 4 ++
+ arch/x86/kernel/cpu/microcode/core.c | 9 +++-
+ arch/x86/kernel/cpu/microcode/intel.c | 59 ++++++++++++++++++++++--
+ arch/x86/kernel/itmt.c | 29 +++++++++++-
+ arch/x86/kernel/tsc.c | 3 ++
+ arch/x86/mm/fault.c | 4 +-
+ block/blk-core.c | 3 ++
+ drivers/Makefile | 15 +++---
+ drivers/ata/libahci.c | 4 +-
+ drivers/base/firmware_loader/main.c | 2 +
+ drivers/cpufreq/intel_pstate.c | 7 +++
+ drivers/idle/intel_idle.c | 44 +++++++++---------
+ drivers/input/serio/i8042.c | 10 ++--
+ drivers/net/dummy.c | 2 +-
+ drivers/nvme/host/core.c | 2 +-
+ drivers/pci/pci.c | 2 +-
+ drivers/powercap/intel_rapl_common.c | 2 +-
+ drivers/thermal/intel/intel_powerclamp.c | 10 ++++
+ fs/xattr.c | 15 +++---
+ include/linux/jbd2.h | 2 +-
+ include/linux/sched.h | 1 +
+ include/linux/wait.h | 2 +
+ include/uapi/linux/if_bonding.h | 2 +-
+ init/do_mounts.c | 16 ++++++-
+ kernel/locking/rwsem.c | 4 +-
+ kernel/module.c | 20 ++++++++
+ kernel/sched/debug.c | 1 +
+ kernel/sched/wait.c | 24 ++++++++++
+ kernel/watchdog.c | 2 +-
+ mm/compaction.c | 2 +-
+ mm/ksm.c | 11 +++--
+ mm/page_alloc.c | 5 +-
+ net/ipv4/inet_connection_sock.c | 2 +-
+ net/ipv4/tcp.c | 4 +-
+ 37 files changed, 258 insertions(+), 71 deletions(-)
+
+diff --git a/arch/x86/include/asm/topology.h b/arch/x86/include/asm/topology.h
+index 2f0b6be8eaab..c31f81e2ea05 100644
+--- a/arch/x86/include/asm/topology.h
++++ b/arch/x86/include/asm/topology.h
+@@ -174,6 +174,7 @@ extern unsigned int __read_mostly sysctl_sched_itmt_enabled;
+
+ /* Interface to set priority of a cpu */
+ void sched_set_itmt_core_prio(int prio, int core_cpu);
++void sched_set_itmt_power_ratio(int power_ratio, int core_cpu);
+
+ /* Interface to notify scheduler that system supports ITMT */
+ int sched_set_itmt_support(void);
+diff --git a/arch/x86/include/asm/vdso/processor.h b/arch/x86/include/asm/vdso/processor.h
+index 57b1a7034c64..e2c45674f989 100644
+--- a/arch/x86/include/asm/vdso/processor.h
++++ b/arch/x86/include/asm/vdso/processor.h
+@@ -10,7 +10,7 @@
+ /* REP NOP (PAUSE) is a good thing to insert into busy-wait loops. */
+ static __always_inline void rep_nop(void)
+ {
+- asm volatile("rep; nop" ::: "memory");
++ asm volatile("lfence" ::: "memory");
+ }
+
+ static __always_inline void cpu_relax(void)
+diff --git a/arch/x86/kernel/alternative.c b/arch/x86/kernel/alternative.c
+index 5007c3ffe96f..c8d04206b2e9 100644
+--- a/arch/x86/kernel/alternative.c
++++ b/arch/x86/kernel/alternative.c
+@@ -829,7 +829,9 @@ void __init alternative_instructions(void)
+ * Then patch alternatives, such that those paravirt calls that are in
+ * alternatives can be overwritten by their immediate fragments.
+ */
++ printk("clr: Applying alternatives\n");
+ apply_alternatives(__alt_instructions, __alt_instructions_end);
++ printk("clr: Applying alternatives done\n");
+
+ #ifdef CONFIG_SMP
+ /* Patch to UP if other cpus not imminent. */
+diff --git a/arch/x86/kernel/cpu/intel_epb.c b/arch/x86/kernel/cpu/intel_epb.c
+index fbaf12e43f41..c8c2d6f1a8ac 100644
+--- a/arch/x86/kernel/cpu/intel_epb.c
++++ b/arch/x86/kernel/cpu/intel_epb.c
+@@ -166,6 +166,10 @@ static ssize_t energy_perf_bias_store(struct device *dev,
+ if (ret < 0)
+ return ret;
+
++ /* update the ITMT scheduler logic to use the power policy data */
++ /* scale the val up by 2 so the range is 224 - 256 */
++ sched_set_itmt_power_ratio(256 - val * 2, cpu);
++
+ return count;
+ }
+
+diff --git a/arch/x86/kernel/cpu/microcode/core.c b/arch/x86/kernel/cpu/microcode/core.c
+index f955d25076ba..fb3925d1f788 100644
+--- a/arch/x86/kernel/cpu/microcode/core.c
++++ b/arch/x86/kernel/cpu/microcode/core.c
+@@ -587,6 +587,7 @@ static int microcode_reload_late(void)
+ atomic_set(&late_cpus_in, 0);
+ atomic_set(&late_cpus_out, 0);
+
++ printk ("Going to do stop_machine\n");
+ ret = stop_machine_cpuslocked(__reload_late, NULL, cpu_online_mask);
+ if (ret == 0)
+ microcode_check();
+@@ -600,18 +601,23 @@ static ssize_t reload_store(struct device *dev,
+ struct device_attribute *attr,
+ const char *buf, size_t size)
+ {
++ extern bool force_ucode_load;
+ enum ucode_state tmp_ret = UCODE_OK;
+ int bsp = boot_cpu_data.cpu_index;
+ unsigned long val;
++ bool orig_cmd_line = force_ucode_load;
+ ssize_t ret = 0;
+
+ ret = kstrtoul(buf, 0, &val);
+ if (ret)
+ return ret;
+
+- if (val != 1)
++ if (!val || val > 2)
+ return size;
+
++ if (val == 2)
++ force_ucode_load = true;
++
+ cpus_read_lock();
+
+ ret = check_online_cpus();
+@@ -627,6 +633,7 @@ static ssize_t reload_store(struct device *dev,
+ mutex_unlock(&microcode_mutex);
+
+ put:
++ force_ucode_load = orig_cmd_line;
+ cpus_read_unlock();
+
+ if (ret == 0)
+diff --git a/arch/x86/kernel/cpu/microcode/intel.c b/arch/x86/kernel/cpu/microcode/intel.c
+index d28a9f8f3fec..4cd155b0a42f 100644
+--- a/arch/x86/kernel/cpu/microcode/intel.c
++++ b/arch/x86/kernel/cpu/microcode/intel.c
+@@ -30,6 +30,7 @@
+ #include <linux/uio.h>
+ #include <linux/mm.h>
+
++#include <asm/cmdline.h>
+ #include <asm/microcode_intel.h>
+ #include <asm/intel-family.h>
+ #include <asm/processor.h>
+@@ -38,6 +39,7 @@
+ #include <asm/msr.h>
+
+ static const char ucode_path[] = "kernel/x86/microcode/GenuineIntel.bin";
++bool force_ucode_load = false;
+
+ /* Current microcode patch used in early patching on the APs. */
+ static struct microcode_intel *intel_ucode_patch;
+@@ -94,8 +96,18 @@ static int has_newer_microcode(void *mc, unsigned int csig, int cpf, int new_rev
+ {
+ struct microcode_header_intel *mc_hdr = mc;
+
+- if (mc_hdr->rev <= new_rev)
++ //if (mc_hdr->rev <= new_rev)
++ if (mc_hdr->rev < new_rev) {
++ printk ("Returning NO_NEW old = 0x%x new = 0x%x\n",
++ mc_hdr->rev, new_rev);
+ return 0;
++ }
++ if ((mc_hdr->rev == new_rev) && !force_ucode_load) {
++ printk ("SAME REV: no_force Returning NO_NEW old = 0x%x new = 0x%x\n",
++ mc_hdr->rev, new_rev);
++ return 0;
++ }
++ printk ("ucode: force loading same rev\n");
+
+ return find_matching_signature(mc, csig, cpf);
+ }
+@@ -557,11 +569,20 @@ static int apply_microcode_early(struct ucode_cpu_info *uci, bool early)
+ * already.
+ */
+ rev = intel_get_microcode_revision();
+- if (rev >= mc->hdr.rev) {
++ if (rev > mc->hdr.rev) {
+ uci->cpu_sig.rev = rev;
+ return UCODE_OK;
+ }
+
++ if (rev == mc->hdr.rev) {
++ if (!force_ucode_load) {
++ printk ("Matching ucode rev, no update\n");
++ return UCODE_OK;
++ } else {
++ printk ("Matching ucode rev.. force updating\n");
++ }
++ }
++
+ /*
+ * Writeback and invalidate caches before updating microcode to avoid
+ * internal issues depending on what the microcode is updating.
+@@ -613,6 +634,29 @@ int __init save_microcode_in_initrd_intel(void)
+ return 0;
+ }
+
++static bool check_force_ucode_bsp(void)
++{
++ static const char *__force_ucode_str = "force_ucode_load";
++
++#ifdef CONFIG_X86_32
++ const char *cmdline = (const char *)__pa_nodebug(boot_command_line);
++ const char *option = (const char *)__pa_nodebug(__force_ucode_str);
++ bool *res = (bool *)__pa_nodebug(&force_ucode_load);
++
++#else /* CONFIG_X86_64 */
++ const char *cmdline = boot_command_line;
++ const char *option = __force_ucode_str;
++ bool *res = &force_ucode_load;
++#endif
++
++ if (cmdline_find_option_bool(cmdline, option)) {
++ printk("cmdline forcing ucode update for same rev\n");
++ *res = true;
++ }
++
++ return *res;
++}
++
+ /*
+ * @res_patch, output: a pointer to the patch we found.
+ */
+@@ -646,6 +690,9 @@ void __init load_ucode_intel_bsp(void)
+ {
+ struct microcode_intel *patch;
+ struct ucode_cpu_info uci;
++ bool force_bsp;
++
++ force_bsp = check_force_ucode_bsp();
+
+ patch = __load_ucode_intel(&uci);
+ if (!patch)
+@@ -694,8 +741,12 @@ static struct microcode_intel *find_patch(struct ucode_cpu_info *uci)
+
+ phdr = (struct microcode_header_intel *)iter->data;
+
+- if (phdr->rev <= uci->cpu_sig.rev)
++ if (phdr->rev < uci->cpu_sig.rev)
+ continue;
++ if (phdr->rev == uci->cpu_sig.rev && !force_ucode_load)
++ continue;
++ else
++ printk ("same rev forcing ucode\n");
+
+ if (!find_matching_signature(phdr,
+ uci->cpu_sig.sig,
+@@ -779,7 +830,7 @@ static enum ucode_state apply_microcode_intel(int cpu)
+ * already.
+ */
+ rev = intel_get_microcode_revision();
+- if (rev >= mc->hdr.rev) {
++ if (rev > mc->hdr.rev || (rev == mc->hdr.rev && !force_ucode_load)) {
+ ret = UCODE_OK;
+ goto out;
+ }
+diff --git a/arch/x86/kernel/itmt.c b/arch/x86/kernel/itmt.c
+index 9ff480e94511..d4326e050fb7 100644
+--- a/arch/x86/kernel/itmt.c
++++ b/arch/x86/kernel/itmt.c
+@@ -25,6 +25,7 @@
+
+ static DEFINE_MUTEX(itmt_update_mutex);
+ DEFINE_PER_CPU_READ_MOSTLY(int, sched_core_priority);
++DEFINE_PER_CPU_READ_MOSTLY(int, sched_power_ratio);
+
+ /* Boolean to track if system has ITMT capabilities */
+ static bool __read_mostly sched_itmt_capable;
+@@ -169,7 +170,12 @@ void sched_clear_itmt_support(void)
+
+ int arch_asym_cpu_priority(int cpu)
+ {
+- return per_cpu(sched_core_priority, cpu);
++ int power_ratio = per_cpu(sched_power_ratio, cpu);
++
++ /* a power ratio of 0 (uninitialized) is assumed to be maximum */
++ if (power_ratio == 0)
++ power_ratio = 256 - 2 * 6;
++ return per_cpu(sched_core_priority, cpu) * power_ratio / 256;
+ }
+
+ /**
+@@ -203,3 +209,24 @@ void sched_set_itmt_core_prio(int prio, int core_cpu)
+ i++;
+ }
+ }
++
++/**
++ * sched_set_itmt_power_ratio() - Set CPU priority based on ITMT
++ * @power_ratio: The power scaling ratio [1..256] for the core
++ * @core_cpu: The cpu number associated with the core
++ *
++ * Set a scaling to the cpu performance based on long term power
++ * settings (like EPB).
++ *
++ * Note this is for the policy not for the actual dynamic frequency;
++ * the frequency will increase itself as workloads run on a core.
++ */
++
++void sched_set_itmt_power_ratio(int power_ratio, int core_cpu)
++{
++ int cpu;
++
++ for_each_cpu(cpu, topology_sibling_cpumask(core_cpu)) {
++ per_cpu(sched_power_ratio, cpu) = power_ratio;
++ }
++}
+diff --git a/arch/x86/kernel/tsc.c b/arch/x86/kernel/tsc.c
+index a698196377be..5f3ee7c31c8a 100644
+--- a/arch/x86/kernel/tsc.c
++++ b/arch/x86/kernel/tsc.c
+@@ -1569,6 +1569,9 @@ unsigned long calibrate_delay_is_known(void)
+ if (!constant_tsc || !mask)
+ return 0;
+
++ if (cpu != 0)
++ return cpu_data(0).loops_per_jiffy;
++
+ sibling = cpumask_any_but(mask, cpu);
+ if (sibling < nr_cpu_ids)
+ return cpu_data(sibling).loops_per_jiffy;
+diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
+index d0074c6ed31a..aeeae77fe5dd 100644
+--- a/arch/x86/mm/fault.c
++++ b/arch/x86/mm/fault.c
+@@ -776,9 +776,9 @@ show_signal_msg(struct pt_regs *regs, unsigned long error_code,
+ if (!printk_ratelimit())
+ return;
+
+- printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx",
++ printk("%s%s[%d]: segfault at %lx ip %px sp %px error %lx cpu %i",
+ loglvl, tsk->comm, task_pid_nr(tsk), address,
+- (void *)regs->ip, (void *)regs->sp, error_code);
++ (void *)regs->ip, (void *)regs->sp, error_code, raw_smp_processor_id());
+
+ print_vma_addr(KERN_CONT " in ", regs->ip);
+
+diff --git a/block/blk-core.c b/block/blk-core.c
+index 97f8bc8d3a79..004628768c61 100644
+--- a/block/blk-core.c
++++ b/block/blk-core.c
+@@ -946,6 +946,9 @@ void submit_bio(struct bio *bio)
+ } else {
+ task_io_account_read(bio->bi_iter.bi_size);
+ count_vm_events(PGPGIN, count);
++
++ if (bio->bi_opf & REQ_PREFLUSH)
++ current->fsync_count++;
+ }
+ }
+
+diff --git a/drivers/Makefile b/drivers/Makefile
+index a110338c860c..f91099276a78 100644
+--- a/drivers/Makefile
++++ b/drivers/Makefile
+@@ -59,15 +59,8 @@ obj-y += char/
+ # iommu/ comes before gpu as gpu are using iommu controllers
+ obj-y += iommu/
+
+-# gpu/ comes after char for AGP vs DRM startup and after iommu
+-obj-y += gpu/
+-
+ obj-$(CONFIG_CONNECTOR) += connector/
+
+-# i810fb and intelfb depend on char/agp/
+-obj-$(CONFIG_FB_I810) += video/fbdev/i810/
+-obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/
+-
+ obj-$(CONFIG_PARPORT) += parport/
+ obj-y += base/ block/ misc/ mfd/ nfc/
+ obj-$(CONFIG_LIBNVDIMM) += nvdimm/
+@@ -79,6 +72,14 @@ obj-y += macintosh/
+ obj-y += scsi/
+ obj-y += nvme/
+ obj-$(CONFIG_ATA) += ata/
++
++# gpu/ comes after char for AGP vs DRM startup and after iommu
++obj-y += gpu/
++
++# i810fb and intelfb depend on char/agp/
++obj-$(CONFIG_FB_I810) += video/fbdev/i810/
++obj-$(CONFIG_FB_INTEL) += video/fbdev/intelfb/
++
+ obj-$(CONFIG_TARGET_CORE) += target/
+ obj-$(CONFIG_MTD) += mtd/
+ obj-$(CONFIG_SPI) += spi/
+diff --git a/drivers/ata/libahci.c b/drivers/ata/libahci.c
+index 0ed484e04fd6..bbcb54da7a31 100644
+--- a/drivers/ata/libahci.c
++++ b/drivers/ata/libahci.c
+@@ -33,14 +33,14 @@
+ #include "libata.h"
+
+ static int ahci_skip_host_reset;
+-int ahci_ignore_sss;
++int ahci_ignore_sss=1;
+ EXPORT_SYMBOL_GPL(ahci_ignore_sss);
+
+ module_param_named(skip_host_reset, ahci_skip_host_reset, int, 0444);
+ MODULE_PARM_DESC(skip_host_reset, "skip global host reset (0=don't skip, 1=skip)");
+
+ module_param_named(ignore_sss, ahci_ignore_sss, int, 0444);
+-MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore)");
++MODULE_PARM_DESC(ignore_sss, "Ignore staggered spinup flag (0=don't ignore, 1=ignore [default])");
+
+ static int ahci_set_lpm(struct ata_link *link, enum ata_lpm_policy policy,
+ unsigned hints);
+diff --git a/drivers/base/firmware_loader/main.c b/drivers/base/firmware_loader/main.c
+index 94d1789a233e..678b81768bd9 100644
+--- a/drivers/base/firmware_loader/main.c
++++ b/drivers/base/firmware_loader/main.c
+@@ -407,6 +407,8 @@ static int fw_decompress_xz(struct device *dev, struct fw_priv *fw_priv,
+ static char fw_path_para[256];
+ static const char * const fw_path[] = {
+ fw_path_para,
++ "/etc/firmware/" UTS_RELEASE,
++ "/etc/firmware",
+ "/lib/firmware/updates/" UTS_RELEASE,
+ "/lib/firmware/updates",
+ "/lib/firmware/" UTS_RELEASE,
+diff --git a/drivers/cpufreq/intel_pstate.c b/drivers/cpufreq/intel_pstate.c
+index bc7f7e6759bd..ee33ad7f6f28 100644
+--- a/drivers/cpufreq/intel_pstate.c
++++ b/drivers/cpufreq/intel_pstate.c
+@@ -364,6 +364,13 @@ static void intel_pstate_set_itmt_prio(int cpu)
+ * update them at any time after it has been called.
+ */
+ sched_set_itmt_core_prio(cppc_perf.highest_perf, cpu);
++ /*
++ * On some systems with overclocking enabled, CPPC.highest_perf is hardcoded to 0xff.
++ * In this case we can't use CPPC.highest_perf to enable ITMT.
++ * In this case we can look at MSR_HWP_CAPABILITIES bits [8:0] to decide.
++ */
++ if (cppc_perf.highest_perf == 0xff)
++ cppc_perf.highest_perf = HWP_HIGHEST_PERF(READ_ONCE(all_cpu_data[cpu]->hwp_cap_cached));
+
+ if (max_highest_perf <= min_highest_perf) {
+ if (cppc_perf.highest_perf > max_highest_perf)
+diff --git a/drivers/idle/intel_idle.c b/drivers/idle/intel_idle.c
+index 0b66e25c0e2d..406b41b387d5 100644
+--- a/drivers/idle/intel_idle.c
++++ b/drivers/idle/intel_idle.c
+@@ -511,7 +511,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {
+ .desc = "MWAIT 0x01",
+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
+ .exit_latency = 10,
+- .target_residency = 20,
++ .target_residency = 120,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -519,7 +519,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {
+ .desc = "MWAIT 0x10",
+ .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 33,
+- .target_residency = 100,
++ .target_residency = 900,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -527,7 +527,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {
+ .desc = "MWAIT 0x20",
+ .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 133,
+- .target_residency = 400,
++ .target_residency = 1000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -535,7 +535,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {
+ .desc = "MWAIT 0x32",
+ .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 166,
+- .target_residency = 500,
++ .target_residency = 1500,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -543,7 +543,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {
+ .desc = "MWAIT 0x40",
+ .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 300,
+- .target_residency = 900,
++ .target_residency = 2000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -551,7 +551,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {
+ .desc = "MWAIT 0x50",
+ .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 600,
+- .target_residency = 1800,
++ .target_residency = 5000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -559,7 +559,7 @@ static struct cpuidle_state hsw_cstates[] __initdata = {
+ .desc = "MWAIT 0x60",
+ .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 2600,
+- .target_residency = 7700,
++ .target_residency = 9000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -579,7 +579,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {
+ .desc = "MWAIT 0x01",
+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
+ .exit_latency = 10,
+- .target_residency = 20,
++ .target_residency = 120,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -587,7 +587,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {
+ .desc = "MWAIT 0x10",
+ .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 40,
+- .target_residency = 100,
++ .target_residency = 1000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -595,7 +595,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {
+ .desc = "MWAIT 0x20",
+ .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 133,
+- .target_residency = 400,
++ .target_residency = 1000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -603,7 +603,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {
+ .desc = "MWAIT 0x32",
+ .flags = MWAIT2flg(0x32) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 166,
+- .target_residency = 500,
++ .target_residency = 2000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -611,7 +611,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {
+ .desc = "MWAIT 0x40",
+ .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 300,
+- .target_residency = 900,
++ .target_residency = 4000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -619,7 +619,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {
+ .desc = "MWAIT 0x50",
+ .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 600,
+- .target_residency = 1800,
++ .target_residency = 7000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -627,7 +627,7 @@ static struct cpuidle_state bdw_cstates[] __initdata = {
+ .desc = "MWAIT 0x60",
+ .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 2600,
+- .target_residency = 7700,
++ .target_residency = 9000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -648,7 +648,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {
+ .desc = "MWAIT 0x01",
+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
+ .exit_latency = 10,
+- .target_residency = 20,
++ .target_residency = 120,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -656,7 +656,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {
+ .desc = "MWAIT 0x10",
+ .flags = MWAIT2flg(0x10) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 70,
+- .target_residency = 100,
++ .target_residency = 1000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -664,7 +664,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {
+ .desc = "MWAIT 0x20",
+ .flags = MWAIT2flg(0x20) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 85,
+- .target_residency = 200,
++ .target_residency = 600,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -672,7 +672,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {
+ .desc = "MWAIT 0x33",
+ .flags = MWAIT2flg(0x33) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 124,
+- .target_residency = 800,
++ .target_residency = 3000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -680,7 +680,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {
+ .desc = "MWAIT 0x40",
+ .flags = MWAIT2flg(0x40) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 200,
+- .target_residency = 800,
++ .target_residency = 3200,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -688,7 +688,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {
+ .desc = "MWAIT 0x50",
+ .flags = MWAIT2flg(0x50) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 480,
+- .target_residency = 5000,
++ .target_residency = 9000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -696,7 +696,7 @@ static struct cpuidle_state skl_cstates[] __initdata = {
+ .desc = "MWAIT 0x60",
+ .flags = MWAIT2flg(0x60) | CPUIDLE_FLAG_TLB_FLUSHED,
+ .exit_latency = 890,
+- .target_residency = 5000,
++ .target_residency = 9000,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+@@ -717,7 +717,7 @@ static struct cpuidle_state skx_cstates[] __initdata = {
+ .desc = "MWAIT 0x01",
+ .flags = MWAIT2flg(0x01) | CPUIDLE_FLAG_ALWAYS_ENABLE,
+ .exit_latency = 10,
+- .target_residency = 20,
++ .target_residency = 300,
+ .enter = &intel_idle,
+ .enter_s2idle = intel_idle_s2idle, },
+ {
+diff --git a/drivers/input/serio/i8042.c b/drivers/input/serio/i8042.c
+index 3fc0a89cc785..a7c103f9dfd3 100644
+--- a/drivers/input/serio/i8042.c
++++ b/drivers/input/serio/i8042.c
+@@ -621,7 +621,7 @@ static int i8042_enable_kbd_port(void)
+ if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) {
+ i8042_ctr &= ~I8042_CTR_KBDINT;
+ i8042_ctr |= I8042_CTR_KBDDIS;
+- pr_err("Failed to enable KBD port\n");
++ pr_info("Failed to enable KBD port\n");
+ return -EIO;
+ }
+
+@@ -640,7 +640,7 @@ static int i8042_enable_aux_port(void)
+ if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) {
+ i8042_ctr &= ~I8042_CTR_AUXINT;
+ i8042_ctr |= I8042_CTR_AUXDIS;
+- pr_err("Failed to enable AUX port\n");
++ pr_info("Failed to enable AUX port\n");
+ return -EIO;
+ }
+
+@@ -732,7 +732,7 @@ static int i8042_check_mux(void)
+ i8042_ctr &= ~I8042_CTR_AUXINT;
+
+ if (i8042_command(&i8042_ctr, I8042_CMD_CTL_WCTR)) {
+- pr_err("Failed to disable AUX port, can't use MUX\n");
++ pr_info("Failed to disable AUX port, can't use MUX\n");
+ return -EIO;
+ }
+
+@@ -955,7 +955,7 @@ static int i8042_controller_selftest(void)
+ do {
+
+ if (i8042_command(&param, I8042_CMD_CTL_TEST)) {
+- pr_err("i8042 controller selftest timeout\n");
++ pr_info("i8042 controller selftest timeout\n");
+ return -ENODEV;
+ }
+
+@@ -977,7 +977,7 @@ static int i8042_controller_selftest(void)
+ pr_info("giving up on controller selftest, continuing anyway...\n");
+ return 0;
+ #else
+- pr_err("i8042 controller selftest failed\n");
++ pr_info("i8042 controller selftest failed\n");
+ return -EIO;
+ #endif
+ }
+diff --git a/drivers/net/dummy.c b/drivers/net/dummy.c
+index f82ad7419508..5e8faa70aad6 100644
+--- a/drivers/net/dummy.c
++++ b/drivers/net/dummy.c
+@@ -43,7 +43,7 @@
+
+ #define DRV_NAME "dummy"
+
+-static int numdummies = 1;
++static int numdummies = 0;
+
+ /* fake multicast ability */
+ static void set_multicast_list(struct net_device *dev)
+diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
+index 5e0bfda04bd7..3024a992d5ac 100644
+--- a/drivers/nvme/host/core.c
++++ b/drivers/nvme/host/core.c
+@@ -48,7 +48,7 @@ static u8 nvme_max_retries = 5;
+ module_param_named(max_retries, nvme_max_retries, byte, 0644);
+ MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
+
+-static unsigned long default_ps_max_latency_us = 100000;
++static unsigned long default_ps_max_latency_us = 200;
+ module_param(default_ps_max_latency_us, ulong, 0644);
+ MODULE_PARM_DESC(default_ps_max_latency_us,
+ "max power saving latency for new devices; use PM QOS to change per device");
+diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c
+index 9ecce435fb3f..796f280eccfe 100644
+--- a/drivers/pci/pci.c
++++ b/drivers/pci/pci.c
+@@ -60,7 +60,7 @@ struct pci_pme_device {
+ struct pci_dev *dev;
+ };
+
+-#define PME_TIMEOUT 1000 /* How long between PME checks */
++#define PME_TIMEOUT 4000 /* How long between PME checks */
+
+ static void pci_dev_d3_sleep(struct pci_dev *dev)
+ {
+diff --git a/drivers/powercap/intel_rapl_common.c b/drivers/powercap/intel_rapl_common.c
+index 07611a00b78f..4031d810def5 100644
+--- a/drivers/powercap/intel_rapl_common.c
++++ b/drivers/powercap/intel_rapl_common.c
+@@ -1512,7 +1512,7 @@ static int __init rapl_init(void)
+
+ id = x86_match_cpu(rapl_ids);
+ if (!id) {
+- pr_err("driver does not support CPU family %d model %d\n",
++ pr_info("driver does not support CPU family %d model %d\n",
+ boot_cpu_data.x86, boot_cpu_data.x86_model);
+
+ return -ENODEV;
+diff --git a/drivers/thermal/intel/intel_powerclamp.c b/drivers/thermal/intel/intel_powerclamp.c
+index 14256421d98c..8df2e604ceef 100644
+--- a/drivers/thermal/intel/intel_powerclamp.c
++++ b/drivers/thermal/intel/intel_powerclamp.c
+@@ -647,6 +647,11 @@ static const struct thermal_cooling_device_ops powerclamp_cooling_ops = {
+ .set_cur_state = powerclamp_set_cur_state,
+ };
+
++static const struct x86_cpu_id amd_cpu[] = {
++ { X86_VENDOR_AMD },
++ {},
++};
++
+ static const struct x86_cpu_id __initconst intel_powerclamp_ids[] = {
+ X86_MATCH_VENDOR_FEATURE(INTEL, X86_FEATURE_MWAIT, NULL),
+ {}
+@@ -656,6 +661,11 @@ MODULE_DEVICE_TABLE(x86cpu, intel_powerclamp_ids);
+ static int __init powerclamp_probe(void)
+ {
+
++ if (x86_match_cpu(amd_cpu)){
++ pr_info("Intel PowerClamp does not support AMD CPUs\n");
++ return -ENODEV;
++ }
++
+ if (!x86_match_cpu(intel_powerclamp_ids)) {
+ pr_err("CPU does not support MWAIT\n");
+ return -ENODEV;
+diff --git a/fs/xattr.c b/fs/xattr.c
+index 5c8c5175b385..0f98df2ea2e2 100644
+--- a/fs/xattr.c
++++ b/fs/xattr.c
+@@ -120,16 +120,17 @@ xattr_permission(struct user_namespace *mnt_userns, struct inode *inode,
+ }
+
+ /*
+- * In the user.* namespace, only regular files and directories can have
+- * extended attributes. For sticky directories, only the owner and
+- * privileged users can write attributes.
++ * In the user.* namespace, only regular files, symbolic links, and
++ * directories can have extended attributes. For symbolic links and
++ * sticky directories, only the owner and privileged users can write
++ * attributes.
+ */
+ if (!strncmp(name, XATTR_USER_PREFIX, XATTR_USER_PREFIX_LEN)) {
+- if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode))
++ if (!S_ISREG(inode->i_mode) && !S_ISDIR(inode->i_mode) && !S_ISLNK(inode->i_mode))
+ return (mask & MAY_WRITE) ? -EPERM : -ENODATA;
+- if (S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX) &&
+- (mask & MAY_WRITE) &&
+- !inode_owner_or_capable(mnt_userns, inode))
++ if (((S_ISDIR(inode->i_mode) && (inode->i_mode & S_ISVTX))
++ || S_ISLNK(inode->i_mode)) && (mask & MAY_WRITE)
++ && !inode_owner_or_capable(mnt_userns, inode))
+ return -EPERM;
+ }
+
+diff --git a/include/linux/jbd2.h b/include/linux/jbd2.h
+index fd933c45281a..e70409a77cfa 100644
+--- a/include/linux/jbd2.h
++++ b/include/linux/jbd2.h
+@@ -45,7 +45,7 @@
+ /*
+ * The default maximum commit age, in seconds.
+ */
+-#define JBD2_DEFAULT_MAX_COMMIT_AGE 5
++#define JBD2_DEFAULT_MAX_COMMIT_AGE 30
+
+ #ifdef CONFIG_JBD2_DEBUG
+ /*
+diff --git a/include/linux/sched.h b/include/linux/sched.h
+index 508b91d57470..c1b38df8ac72 100644
+--- a/include/linux/sched.h
++++ b/include/linux/sched.h
+@@ -1050,6 +1050,7 @@ struct task_struct {
+ /* Cached requested key. */
+ struct key *cached_requested_key;
+ #endif
++ int fsync_count;
+
+ /*
+ * executable name, excluding path.
+diff --git a/include/linux/wait.h b/include/linux/wait.h
+index 851e07da2583..85653fc33274 100644
+--- a/include/linux/wait.h
++++ b/include/linux/wait.h
+@@ -165,6 +165,7 @@ static inline bool wq_has_sleeper(struct wait_queue_head *wq_head)
+
+ extern void add_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
+ extern void add_wait_queue_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
++extern void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
+ extern void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
+ extern void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
+
+@@ -1163,6 +1164,7 @@ do { \
+ */
+ void prepare_to_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
+ bool prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
++void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
+ long prepare_to_wait_event(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state);
+ void finish_wait(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry);
+ long wait_woken(struct wait_queue_entry *wq_entry, unsigned mode, long timeout);
+diff --git a/include/uapi/linux/if_bonding.h b/include/uapi/linux/if_bonding.h
+index d174914a837d..bf8e2af101a3 100644
+--- a/include/uapi/linux/if_bonding.h
++++ b/include/uapi/linux/if_bonding.h
+@@ -82,7 +82,7 @@
+ #define BOND_STATE_ACTIVE 0 /* link is active */
+ #define BOND_STATE_BACKUP 1 /* link is backup */
+
+-#define BOND_DEFAULT_MAX_BONDS 1 /* Default maximum number of devices to support */
++#define BOND_DEFAULT_MAX_BONDS 0 /* Default maximum number of devices to support */
+
+ #define BOND_DEFAULT_TX_QUEUES 16 /* Default number of tx queues per device */
+
+diff --git a/init/do_mounts.c b/init/do_mounts.c
+index 762b534978d9..107b96927049 100644
+--- a/init/do_mounts.c
++++ b/init/do_mounts.c
+@@ -284,8 +284,18 @@ dev_t name_to_dev_t(const char *name)
+ if (strcmp(name, "/dev/ram") == 0)
+ return Root_RAM0;
+ #ifdef CONFIG_BLOCK
+- if (strncmp(name, "PARTUUID=", 9) == 0)
+- return devt_from_partuuid(name + 9);
++ if (strncmp(name, "PARTUUID=", 9) == 0) {
++ dev_t res;
++ int needtowait = 40<<1;
++ res = devt_from_partuuid(name + 9);
++ while (!res && needtowait) {
++ /* waiting 0.5 sec */
++ msleep(500);
++ res = devt_from_partuuid(name + 9);
++ needtowait--;
++ }
++ return res;
++ }
+ if (strncmp(name, "PARTLABEL=", 10) == 0)
+ return devt_from_partlabel(name + 10);
+ if (strncmp(name, "/dev/", 5) == 0)
+@@ -613,7 +623,9 @@ void __init prepare_namespace(void)
+ * For example, it is not atypical to wait 5 seconds here
+ * for the touchpad of a laptop to initialize.
+ */
++ async_synchronize_full();
+ wait_for_device_probe();
++ async_synchronize_full();
+
+ md_run_setup();
+
+diff --git a/kernel/locking/rwsem.c b/kernel/locking/rwsem.c
+index 69aba4abe104..b4818ccad5de 100644
+--- a/kernel/locking/rwsem.c
++++ b/kernel/locking/rwsem.c
+@@ -707,6 +707,7 @@ rwsem_spin_on_owner(struct rw_semaphore *sem)
+ struct task_struct *new, *owner;
+ unsigned long flags, new_flags;
+ enum owner_state state;
++ int i = 0;
+
+ lockdep_assert_preemption_disabled();
+
+@@ -743,7 +744,8 @@ rwsem_spin_on_owner(struct rw_semaphore *sem)
+ break;
+ }
+
+- cpu_relax();
++ if (i++ > 1000)
++ cpu_relax();
+ }
+
+ return state;
+diff --git a/kernel/module.c b/kernel/module.c
+index 24dab046e16c..2ac7eea9f93b 100644
+--- a/kernel/module.c
++++ b/kernel/module.c
+@@ -57,6 +57,7 @@
+ #include <linux/bsearch.h>
+ #include <linux/dynamic_debug.h>
+ #include <linux/audit.h>
++#include <linux/efi.h>
+ #include <uapi/linux/module.h>
+ #include "module-internal.h"
+
+@@ -270,6 +271,10 @@ static void module_assert_mutex_or_preempt(void)
+ #ifdef CONFIG_MODULE_SIG
+ static bool sig_enforce = IS_ENABLED(CONFIG_MODULE_SIG_FORCE);
+ module_param(sig_enforce, bool_enable_only, 0644);
++/* Allow disabling module signature requirement by adding boot param */
++static bool sig_unenforce = false;
++module_param(sig_unenforce, bool_enable_only, 0644);
++
+
+ void set_module_sig_enforced(void)
+ {
+@@ -416,6 +421,8 @@ extern const struct kernel_symbol __stop___ksymtab_gpl[];
+ extern const s32 __start___kcrctab[];
+ extern const s32 __start___kcrctab_gpl[];
+
++extern struct boot_params boot_params;
++
+ #ifndef CONFIG_MODVERSIONS
+ #define symversion(base, idx) NULL
+ #else
+@@ -4676,6 +4683,19 @@ static const struct proc_ops modules_proc_ops = {
+ static int __init proc_modules_init(void)
+ {
+ proc_create("modules", 0, NULL, &modules_proc_ops);
++
++#ifdef CONFIG_MODULE_SIG_FORCE
++ switch (boot_params.secure_boot) {
++ case efi_secureboot_mode_unset:
++ case efi_secureboot_mode_unknown:
++ case efi_secureboot_mode_disabled:
++ /*
++ * sig_unenforce is only applied if SecureBoot is not
++ * enabled.
++ */
++ sig_enforce = !sig_unenforce;
++ }
++#endif
+ return 0;
+ }
+ module_init(proc_modules_init);
+diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
+index aa29211de1bf..22bbc3687cdf 100644
+--- a/kernel/sched/debug.c
++++ b/kernel/sched/debug.c
+@@ -970,6 +970,7 @@ void proc_sched_show_task(struct task_struct *p, struct pid_namespace *ns,
+ PN(se.exec_start);
+ PN(se.vruntime);
+ PN(se.sum_exec_runtime);
++ P(fsync_count);
+
+ nr_switches = p->nvcsw + p->nivcsw;
+
+diff --git a/kernel/sched/wait.c b/kernel/sched/wait.c
+index eca38107b32f..0306fa23b4f8 100644
+--- a/kernel/sched/wait.c
++++ b/kernel/sched/wait.c
+@@ -48,6 +48,17 @@ void add_wait_queue_priority(struct wait_queue_head *wq_head, struct wait_queue_
+ }
+ EXPORT_SYMBOL_GPL(add_wait_queue_priority);
+
++void add_wait_queue_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
++{
++ unsigned long flags;
++
++ wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
++ spin_lock_irqsave(&wq_head->lock, flags);
++ __add_wait_queue(wq_head, wq_entry);
++ spin_unlock_irqrestore(&wq_head->lock, flags);
++}
++EXPORT_SYMBOL(add_wait_queue_exclusive_lifo);
++
+ void remove_wait_queue(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry)
+ {
+ unsigned long flags;
+@@ -290,6 +301,19 @@ prepare_to_wait_exclusive(struct wait_queue_head *wq_head, struct wait_queue_ent
+ }
+ EXPORT_SYMBOL(prepare_to_wait_exclusive);
+
++void prepare_to_wait_exclusive_lifo(struct wait_queue_head *wq_head, struct wait_queue_entry *wq_entry, int state)
++{
++ unsigned long flags;
++
++ wq_entry->flags |= WQ_FLAG_EXCLUSIVE;
++ spin_lock_irqsave(&wq_head->lock, flags);
++ if (list_empty(&wq_entry->entry))
++ __add_wait_queue(wq_head, wq_entry);
++ set_current_state(state);
++ spin_unlock_irqrestore(&wq_head->lock, flags);
++}
++EXPORT_SYMBOL(prepare_to_wait_exclusive_lifo);
++
+ void init_wait_entry(struct wait_queue_entry *wq_entry, int flags)
+ {
+ wq_entry->flags = flags;
+diff --git a/kernel/watchdog.c b/kernel/watchdog.c
+index 99afb88d2e85..e24483521e17 100644
+--- a/kernel/watchdog.c
++++ b/kernel/watchdog.c
+@@ -41,7 +41,7 @@ unsigned long __read_mostly watchdog_enabled;
+ int __read_mostly watchdog_user_enabled = 1;
+ int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT;
+ int __read_mostly soft_watchdog_user_enabled = 1;
+-int __read_mostly watchdog_thresh = 10;
++int __read_mostly watchdog_thresh = 40;
+ static int __read_mostly nmi_watchdog_available;
+
+ struct cpumask watchdog_cpumask __read_mostly;
+diff --git a/mm/compaction.c b/mm/compaction.c
+index b4e94cda3019..e9a36942c1fa 100644
+--- a/mm/compaction.c
++++ b/mm/compaction.c
+@@ -53,7 +53,7 @@ static inline void count_compact_events(enum vm_event_item item, long delta)
+ /*
+ * Fragmentation score check interval for proactive compaction purposes.
+ */
+-static const unsigned int HPAGE_FRAG_CHECK_INTERVAL_MSEC = 500;
++static const unsigned int HPAGE_FRAG_CHECK_INTERVAL_MSEC = 5000;
+
+ /*
+ * Page order with-respect-to which proactive compaction
+diff --git a/mm/ksm.c b/mm/ksm.c
+index c20bd4d9a0d9..702a2a111fef 100644
+--- a/mm/ksm.c
++++ b/mm/ksm.c
+@@ -2421,9 +2421,14 @@ static int ksm_scan_thread(void *nothing)
+
+ if (ksmd_should_run()) {
+ sleep_ms = READ_ONCE(ksm_thread_sleep_millisecs);
+- wait_event_interruptible_timeout(ksm_iter_wait,
+- sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs),
+- msecs_to_jiffies(sleep_ms));
++ if (sleep_ms >= 1000)
++ wait_event_interruptible_timeout(ksm_iter_wait,
++ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs),
++ msecs_to_jiffies(round_jiffies_relative(sleep_ms)));
++ else
++ wait_event_interruptible_timeout(ksm_iter_wait,
++ sleep_ms != READ_ONCE(ksm_thread_sleep_millisecs),
++ msecs_to_jiffies(sleep_ms));
+ } else {
+ wait_event_freezable(ksm_thread_wait,
+ ksmd_should_run() || kthread_should_stop());
+diff --git a/mm/page_alloc.c b/mm/page_alloc.c
+index 3589febc6d31..aa5f6d702694 100644
+--- a/mm/page_alloc.c
++++ b/mm/page_alloc.c
+@@ -6836,11 +6836,11 @@ static int zone_batchsize(struct zone *zone)
+
+ /*
+ * The number of pages to batch allocate is either ~0.1%
+- * of the zone or 1MB, whichever is smaller. The batch
++ * of the zone or 4MB, whichever is smaller. The batch
+ * size is striking a balance between allocation latency
+ * and zone lock contention.
+ */
+- batch = min(zone_managed_pages(zone) >> 10, (1024 * 1024) / PAGE_SIZE);
++ batch = min(zone_managed_pages(zone) >> 10, 4 * (1024 * 1024) / PAGE_SIZE);
+ batch /= 4; /* We effectively *= 4 below */
+ if (batch < 1)
+ batch = 1;
+@@ -6918,6 +6918,7 @@ static int zone_highsize(struct zone *zone, int batch, int cpu_online)
+ * historical relationship between high and batch.
+ */
+ high = max(high, batch << 2);
++ high = max(high, 1024);
+
+ return high;
+ #else
+diff --git a/net/ipv4/inet_connection_sock.c b/net/ipv4/inet_connection_sock.c
+index fc2a985f6064..02dc861de3d5 100644
+--- a/net/ipv4/inet_connection_sock.c
++++ b/net/ipv4/inet_connection_sock.c
+@@ -441,7 +441,7 @@ static int inet_csk_wait_for_connect(struct sock *sk, long timeo)
+ * having to remove and re-insert us on the wait queue.
+ */
+ for (;;) {
+- prepare_to_wait_exclusive(sk_sleep(sk), &wait,
++ prepare_to_wait_exclusive_lifo(sk_sleep(sk), &wait,
+ TASK_INTERRUPTIBLE);
+ release_sock(sk);
+ if (reqsk_queue_empty(&icsk->icsk_accept_queue))
+diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
+index 3b75836db19b..51b67c96868d 100644
+--- a/net/ipv4/tcp.c
++++ b/net/ipv4/tcp.c
+@@ -4593,8 +4593,8 @@ void __init tcp_init(void)
+ tcp_init_mem();
+ /* Set per-socket limits to no more than 1/128 the pressure threshold */
+ limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
+- max_wshare = min(4UL*1024*1024, limit);
+- max_rshare = min(6UL*1024*1024, limit);
++ max_wshare = min(16UL*1024*1024, limit);
++ max_rshare = min(16UL*1024*1024, limit);
+
+ init_net.ipv4.sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
+ init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
+--
+2.34.1
+
+From 1611858732c35596279cd94f57afda62f3b83493 Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sun, 23 Jan 2022 16:53:23 +0000
+Subject: [PATCH] cpu
+
+---
+ arch/x86/Kconfig.cpu | 332 ++++++++++++++++++++++++++++++--
+ arch/x86/Makefile | 40 +++-
+ arch/x86/include/asm/vermagic.h | 66 +++++++
+ init/Kconfig | 1 -
+ 4 files changed, 424 insertions(+), 15 deletions(-)
+
+diff --git a/arch/x86/Kconfig.cpu b/arch/x86/Kconfig.cpu
+index 542377cd419d..22b919cdb6d1 100644
+--- a/arch/x86/Kconfig.cpu
++++ b/arch/x86/Kconfig.cpu
+@@ -157,7 +157,7 @@ config MPENTIUM4
+
+
+ config MK6
+- bool "K6/K6-II/K6-III"
++ bool "AMD K6/K6-II/K6-III"
+ depends on X86_32
+ help
+ Select this for an AMD K6-family processor. Enables use of
+@@ -165,7 +165,7 @@ config MK6
+ flags to GCC.
+
+ config MK7
+- bool "Athlon/Duron/K7"
++ bool "AMD Athlon/Duron/K7"
+ depends on X86_32
+ help
+ Select this for an AMD Athlon K7-family processor. Enables use of
+@@ -173,12 +173,98 @@ config MK7
+ flags to GCC.
+
+ config MK8
+- bool "Opteron/Athlon64/Hammer/K8"
++ bool "AMD Opteron/Athlon64/Hammer/K8"
+ help
+ Select this for an AMD Opteron or Athlon64 Hammer-family processor.
+ Enables use of some extended instructions, and passes appropriate
+ optimization flags to GCC.
+
++config MK8SSE3
++ bool "AMD Opteron/Athlon64/Hammer/K8 with SSE3"
++ help
++ Select this for improved AMD Opteron or Athlon64 Hammer-family processors.
++ Enables use of some extended instructions, and passes appropriate
++ optimization flags to GCC.
++
++config MK10
++ bool "AMD 61xx/7x50/PhenomX3/X4/II/K10"
++ help
++ Select this for an AMD 61xx Eight-Core Magny-Cours, Athlon X2 7x50,
++ Phenom X3/X4/II, Athlon II X2/X3/X4, or Turion II-family processor.
++ Enables use of some extended instructions, and passes appropriate
++ optimization flags to GCC.
++
++config MBARCELONA
++ bool "AMD Barcelona"
++ help
++ Select this for AMD Family 10h Barcelona processors.
++
++ Enables -march=barcelona
++
++config MBOBCAT
++ bool "AMD Bobcat"
++ help
++ Select this for AMD Family 14h Bobcat processors.
++
++ Enables -march=btver1
++
++config MJAGUAR
++ bool "AMD Jaguar"
++ help
++ Select this for AMD Family 16h Jaguar processors.
++
++ Enables -march=btver2
++
++config MBULLDOZER
++ bool "AMD Bulldozer"
++ help
++ Select this for AMD Family 15h Bulldozer processors.
++
++ Enables -march=bdver1
++
++config MPILEDRIVER
++ bool "AMD Piledriver"
++ help
++ Select this for AMD Family 15h Piledriver processors.
++
++ Enables -march=bdver2
++
++config MSTEAMROLLER
++ bool "AMD Steamroller"
++ help
++ Select this for AMD Family 15h Steamroller processors.
++
++ Enables -march=bdver3
++
++config MEXCAVATOR
++ bool "AMD Excavator"
++ help
++ Select this for AMD Family 15h Excavator processors.
++
++ Enables -march=bdver4
++
++config MZEN
++ bool "AMD Zen"
++ help
++ Select this for AMD Family 17h Zen processors.
++
++ Enables -march=znver1
++
++config MZEN2
++ bool "AMD Zen 2"
++ help
++ Select this for AMD Family 17h Zen 2 processors.
++
++ Enables -march=znver2
++
++config MZEN3
++ bool "AMD Zen 3"
++ depends on (CC_IS_GCC && GCC_VERSION >= 100300) || (CC_IS_CLANG && CLANG_VERSION >= 120000)
++ help
++ Select this for AMD Family 19h Zen 3 processors.
++
++ Enables -march=znver3
++
+ config MCRUSOE
+ bool "Crusoe"
+ depends on X86_32
+@@ -270,7 +356,7 @@ config MPSC
+ in /proc/cpuinfo. Family 15 is an older Xeon, Family 6 a newer one.
+
+ config MCORE2
+- bool "Core 2/newer Xeon"
++ bool "Intel Core 2"
+ help
+
+ Select this for Intel Core 2 and newer Core 2 Xeons (Xeon 51xx and
+@@ -278,6 +364,8 @@ config MCORE2
+ family in /proc/cpuinfo. Newer ones have 6 and older ones 15
+ (not a typo)
+
++ Enables -march=core2
++
+ config MATOM
+ bool "Intel Atom"
+ help
+@@ -287,6 +375,182 @@ config MATOM
+ accordingly optimized code. Use a recent GCC with specific Atom
+ support in order to fully benefit from selecting this option.
+
++config MNEHALEM
++ bool "Intel Nehalem"
++ select X86_P6_NOP
++ help
++
++ Select this for 1st Gen Core processors in the Nehalem family.
++
++ Enables -march=nehalem
++
++config MWESTMERE
++ bool "Intel Westmere"
++ select X86_P6_NOP
++ help
++
++ Select this for the Intel Westmere formerly Nehalem-C family.
++
++ Enables -march=westmere
++
++config MSILVERMONT
++ bool "Intel Silvermont"
++ select X86_P6_NOP
++ help
++
++ Select this for the Intel Silvermont platform.
++
++ Enables -march=silvermont
++
++config MGOLDMONT
++ bool "Intel Goldmont"
++ select X86_P6_NOP
++ help
++
++ Select this for the Intel Goldmont platform including Apollo Lake and Denverton.
++
++ Enables -march=goldmont
++
++config MGOLDMONTPLUS
++ bool "Intel Goldmont Plus"
++ select X86_P6_NOP
++ help
++
++ Select this for the Intel Goldmont Plus platform including Gemini Lake.
++
++ Enables -march=goldmont-plus
++
++config MSANDYBRIDGE
++ bool "Intel Sandy Bridge"
++ select X86_P6_NOP
++ help
++
++ Select this for 2nd Gen Core processors in the Sandy Bridge family.
++
++ Enables -march=sandybridge
++
++config MIVYBRIDGE
++ bool "Intel Ivy Bridge"
++ select X86_P6_NOP
++ help
++
++ Select this for 3rd Gen Core processors in the Ivy Bridge family.
++
++ Enables -march=ivybridge
++
++config MHASWELL
++ bool "Intel Haswell"
++ select X86_P6_NOP
++ help
++
++ Select this for 4th Gen Core processors in the Haswell family.
++
++ Enables -march=haswell
++
++config MBROADWELL
++ bool "Intel Broadwell"
++ select X86_P6_NOP
++ help
++
++ Select this for 5th Gen Core processors in the Broadwell family.
++
++ Enables -march=broadwell
++
++config MSKYLAKE
++ bool "Intel Skylake"
++ select X86_P6_NOP
++ help
++
++ Select this for 6th Gen Core processors in the Skylake family.
++
++ Enables -march=skylake
++
++config MSKYLAKEX
++ bool "Intel Skylake X"
++ select X86_P6_NOP
++ help
++
++ Select this for 6th Gen Core processors in the Skylake X family.
++
++ Enables -march=skylake-avx512
++
++config MCANNONLAKE
++ bool "Intel Cannon Lake"
++ select X86_P6_NOP
++ help
++
++ Select this for 8th Gen Core processors
++
++ Enables -march=cannonlake
++
++config MICELAKE
++ bool "Intel Ice Lake"
++ select X86_P6_NOP
++ help
++
++ Select this for 10th Gen Core processors in the Ice Lake family.
++
++ Enables -march=icelake-client
++
++config MCASCADELAKE
++ bool "Intel Cascade Lake"
++ select X86_P6_NOP
++ help
++
++ Select this for Xeon processors in the Cascade Lake family.
++
++ Enables -march=cascadelake
++
++config MCOOPERLAKE
++ bool "Intel Cooper Lake"
++ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000)
++ select X86_P6_NOP
++ help
++
++ Select this for Xeon processors in the Cooper Lake family.
++
++ Enables -march=cooperlake
++
++config MTIGERLAKE
++ bool "Intel Tiger Lake"
++ depends on (CC_IS_GCC && GCC_VERSION > 100100) || (CC_IS_CLANG && CLANG_VERSION >= 100000)
++ select X86_P6_NOP
++ help
++
++ Select this for third-generation 10 nm process processors in the Tiger Lake family.
++
++ Enables -march=tigerlake
++
++config MSAPPHIRERAPIDS
++ bool "Intel Sapphire Rapids"
++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)
++ select X86_P6_NOP
++ help
++
++ Select this for third-generation 10 nm process processors in the Sapphire Rapids family.
++
++ Enables -march=sapphirerapids
++
++config MROCKETLAKE
++ bool "Intel Rocket Lake"
++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)
++ select X86_P6_NOP
++ help
++
++ Select this for eleventh-generation processors in the Rocket Lake family.
++
++ Enables -march=rocketlake
++
++config MALDERLAKE
++ bool "Intel Alder Lake"
++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)
++ select X86_P6_NOP
++ help
++
++ Select this for twelfth-generation processors in the Alder Lake family.
++
++ Enables -march=alderlake
++
+ config GENERIC_CPU
+ bool "Generic-x86-64"
+ depends on X86_64
+@@ -294,6 +558,50 @@ config GENERIC_CPU
+ Generic x86-64 CPU.
+ Run equally well on all x86-64 CPUs.
+
++config GENERIC_CPU2
++ bool "Generic-x86-64-v2"
++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)
++ depends on X86_64
++ help
++ Generic x86-64 CPU.
++ Run equally well on all x86-64 CPUs with min support of x86-64-v2.
++
++config GENERIC_CPU3
++ bool "Generic-x86-64-v3"
++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)
++ depends on X86_64
++ help
++ Generic x86-64-v3 CPU with v3 instructions.
++ Run equally well on all x86-64 CPUs with min support of x86-64-v3.
++
++config GENERIC_CPU4
++ bool "Generic-x86-64-v4"
++ depends on (CC_IS_GCC && GCC_VERSION > 110000) || (CC_IS_CLANG && CLANG_VERSION >= 120000)
++ depends on X86_64
++ help
++ Generic x86-64 CPU with v4 instructions.
++ Run equally well on all x86-64 CPUs with min support of x86-64-v4.
++
++config MNATIVE_INTEL
++ bool "Intel-Native optimizations autodetected by the compiler"
++ help
++
++ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects
++ the optimum settings to use based on your processor. Do NOT use this
++ for AMD CPUs. Intel Only!
++
++ Enables -march=native
++
++config MNATIVE_AMD
++ bool "AMD-Native optimizations autodetected by the compiler"
++ help
++
++ Clang 3.8, GCC 4.2 and above support -march=native, which automatically detects
++ the optimum settings to use based on your processor. Do NOT use this
++ for Intel CPUs. AMD Only!
++
++ Enables -march=native
++
+ endchoice
+
+ config X86_GENERIC
+@@ -318,7 +626,7 @@ config X86_INTERNODE_CACHE_SHIFT
+ config X86_L1_CACHE_SHIFT
+ int
+ default "7" if MPENTIUM4 || MPSC
+- default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || X86_GENERIC || GENERIC_CPU
++ default "6" if MK7 || MK8 || MPENTIUMM || MCORE2 || MATOM || MVIAC7 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD || X86_GENERIC || GENERIC_CPU || GENERIC_CPU2 || GENERIC_CPU3 || GENERIC_CPU4
+ default "4" if MELAN || M486SX || M486 || MGEODEGX1
+ default "5" if MWINCHIP3D || MWINCHIPC6 || MCRUSOE || MEFFICEON || MCYRIXIII || MK6 || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || M586 || MVIAC3_2 || MGEODE_LX
+
+@@ -336,11 +644,11 @@ config X86_ALIGNMENT_16
+
+ config X86_INTEL_USERCOPY
+ def_bool y
+- depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2
++ depends on MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M586MMX || X86_GENERIC || MK8 || MK7 || MEFFICEON || MCORE2 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL
+
+ config X86_USE_PPRO_CHECKSUM
+ def_bool y
+- depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM
++ depends on MWINCHIP3D || MWINCHIPC6 || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MK8 || MVIAC3_2 || MVIAC7 || MEFFICEON || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD
+
+ #
+ # P6_NOPs are a relatively minor optimization that require a family >=
+@@ -356,26 +664,26 @@ config X86_USE_PPRO_CHECKSUM
+ config X86_P6_NOP
+ def_bool y
+ depends on X86_64
+- depends on (MCORE2 || MPENTIUM4 || MPSC)
++ depends on (MCORE2 || MPENTIUM4 || MPSC || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL)
+
+ config X86_TSC
+ def_bool y
+- depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM) || X86_64
++ depends on (MWINCHIP3D || MCRUSOE || MEFFICEON || MCYRIXIII || MK7 || MK6 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586MMX || M586TSC || MK8 || MVIAC3_2 || MVIAC7 || MGEODEGX1 || MGEODE_LX || MCORE2 || MATOM || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD) || X86_64
+
+ config X86_CMPXCHG64
+ def_bool y
+- depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8
++ depends on X86_PAE || X86_64 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || M586TSC || M586MMX || MATOM || MGEODE_LX || MGEODEGX1 || MK6 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD
+
+ # this should be set for all -march=.. options where the compiler
+ # generates cmov.
+ config X86_CMOV
+ def_bool y
+- depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX)
++ depends on (MK8 || MK7 || MCORE2 || MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MCRUSOE || MEFFICEON || X86_64 || MATOM || MGEODE_LX || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD)
+
+ config X86_MINIMUM_CPU_FAMILY
+ int
+ default "64" if X86_64
+- default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8)
++ default "6" if X86_32 && (MPENTIUM4 || MPENTIUMM || MPENTIUMIII || MPENTIUMII || M686 || MVIAC3_2 || MVIAC7 || MEFFICEON || MATOM || MCRUSOE || MCORE2 || MK7 || MK8 || MK8SSE3 || MK10 || MBARCELONA || MBOBCAT || MJAGUAR || MBULLDOZER || MPILEDRIVER || MSTEAMROLLER || MEXCAVATOR || MZEN || MZEN2 || MZEN3 || MNEHALEM || MWESTMERE || MSILVERMONT || MGOLDMONT || MGOLDMONTPLUS || MSANDYBRIDGE || MIVYBRIDGE || MHASWELL || MBROADWELL || MSKYLAKE || MSKYLAKEX || MCANNONLAKE || MICELAKE || MCASCADELAKE || MCOOPERLAKE || MTIGERLAKE || MSAPPHIRERAPIDS || MROCKETLAKE || MALDERLAKE || MNATIVE_INTEL || MNATIVE_AMD)
+ default "5" if X86_32 && X86_CMPXCHG64
+ default "4"
+
+diff --git a/arch/x86/Makefile b/arch/x86/Makefile
+index e84cdd409b64..7d3bbf060079 100644
+--- a/arch/x86/Makefile
++++ b/arch/x86/Makefile
+@@ -131,8 +131,44 @@ else
+ # FIXME - should be integrated in Makefile.cpu (Makefile_32.cpu)
+ cflags-$(CONFIG_MK8) += -march=k8
+ cflags-$(CONFIG_MPSC) += -march=nocona
+- cflags-$(CONFIG_MCORE2) += -march=core2
+- cflags-$(CONFIG_MATOM) += -march=atom
++ cflags-$(CONFIG_MK8SSE3) += -march=k8-sse3
++ cflags-$(CONFIG_MK10) += -march=amdfam10
++ cflags-$(CONFIG_MBARCELONA) += -march=barcelona
++ cflags-$(CONFIG_MBOBCAT) += -march=btver1
++ cflags-$(CONFIG_MJAGUAR) += -march=btver2
++ cflags-$(CONFIG_MBULLDOZER) += -march=bdver1
++ cflags-$(CONFIG_MPILEDRIVER) += -march=bdver2 -mno-tbm
++ cflags-$(CONFIG_MSTEAMROLLER) += -march=bdver3 -mno-tbm
++ cflags-$(CONFIG_MEXCAVATOR) += -march=bdver4 -mno-tbm
++ cflags-$(CONFIG_MZEN) += -march=znver1
++ cflags-$(CONFIG_MZEN2) += -march=znver2
++ cflags-$(CONFIG_MZEN3) += -march=znver3
++ cflags-$(CONFIG_MNATIVE_INTEL) += -march=native
++ cflags-$(CONFIG_MNATIVE_AMD) += -march=native
++ cflags-$(CONFIG_MATOM) += -march=bonnell
++ cflags-$(CONFIG_MCORE2) += -march=core2
++ cflags-$(CONFIG_MNEHALEM) += -march=nehalem
++ cflags-$(CONFIG_MWESTMERE) += -march=westmere
++ cflags-$(CONFIG_MSILVERMONT) += -march=silvermont
++ cflags-$(CONFIG_MGOLDMONT) += -march=goldmont
++ cflags-$(CONFIG_MGOLDMONTPLUS) += -march=goldmont-plus
++ cflags-$(CONFIG_MSANDYBRIDGE) += -march=sandybridge
++ cflags-$(CONFIG_MIVYBRIDGE) += -march=ivybridge
++ cflags-$(CONFIG_MHASWELL) += -march=haswell
++ cflags-$(CONFIG_MBROADWELL) += -march=broadwell
++ cflags-$(CONFIG_MSKYLAKE) += -march=skylake
++ cflags-$(CONFIG_MSKYLAKEX) += -march=skylake-avx512
++ cflags-$(CONFIG_MCANNONLAKE) += -march=cannonlake
++ cflags-$(CONFIG_MICELAKE) += -march=icelake-client
++ cflags-$(CONFIG_MCASCADELAKE) += -march=cascadelake
++ cflags-$(CONFIG_MCOOPERLAKE) += -march=cooperlake
++ cflags-$(CONFIG_MTIGERLAKE) += -march=tigerlake
++ cflags-$(CONFIG_MSAPPHIRERAPIDS) += -march=sapphirerapids
++ cflags-$(CONFIG_MROCKETLAKE) += -march=rocketlake
++ cflags-$(CONFIG_MALDERLAKE) += -march=alderlake
++ cflags-$(CONFIG_GENERIC_CPU2) += -march=x86-64-v2
++ cflags-$(CONFIG_GENERIC_CPU3) += -march=x86-64-v3
++ cflags-$(CONFIG_GENERIC_CPU4) += -march=x86-64-v4
+ cflags-$(CONFIG_GENERIC_CPU) += -mtune=generic
+ KBUILD_CFLAGS += $(cflags-y)
+
+diff --git a/arch/x86/include/asm/vermagic.h b/arch/x86/include/asm/vermagic.h
+index 75884d2cdec3..4e6a08d4c7e5 100644
+--- a/arch/x86/include/asm/vermagic.h
++++ b/arch/x86/include/asm/vermagic.h
+@@ -17,6 +17,48 @@
+ #define MODULE_PROC_FAMILY "586MMX "
+ #elif defined CONFIG_MCORE2
+ #define MODULE_PROC_FAMILY "CORE2 "
++#elif defined CONFIG_MNATIVE_INTEL
++#define MODULE_PROC_FAMILY "NATIVE_INTEL "
++#elif defined CONFIG_MNATIVE_AMD
++#define MODULE_PROC_FAMILY "NATIVE_AMD "
++#elif defined CONFIG_MNEHALEM
++#define MODULE_PROC_FAMILY "NEHALEM "
++#elif defined CONFIG_MWESTMERE
++#define MODULE_PROC_FAMILY "WESTMERE "
++#elif defined CONFIG_MSILVERMONT
++#define MODULE_PROC_FAMILY "SILVERMONT "
++#elif defined CONFIG_MGOLDMONT
++#define MODULE_PROC_FAMILY "GOLDMONT "
++#elif defined CONFIG_MGOLDMONTPLUS
++#define MODULE_PROC_FAMILY "GOLDMONTPLUS "
++#elif defined CONFIG_MSANDYBRIDGE
++#define MODULE_PROC_FAMILY "SANDYBRIDGE "
++#elif defined CONFIG_MIVYBRIDGE
++#define MODULE_PROC_FAMILY "IVYBRIDGE "
++#elif defined CONFIG_MHASWELL
++#define MODULE_PROC_FAMILY "HASWELL "
++#elif defined CONFIG_MBROADWELL
++#define MODULE_PROC_FAMILY "BROADWELL "
++#elif defined CONFIG_MSKYLAKE
++#define MODULE_PROC_FAMILY "SKYLAKE "
++#elif defined CONFIG_MSKYLAKEX
++#define MODULE_PROC_FAMILY "SKYLAKEX "
++#elif defined CONFIG_MCANNONLAKE
++#define MODULE_PROC_FAMILY "CANNONLAKE "
++#elif defined CONFIG_MICELAKE
++#define MODULE_PROC_FAMILY "ICELAKE "
++#elif defined CONFIG_MCASCADELAKE
++#define MODULE_PROC_FAMILY "CASCADELAKE "
++#elif defined CONFIG_MCOOPERLAKE
++#define MODULE_PROC_FAMILY "COOPERLAKE "
++#elif defined CONFIG_MTIGERLAKE
++#define MODULE_PROC_FAMILY "TIGERLAKE "
++#elif defined CONFIG_MSAPPHIRERAPIDS
++#define MODULE_PROC_FAMILY "SAPPHIRERAPIDS "
++#elif defined CONFIG_ROCKETLAKE
++#define MODULE_PROC_FAMILY "ROCKETLAKE "
++#elif defined CONFIG_MALDERLAKE
++#define MODULE_PROC_FAMILY "ALDERLAKE "
+ #elif defined CONFIG_MATOM
+ #define MODULE_PROC_FAMILY "ATOM "
+ #elif defined CONFIG_M686
+@@ -35,6 +77,30 @@
+ #define MODULE_PROC_FAMILY "K7 "
+ #elif defined CONFIG_MK8
+ #define MODULE_PROC_FAMILY "K8 "
++#elif defined CONFIG_MK8SSE3
++#define MODULE_PROC_FAMILY "K8SSE3 "
++#elif defined CONFIG_MK10
++#define MODULE_PROC_FAMILY "K10 "
++#elif defined CONFIG_MBARCELONA
++#define MODULE_PROC_FAMILY "BARCELONA "
++#elif defined CONFIG_MBOBCAT
++#define MODULE_PROC_FAMILY "BOBCAT "
++#elif defined CONFIG_MBULLDOZER
++#define MODULE_PROC_FAMILY "BULLDOZER "
++#elif defined CONFIG_MPILEDRIVER
++#define MODULE_PROC_FAMILY "PILEDRIVER "
++#elif defined CONFIG_MSTEAMROLLER
++#define MODULE_PROC_FAMILY "STEAMROLLER "
++#elif defined CONFIG_MJAGUAR
++#define MODULE_PROC_FAMILY "JAGUAR "
++#elif defined CONFIG_MEXCAVATOR
++#define MODULE_PROC_FAMILY "EXCAVATOR "
++#elif defined CONFIG_MZEN
++#define MODULE_PROC_FAMILY "ZEN "
++#elif defined CONFIG_MZEN2
++#define MODULE_PROC_FAMILY "ZEN2 "
++#elif defined CONFIG_MZEN3
++#define MODULE_PROC_FAMILY "ZEN3 "
+ #elif defined CONFIG_MELAN
+ #define MODULE_PROC_FAMILY "ELAN "
+ #elif defined CONFIG_MCRUSOE
+diff --git a/init/Kconfig b/init/Kconfig
+index e9119bf54b1f..98b648e3cef6 100644
+--- a/init/Kconfig
++++ b/init/Kconfig
+@@ -1370,7 +1370,6 @@ config CC_OPTIMIZE_FOR_PERFORMANCE
+
+ config CC_OPTIMIZE_FOR_PERFORMANCE_O3
+ bool "Optimize more for performance (-O3)"
+- depends on ARC
+ help
+ Choosing this option will pass "-O3" to your compiler to optimize
+ the kernel yet more for performance.
+--
+2.34.1
+
+From ef65b3a3f9e6dd347d2ae8735dcf347fa5135c8f Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sun, 6 Feb 2022 20:28:25 +0000
+Subject: [PATCH] misc
+
+Signed-off-by: P. Jung <[email protected]>
+---
+ .../admin-guide/kernel-parameters.txt | 9 +
+ Makefile | 5 -
+ drivers/cpufreq/Kconfig | 2 +-
+ drivers/cpufreq/cpufreq_ondemand.c | 6 +-
+ drivers/hid/usbhid/hid-core.c | 8 +
+ drivers/i2c/busses/Kconfig | 9 +
+ drivers/i2c/busses/Makefile | 1 +
+ drivers/i2c/busses/i2c-nct6775.c | 647 ++++++++++++++++++
+ drivers/i2c/busses/i2c-piix4.c | 4 +-
+ drivers/infiniband/core/addr.c | 1 +
+ drivers/pci/quirks.c | 102 +++
+ drivers/scsi/sd.c | 43 +-
+ drivers/scsi/sd.h | 1 +
+ drivers/tty/Kconfig | 13 +
+ include/linux/pagemap.h | 2 +-
+ include/linux/user_namespace.h | 4 +
+ include/uapi/linux/futex.h | 13 +
+ include/uapi/linux/vt.h | 15 +-
+ init/Kconfig | 16 +
+ kernel/Kconfig.hz | 38 +
+ kernel/cpu.c | 1 +
+ kernel/fork.c | 14 +
+ kernel/futex/syscalls.c | 75 +-
+ kernel/sched/core.c | 2 +-
+ kernel/sched/fair.c | 14 +-
+ kernel/sysctl.c | 12 +
+ kernel/user_namespace.c | 7 +
+ mm/internal.h | 1 +
+ mm/page-writeback.c | 4 +-
+ mm/page_alloc.c | 26 +-
+ mm/vmscan.c | 5 +-
+ net/sched/Kconfig | 4 +
+ 32 files changed, 1068 insertions(+), 36 deletions(-)
+ create mode 100644 drivers/i2c/busses/i2c-nct6775.c
+
+diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt
+index f5a27f067db9..cbc216dc0a74 100644
+--- a/Documentation/admin-guide/kernel-parameters.txt
++++ b/Documentation/admin-guide/kernel-parameters.txt
+@@ -3938,6 +3938,15 @@
+ nomsi [MSI] If the PCI_MSI kernel config parameter is
+ enabled, this kernel boot option can be used to
+ disable the use of MSI interrupts system-wide.
++ pcie_acs_override =
++ [PCIE] Override missing PCIe ACS support for:
++ downstream
++ All downstream ports - full ACS capabilities
++ multifunction
++ All multifunction devices - multifunction ACS subset
++ id:nnnn:nnnn
++ Specific device - full ACS capabilities
++ Specified as vid:did (vendor/device ID) in hex
+ noioapicquirk [APIC] Disable all boot interrupt quirks.
+ Safety option to keep boot IRQs enabled. This
+ should never be necessary.
+diff --git a/Makefile b/Makefile
+index 1fc3491096cb..4de96651f8d1 100644
+--- a/Makefile
++++ b/Makefile
+@@ -984,11 +984,6 @@ KBUILD_CFLAGS += -fno-strict-overflow
+ # Make sure -fstack-check isn't enabled (like gentoo apparently did)
+ KBUILD_CFLAGS += -fno-stack-check
+
+-# conserve stack if available
+-ifdef CONFIG_CC_IS_GCC
+-KBUILD_CFLAGS += -fconserve-stack
+-endif
+-
+ # Prohibit date/time macros, which would make the build non-deterministic
+ KBUILD_CFLAGS += -Werror=date-time
+
+diff --git a/drivers/cpufreq/Kconfig b/drivers/cpufreq/Kconfig
+index c3038cdc6865..25bf88b571ae 100644
+--- a/drivers/cpufreq/Kconfig
++++ b/drivers/cpufreq/Kconfig
+@@ -72,7 +72,7 @@ config CPU_FREQ_DEFAULT_GOV_USERSPACE
+
+ config CPU_FREQ_DEFAULT_GOV_ONDEMAND
+ bool "ondemand"
+- depends on !(X86_INTEL_PSTATE && SMP)
++ depends on SMP
+ select CPU_FREQ_GOV_ONDEMAND
+ select CPU_FREQ_GOV_PERFORMANCE
+ help
+diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
+index 6a41ea4729b8..370af073c27f 100644
+--- a/drivers/cpufreq/cpufreq_ondemand.c
++++ b/drivers/cpufreq/cpufreq_ondemand.c
+@@ -18,10 +18,10 @@
+ #include "cpufreq_ondemand.h"
+
+ /* On-demand governor macros */
+-#define DEF_FREQUENCY_UP_THRESHOLD (80)
+-#define DEF_SAMPLING_DOWN_FACTOR (1)
++#define DEF_FREQUENCY_UP_THRESHOLD (65)
++#define DEF_SAMPLING_DOWN_FACTOR (10)
+ #define MAX_SAMPLING_DOWN_FACTOR (100000)
+-#define MICRO_FREQUENCY_UP_THRESHOLD (95)
++#define MICRO_FREQUENCY_UP_THRESHOLD (85)
+ #define MICRO_FREQUENCY_MIN_SAMPLE_RATE (10000)
+ #define MIN_FREQUENCY_UP_THRESHOLD (1)
+ #define MAX_FREQUENCY_UP_THRESHOLD (100)
+diff --git a/drivers/hid/usbhid/hid-core.c b/drivers/hid/usbhid/hid-core.c
+index 54752c85604b..6fa8de3ce7aa 100644
+--- a/drivers/hid/usbhid/hid-core.c
++++ b/drivers/hid/usbhid/hid-core.c
+@@ -45,6 +45,10 @@
+ * Module parameters.
+ */
+
++static unsigned int hid_poll_interval;
++module_param_named(poll, hid_poll_interval, uint, 0644);
++MODULE_PARM_DESC(poll, "Polling interval of any device");
++
+ static unsigned int hid_mousepoll_interval;
+ module_param_named(mousepoll, hid_mousepoll_interval, uint, 0644);
+ MODULE_PARM_DESC(mousepoll, "Polling interval of mice");
+@@ -1107,6 +1111,10 @@ static int usbhid_start(struct hid_device *hid)
+ hid->name, endpoint->bInterval, interval);
+ }
+
++ /* First set polling interval for all devices */
++ if (hid_poll_interval > 0)
++ interval = hid_poll_interval;
++
+ /* Change the polling interval of mice, joysticks
+ * and keyboards.
+ */
+diff --git a/drivers/i2c/busses/Kconfig b/drivers/i2c/busses/Kconfig
+index 42da31c1ab70..c3707e64d132 100644
+--- a/drivers/i2c/busses/Kconfig
++++ b/drivers/i2c/busses/Kconfig
+@@ -219,6 +219,15 @@ config I2C_CHT_WC
+ combined with a FUSB302 Type-C port-controller as such it is advised
+ to also select CONFIG_TYPEC_FUSB302=m.
+
++config I2C_NCT6775
++ tristate "Nuvoton NCT6775 and compatible SMBus controller"
++ help
++ If you say yes to this option, support will be included for the
++ Nuvoton NCT6775 and compatible SMBus controllers.
++
++ This driver can also be built as a module. If so, the module
++ will be called i2c-nct6775.
++
+ config I2C_NFORCE2
+ tristate "Nvidia nForce2, nForce3 and nForce4"
+ depends on PCI
+diff --git a/drivers/i2c/busses/Makefile b/drivers/i2c/busses/Makefile
+index 1d00dce77098..ec8a1197c8bc 100644
+--- a/drivers/i2c/busses/Makefile
++++ b/drivers/i2c/busses/Makefile
+@@ -17,6 +17,7 @@ obj-$(CONFIG_I2C_CHT_WC) += i2c-cht-wc.o
+ obj-$(CONFIG_I2C_I801) += i2c-i801.o
+ obj-$(CONFIG_I2C_ISCH) += i2c-isch.o
+ obj-$(CONFIG_I2C_ISMT) += i2c-ismt.o
++obj-$(CONFIG_I2C_NCT6775) += i2c-nct6775.o
+ obj-$(CONFIG_I2C_NFORCE2) += i2c-nforce2.o
+ obj-$(CONFIG_I2C_NFORCE2_S4985) += i2c-nforce2-s4985.o
+ obj-$(CONFIG_I2C_NVIDIA_GPU) += i2c-nvidia-gpu.o
+diff --git a/drivers/i2c/busses/i2c-nct6775.c b/drivers/i2c/busses/i2c-nct6775.c
+new file mode 100644
+index 000000000000..b59f842f7278
+--- /dev/null
++++ b/drivers/i2c/busses/i2c-nct6775.c
+@@ -0,0 +1,647 @@
++/*
++ * i2c-nct6775 - Driver for the SMBus master functionality of
++ * Nuvoton NCT677x Super-I/O chips
++ *
++ * Copyright (C) 2019 Adam Honse <[email protected]>
++ *
++ * Derived from nct6775 hwmon driver
++ * Copyright (C) 2012 Guenter Roeck <[email protected]>
++ *
++ * This program is free software; you can redistribute it and/or modify
++ * it under the terms of the GNU General Public License as published by
++ * the Free Software Foundation; either version 2 of the License, or
++ * (at your option) any later version.
++ *
++ * This program is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
++ * GNU General Public License for more details.
++ *
++ * You should have received a copy of the GNU General Public License
++ * along with this program; if not, write to the Free Software
++ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
++ *
++ */
++
++#include <linux/module.h>
++#include <linux/init.h>
++#include <linux/slab.h>
++#include <linux/jiffies.h>
++#include <linux/platform_device.h>
++#include <linux/hwmon.h>
++#include <linux/hwmon-sysfs.h>
++#include <linux/hwmon-vid.h>
++#include <linux/err.h>
++#include <linux/mutex.h>
++#include <linux/delay.h>
++#include <linux/ioport.h>
++#include <linux/i2c.h>
++#include <linux/acpi.h>
++#include <linux/bitops.h>
++#include <linux/dmi.h>
++#include <linux/io.h>
++#include <linux/nospec.h>
++
++#define DRVNAME "i2c-nct6775"
++
++/* Nuvoton SMBus address offsets */
++#define SMBHSTDAT (0 + nuvoton_nct6793d_smba)
++#define SMBBLKSZ (1 + nuvoton_nct6793d_smba)
++#define SMBHSTCMD (2 + nuvoton_nct6793d_smba)
++#define SMBHSTIDX (3 + nuvoton_nct6793d_smba) //Index field is the Command field on other controllers
++#define SMBHSTCTL (4 + nuvoton_nct6793d_smba)
++#define SMBHSTADD (5 + nuvoton_nct6793d_smba)
++#define SMBHSTERR (9 + nuvoton_nct6793d_smba)
++#define SMBHSTSTS (0xE + nuvoton_nct6793d_smba)
++
++/* Command register */
++#define NCT6793D_READ_BYTE 0
++#define NCT6793D_READ_WORD 1
++#define NCT6793D_READ_BLOCK 2
++#define NCT6793D_BLOCK_WRITE_READ_PROC_CALL 3
++#define NCT6793D_PROC_CALL 4
++#define NCT6793D_WRITE_BYTE 8
++#define NCT6793D_WRITE_WORD 9
++#define NCT6793D_WRITE_BLOCK 10
++
++/* Control register */
++#define NCT6793D_MANUAL_START 128
++#define NCT6793D_SOFT_RESET 64
++
++/* Error register */
++#define NCT6793D_NO_ACK 32
++
++/* Status register */
++#define NCT6793D_FIFO_EMPTY 1
++#define NCT6793D_FIFO_FULL 2
++#define NCT6793D_MANUAL_ACTIVE 4
++
++#define NCT6775_LD_SMBUS 0x0B
++
++/* Other settings */
++#define MAX_RETRIES 400
++
++enum kinds { nct6106, nct6775, nct6776, nct6779, nct6791, nct6792, nct6793,
++ nct6795, nct6796, nct6798 };
++
++struct nct6775_sio_data {
++ int sioreg;
++ enum kinds kind;
++};
++
++/* used to set data->name = nct6775_device_names[data->sio_kind] */
++static const char * const nct6775_device_names[] = {
++ "nct6106",
++ "nct6775",
++ "nct6776",
++ "nct6779",
++ "nct6791",
++ "nct6792",
++ "nct6793",
++ "nct6795",
++ "nct6796",
++ "nct6798",
++};
++
++static const char * const nct6775_sio_names[] __initconst = {
++ "NCT6106D",
++ "NCT6775F",
++ "NCT6776D/F",
++ "NCT6779D",
++ "NCT6791D",
++ "NCT6792D",
++ "NCT6793D",
++ "NCT6795D",
++ "NCT6796D",
++ "NCT6798D",
++};
++
++#define SIO_REG_LDSEL 0x07 /* Logical device select */
++#define SIO_REG_DEVID 0x20 /* Device ID (2 bytes) */
++#define SIO_REG_SMBA 0x62 /* SMBus base address register */
++
++#define SIO_NCT6106_ID 0xc450
++#define SIO_NCT6775_ID 0xb470
++#define SIO_NCT6776_ID 0xc330
++#define SIO_NCT6779_ID 0xc560
++#define SIO_NCT6791_ID 0xc800
++#define SIO_NCT6792_ID 0xc910
++#define SIO_NCT6793_ID 0xd120
++#define SIO_NCT6795_ID 0xd350
++#define SIO_NCT6796_ID 0xd420
++#define SIO_NCT6798_ID 0xd428
++#define SIO_ID_MASK 0xFFF0
++
++static inline void
++superio_outb(int ioreg, int reg, int val)
++{
++ outb(reg, ioreg);
++ outb(val, ioreg + 1);
++}
++
++static inline int
++superio_inb(int ioreg, int reg)
++{
++ outb(reg, ioreg);
++ return inb(ioreg + 1);
++}
++
++static inline void
++superio_select(int ioreg, int ld)
++{
++ outb(SIO_REG_LDSEL, ioreg);
++ outb(ld, ioreg + 1);
++}
++
++static inline int
++superio_enter(int ioreg)
++{
++ /*
++ * Try to reserve <ioreg> and <ioreg + 1> for exclusive access.
++ */
++ if (!request_muxed_region(ioreg, 2, DRVNAME))
++ return -EBUSY;
++
++ outb(0x87, ioreg);
++ outb(0x87, ioreg);
++
++ return 0;
++}
++
++static inline void
++superio_exit(int ioreg)
++{
++ outb(0xaa, ioreg);
++ outb(0x02, ioreg);
++ outb(0x02, ioreg + 1);
++ release_region(ioreg, 2);
++}
++
++/*
++ * ISA constants
++ */
++
++#define IOREGION_ALIGNMENT (~7)
++#define IOREGION_LENGTH 2
++#define ADDR_REG_OFFSET 0
++#define DATA_REG_OFFSET 1
++
++#define NCT6775_REG_BANK 0x4E
++#define NCT6775_REG_CONFIG 0x40
++
++static struct i2c_adapter *nct6775_adapter;
++
++struct i2c_nct6775_adapdata {
++ unsigned short smba;
++};
++
++/* Return negative errno on error. */
++static s32 nct6775_access(struct i2c_adapter * adap, u16 addr,
++ unsigned short flags, char read_write,
++ u8 command, int size, union i2c_smbus_data * data)
++{
++ struct i2c_nct6775_adapdata *adapdata = i2c_get_adapdata(adap);
++ unsigned short nuvoton_nct6793d_smba = adapdata->smba;
++ int i, len, cnt;
++ union i2c_smbus_data tmp_data;
++ int timeout = 0;
++
++ tmp_data.word = 0;
++ cnt = 0;
++ len = 0;
++
++ outb_p(NCT6793D_SOFT_RESET, SMBHSTCTL);
++
++ switch (size) {
++ case I2C_SMBUS_QUICK:
++ outb_p((addr << 1) | read_write,
++ SMBHSTADD);
++ break;
++ case I2C_SMBUS_BYTE_DATA:
++ tmp_data.byte = data->byte;
++ case I2C_SMBUS_BYTE:
++ outb_p((addr << 1) | read_write,
++ SMBHSTADD);
++ outb_p(command, SMBHSTIDX);
++ if (read_write == I2C_SMBUS_WRITE) {
++ outb_p(tmp_data.byte, SMBHSTDAT);
++ outb_p(NCT6793D_WRITE_BYTE, SMBHSTCMD);
++ }
++ else {
++ outb_p(NCT6793D_READ_BYTE, SMBHSTCMD);
++ }
++ break;
++ case I2C_SMBUS_WORD_DATA:
++ outb_p((addr << 1) | read_write,
++ SMBHSTADD);
++ outb_p(command, SMBHSTIDX);
++ if (read_write == I2C_SMBUS_WRITE) {
++ outb_p(data->word & 0xff, SMBHSTDAT);
++ outb_p((data->word & 0xff00) >> 8, SMBHSTDAT);
++ outb_p(NCT6793D_WRITE_WORD, SMBHSTCMD);
++ }
++ else {
++ outb_p(NCT6793D_READ_WORD, SMBHSTCMD);
++ }
++ break;
++ case I2C_SMBUS_BLOCK_DATA:
++ outb_p((addr << 1) | read_write,
++ SMBHSTADD);
++ outb_p(command, SMBHSTIDX);
++ if (read_write == I2C_SMBUS_WRITE) {
++ len = data->block[0];
++ if (len == 0 || len > I2C_SMBUS_BLOCK_MAX)
++ return -EINVAL;
++ outb_p(len, SMBBLKSZ);
++
++ cnt = 1;
++ if (len >= 4) {
++ for (i = cnt; i <= 4; i++) {
++ outb_p(data->block[i], SMBHSTDAT);
++ }
++
++ len -= 4;
++ cnt += 4;
++ }
++ else {
++ for (i = cnt; i <= len; i++ ) {
++ outb_p(data->block[i], SMBHSTDAT);
++ }
++
++ len = 0;
++ }
++
++ outb_p(NCT6793D_WRITE_BLOCK, SMBHSTCMD);
++ }
++ else {
++ return -ENOTSUPP;
++ }
++ break;
++ default:
++ dev_warn(&adap->dev, "Unsupported transaction %d\n", size);
++ return -EOPNOTSUPP;
++ }
++
++ outb_p(NCT6793D_MANUAL_START, SMBHSTCTL);
++
++ while ((size == I2C_SMBUS_BLOCK_DATA) && (len > 0)) {
++ if (read_write == I2C_SMBUS_WRITE) {
++ timeout = 0;
++ while ((inb_p(SMBHSTSTS) & NCT6793D_FIFO_EMPTY) == 0)
++ {
++ if(timeout > MAX_RETRIES)
++ {
++ return -ETIMEDOUT;
++ }
++ usleep_range(250, 500);
++ timeout++;
++ }
++
++ //Load more bytes into FIFO
++ if (len >= 4) {
++ for (i = cnt; i <= (cnt + 4); i++) {
++ outb_p(data->block[i], SMBHSTDAT);
++ }
++
++ len -= 4;
++ cnt += 4;
++ }
++ else {
++ for (i = cnt; i <= (cnt + len); i++) {
++ outb_p(data->block[i], SMBHSTDAT);
++ }
++
++ len = 0;
++ }
++ }
++ else {
++ return -ENOTSUPP;
++ }
++
++ }
++
++ //wait for manual mode to complete
++ timeout = 0;
++ while ((inb_p(SMBHSTSTS) & NCT6793D_MANUAL_ACTIVE) != 0)
++ {
++ if(timeout > MAX_RETRIES)
++ {
++ return -ETIMEDOUT;
++ }
++ usleep_range(250, 500);
++ timeout++;
++ }
++
++ if ((inb_p(SMBHSTERR) & NCT6793D_NO_ACK) != 0) {
++ return -ENXIO;
++ }
++ else if ((read_write == I2C_SMBUS_WRITE) || (size == I2C_SMBUS_QUICK)) {
++ return 0;
++ }
++
++ switch (size) {
++ case I2C_SMBUS_QUICK:
++ case I2C_SMBUS_BYTE_DATA:
++ data->byte = inb_p(SMBHSTDAT);
++ break;
++ case I2C_SMBUS_WORD_DATA:
++ data->word = inb_p(SMBHSTDAT) + (inb_p(SMBHSTDAT) << 8);
++ break;
++ }
++ return 0;
++}
++
++static u32 nct6775_func(struct i2c_adapter *adapter)
++{
++ return I2C_FUNC_SMBUS_QUICK | I2C_FUNC_SMBUS_BYTE |
++ I2C_FUNC_SMBUS_BYTE_DATA | I2C_FUNC_SMBUS_WORD_DATA |
++ I2C_FUNC_SMBUS_BLOCK_DATA;
++}
++
++static const struct i2c_algorithm smbus_algorithm = {
++ .smbus_xfer = nct6775_access,
++ .functionality = nct6775_func,
++};
++
++static int nct6775_add_adapter(unsigned short smba, const char *name, struct i2c_adapter **padap)
++{
++ struct i2c_adapter *adap;
++ struct i2c_nct6775_adapdata *adapdata;
++ int retval;
++
++ adap = kzalloc(sizeof(*adap), GFP_KERNEL);
++ if (adap == NULL) {
++ return -ENOMEM;
++ }
++
++ adap->owner = THIS_MODULE;
++ adap->class = I2C_CLASS_HWMON | I2C_CLASS_SPD;
++ adap->algo = &smbus_algorithm;
++
++ adapdata = kzalloc(sizeof(*adapdata), GFP_KERNEL);
++ if (adapdata == NULL) {
++ kfree(adap);
++ return -ENOMEM;
++ }
++
++ adapdata->smba = smba;
++
++ snprintf(adap->name, sizeof(adap->name),
++ "SMBus NCT67xx adapter%s at %04x", name, smba);
++
++ i2c_set_adapdata(adap, adapdata);
++
++ retval = i2c_add_adapter(adap);
++ if (retval) {
++ kfree(adapdata);
++ kfree(adap);
++ return retval;
++ }
++
++ *padap = adap;
++ return 0;
++}
++
++static void nct6775_remove_adapter(struct i2c_adapter *adap)
++{
++ struct i2c_nct6775_adapdata *adapdata = i2c_get_adapdata(adap);
++
++ if (adapdata->smba) {
++ i2c_del_adapter(adap);
++ kfree(adapdata);
++ kfree(adap);
++ }
++}
++
++//static SIMPLE_DEV_PM_OPS(nct6775_dev_pm_ops, nct6775_suspend, nct6775_resume);
++
++/*
++ * when Super-I/O functions move to a separate file, the Super-I/O
++ * bus will manage the lifetime of the device and this module will only keep
++ * track of the nct6775 driver. But since we use platform_device_alloc(), we
++ * must keep track of the device
++ */
++static struct platform_device *pdev[2];
++
++static int nct6775_probe(struct platform_device *pdev)
++{
++ struct device *dev = &pdev->dev;
++ struct nct6775_sio_data *sio_data = dev_get_platdata(dev);
++ struct resource *res;
++
++ res = platform_get_resource(pdev, IORESOURCE_IO, 0);
++ if (!devm_request_region(&pdev->dev, res->start, IOREGION_LENGTH,
++ DRVNAME))
++ return -EBUSY;
++
++ switch (sio_data->kind) {
++ case nct6791:
++ case nct6792:
++ case nct6793:
++ case nct6795:
++ case nct6796:
++ case nct6798:
++ nct6775_add_adapter(res->start, "", &nct6775_adapter);
++ break;
++ default:
++ return -ENODEV;
++ }
++
++ return 0;
++}
++/*
++static void nct6791_enable_io_mapping(int sioaddr)
++{
++ int val;
++
++ val = superio_inb(sioaddr, NCT6791_REG_HM_IO_SPACE_LOCK_ENABLE);
++ if (val & 0x10) {
++ pr_info("Enabling hardware monitor logical device mappings.\n");
++ superio_outb(sioaddr, NCT6791_REG_HM_IO_SPACE_LOCK_ENABLE,
++ val & ~0x10);
++ }
++}*/
++
++static struct platform_driver i2c_nct6775_driver = {
++ .driver = {
++ .name = DRVNAME,
++// .pm = &nct6775_dev_pm_ops,
++ },
++ .probe = nct6775_probe,
++};
++
++static void __exit i2c_nct6775_exit(void)
++{
++ int i;
++
++ if(nct6775_adapter)
++ nct6775_remove_adapter(nct6775_adapter);
++
++ for (i = 0; i < ARRAY_SIZE(pdev); i++) {
++ if (pdev[i])
++ platform_device_unregister(pdev[i]);
++ }
++ platform_driver_unregister(&i2c_nct6775_driver);
++}
++
++/* nct6775_find() looks for a '627 in the Super-I/O config space */
++static int __init nct6775_find(int sioaddr, struct nct6775_sio_data *sio_data)
++{
++ u16 val;
++ int err;
++ int addr;
++
++ err = superio_enter(sioaddr);
++ if (err)
++ return err;
++
++ val = (superio_inb(sioaddr, SIO_REG_DEVID) << 8) |
++ superio_inb(sioaddr, SIO_REG_DEVID + 1);
++
++ switch (val & SIO_ID_MASK) {
++ case SIO_NCT6106_ID:
++ sio_data->kind = nct6106;
++ break;
++ case SIO_NCT6775_ID:
++ sio_data->kind = nct6775;
++ break;
++ case SIO_NCT6776_ID:
++ sio_data->kind = nct6776;
++ break;
++ case SIO_NCT6779_ID:
++ sio_data->kind = nct6779;
++ break;
++ case SIO_NCT6791_ID:
++ sio_data->kind = nct6791;
++ break;
++ case SIO_NCT6792_ID:
++ sio_data->kind = nct6792;
++ break;
++ case SIO_NCT6793_ID:
++ sio_data->kind = nct6793;
++ break;
++ case SIO_NCT6795_ID:
++ sio_data->kind = nct6795;
++ break;
++ case SIO_NCT6796_ID:
++ sio_data->kind = nct6796;
++ break;
++ case SIO_NCT6798_ID:
++ sio_data->kind = nct6798;
++ break;
++ default:
++ if (val != 0xffff)
++ pr_debug("unsupported chip ID: 0x%04x\n", val);
++ superio_exit(sioaddr);
++ return -ENODEV;
++ }
++
++ /* We have a known chip, find the SMBus I/O address */
++ superio_select(sioaddr, NCT6775_LD_SMBUS);
++ val = (superio_inb(sioaddr, SIO_REG_SMBA) << 8)
++ | superio_inb(sioaddr, SIO_REG_SMBA + 1);
++ addr = val & IOREGION_ALIGNMENT;
++ if (addr == 0) {
++ pr_err("Refusing to enable a Super-I/O device with a base I/O port 0\n");
++ superio_exit(sioaddr);
++ return -ENODEV;
++ }
++
++ //if (sio_data->kind == nct6791 || sio_data->kind == nct6792 ||
++ // sio_data->kind == nct6793 || sio_data->kind == nct6795 ||
++ // sio_data->kind == nct6796)
++ // nct6791_enable_io_mapping(sioaddr);
++
++ superio_exit(sioaddr);
++ pr_info("Found %s or compatible chip at %#x:%#x\n",
++ nct6775_sio_names[sio_data->kind], sioaddr, addr);
++ sio_data->sioreg = sioaddr;
++
++ return addr;
++}
++
++static int __init i2c_nct6775_init(void)
++{
++ int i, err;
++ bool found = false;
++ int address;
++ struct resource res;
++ struct nct6775_sio_data sio_data;
++ int sioaddr[2] = { 0x2e, 0x4e };
++
++ err = platform_driver_register(&i2c_nct6775_driver);
++ if (err)
++ return err;
++
++ /*
++ * initialize sio_data->kind and sio_data->sioreg.
++ *
++ * when Super-I/O functions move to a separate file, the Super-I/O
++ * driver will probe 0x2e and 0x4e and auto-detect the presence of a
++ * nct6775 hardware monitor, and call probe()
++ */
++ for (i = 0; i < ARRAY_SIZE(pdev); i++) {
++ address = nct6775_find(sioaddr[i], &sio_data);
++ if (address <= 0)
++ continue;
++
++ found = true;
++
++ pdev[i] = platform_device_alloc(DRVNAME, address);
++ if (!pdev[i]) {
++ err = -ENOMEM;
++ goto exit_device_unregister;
++ }
++
++ err = platform_device_add_data(pdev[i], &sio_data,
++ sizeof(struct nct6775_sio_data));
++ if (err)
++ goto exit_device_put;
++
++ memset(&res, 0, sizeof(res));
++ res.name = DRVNAME;
++ res.start = address;
++ res.end = address + IOREGION_LENGTH - 1;
++ res.flags = IORESOURCE_IO;
++
++ err = acpi_check_resource_conflict(&res);
++ if (err) {
++ platform_device_put(pdev[i]);
++ pdev[i] = NULL;
++ continue;
++ }
++
++ err = platform_device_add_resources(pdev[i], &res, 1);
++ if (err)
++ goto exit_device_put;
++
++ /* platform_device_add calls probe() */
++ err = platform_device_add(pdev[i]);
++ if (err)
++ goto exit_device_put;
++ }
++ if (!found) {
++ err = -ENODEV;
++ goto exit_unregister;
++ }
++
++ return 0;
++
++exit_device_put:
++ platform_device_put(pdev[i]);
++exit_device_unregister:
++ while (--i >= 0) {
++ if (pdev[i])
++ platform_device_unregister(pdev[i]);
++ }
++exit_unregister:
++ platform_driver_unregister(&i2c_nct6775_driver);
++ return err;
++}
++
++MODULE_AUTHOR("Adam Honse <[email protected]>");
++MODULE_DESCRIPTION("SMBus driver for NCT6775F and compatible chips");
++MODULE_LICENSE("GPL");
++
++module_init(i2c_nct6775_init);
++module_exit(i2c_nct6775_exit);
+diff --git a/drivers/i2c/busses/i2c-piix4.c b/drivers/i2c/busses/i2c-piix4.c
+index 8c1b31ed0c42..0b1b36fa0b71 100644
+--- a/drivers/i2c/busses/i2c-piix4.c
++++ b/drivers/i2c/busses/i2c-piix4.c
+@@ -467,11 +467,11 @@ static int piix4_transaction(struct i2c_adapter *piix4_adapter)
+ if (srvrworks_csb5_delay) /* Extra delay for SERVERWORKS_CSB5 */
+ usleep_range(2000, 2100);
+ else
+- usleep_range(250, 500);
++ usleep_range(25, 50);
+
+ while ((++timeout < MAX_TIMEOUT) &&
+ ((temp = inb_p(SMBHSTSTS)) & 0x01))
+- usleep_range(250, 500);
++ usleep_range(25, 50);
+
+ /* If the SMBus is still busy, we give up */
+ if (timeout == MAX_TIMEOUT) {
+diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
+index 65e3e7df8a4b..b41afee7747e 100644
+--- a/drivers/infiniband/core/addr.c
++++ b/drivers/infiniband/core/addr.c
+@@ -821,6 +821,7 @@ int rdma_addr_find_l2_eth_by_grh(const union ib_gid *sgid,
+ union {
+ struct sockaddr_in _sockaddr_in;
+ struct sockaddr_in6 _sockaddr_in6;
++ struct sockaddr_ib _sockaddr_ib;
+ } sgid_addr, dgid_addr;
+ int ret;
+
+diff --git a/drivers/pci/quirks.c b/drivers/pci/quirks.c
+index d2dd6a6cda60..ccb05be311ff 100644
+--- a/drivers/pci/quirks.c
++++ b/drivers/pci/quirks.c
+@@ -3600,6 +3600,106 @@ static void quirk_nvidia_no_bus_reset(struct pci_dev *dev)
+ DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_NVIDIA, PCI_ANY_ID,
+ quirk_nvidia_no_bus_reset);
+
++static bool acs_on_downstream;
++static bool acs_on_multifunction;
++
++#define NUM_ACS_IDS 16
++struct acs_on_id {
++ unsigned short vendor;
++ unsigned short device;
++};
++static struct acs_on_id acs_on_ids[NUM_ACS_IDS];
++static u8 max_acs_id;
++
++static __init int pcie_acs_override_setup(char *p)
++{
++ if (!p)
++ return -EINVAL;
++
++ while (*p) {
++ if (!strncmp(p, "downstream", 10))
++ acs_on_downstream = true;
++ if (!strncmp(p, "multifunction", 13))
++ acs_on_multifunction = true;
++ if (!strncmp(p, "id:", 3)) {
++ char opt[5];
++ int ret;
++ long val;
++
++ if (max_acs_id >= NUM_ACS_IDS - 1) {
++ pr_warn("Out of PCIe ACS override slots (%d)\n",
++ NUM_ACS_IDS);
++ goto next;
++ }
++
++ p += 3;
++ snprintf(opt, 5, "%s", p);
++ ret = kstrtol(opt, 16, &val);
++ if (ret) {
++ pr_warn("PCIe ACS ID parse error %d\n", ret);
++ goto next;
++ }
++ acs_on_ids[max_acs_id].vendor = val;
++
++ p += strcspn(p, ":");
++ if (*p != ':') {
++ pr_warn("PCIe ACS invalid ID\n");
++ goto next;
++ }
++
++ p++;
++ snprintf(opt, 5, "%s", p);
++ ret = kstrtol(opt, 16, &val);
++ if (ret) {
++ pr_warn("PCIe ACS ID parse error %d\n", ret);
++ goto next;
++ }
++ acs_on_ids[max_acs_id].device = val;
++ max_acs_id++;
++ }
++next:
++ p += strcspn(p, ",");
++ if (*p == ',')
++ p++;
++ }
++
++ if (acs_on_downstream || acs_on_multifunction || max_acs_id)
++ pr_warn("Warning: PCIe ACS overrides enabled; This may allow non-IOMMU protected peer-to-peer DMA\n");
++
++ return 0;
++}
++early_param("pcie_acs_override", pcie_acs_override_setup);
++
++static int pcie_acs_overrides(struct pci_dev *dev, u16 acs_flags)
++{
++ int i;
++
++ /* Never override ACS for legacy devices or devices with ACS caps */
++ if (!pci_is_pcie(dev) ||
++ pci_find_ext_capability(dev, PCI_EXT_CAP_ID_ACS))
++ return -ENOTTY;
++
++ for (i = 0; i < max_acs_id; i++)
++ if (acs_on_ids[i].vendor == dev->vendor &&
++ acs_on_ids[i].device == dev->device)
++ return 1;
++
++ switch (pci_pcie_type(dev)) {
++ case PCI_EXP_TYPE_DOWNSTREAM:
++ case PCI_EXP_TYPE_ROOT_PORT:
++ if (acs_on_downstream)
++ return 1;
++ break;
++ case PCI_EXP_TYPE_ENDPOINT:
++ case PCI_EXP_TYPE_UPSTREAM:
++ case PCI_EXP_TYPE_LEG_END:
++ case PCI_EXP_TYPE_RC_END:
++ if (acs_on_multifunction && dev->multifunction)
++ return 1;
++ }
++
++ return -ENOTTY;
++}
+ /*
+ * Some Atheros AR9xxx and QCA988x chips do not behave after a bus reset.
+ * The device will throw a Link Down error on AER-capable systems and
+@@ -4953,6 +5053,8 @@ static const struct pci_dev_acs_enabled {
+ { PCI_VENDOR_ID_NXP, 0x8d9b, pci_quirk_nxp_rp_acs },
+ /* Zhaoxin Root/Downstream Ports */
+ { PCI_VENDOR_ID_ZHAOXIN, PCI_ANY_ID, pci_quirk_zhaoxin_pcie_ports_acs },
++ /* PCIe ACS overrides */
++ { PCI_ANY_ID, PCI_ANY_ID, pcie_acs_overrides },
+ { 0 }
+ };
+
+diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
+index 62eb9921cc94..58952be562f6 100644
+--- a/drivers/scsi/sd.c
++++ b/drivers/scsi/sd.c
+@@ -2947,7 +2947,6 @@ static void sd_read_app_tag_own(struct scsi_disk *sdkp, unsigned char *buffer)
+ */
+ static void sd_read_block_limits(struct scsi_disk *sdkp)
+ {
+- unsigned int sector_sz = sdkp->device->sector_size;
+ const int vpd_len = 64;
+ unsigned char *buffer = kmalloc(vpd_len, GFP_KERNEL);
+
+@@ -2956,9 +2955,7 @@ static void sd_read_block_limits(struct scsi_disk *sdkp)
+ scsi_get_vpd_page(sdkp->device, 0xb0, buffer, vpd_len))
+ goto out;
+
+- blk_queue_io_min(sdkp->disk->queue,
+- get_unaligned_be16(&buffer[6]) * sector_sz);
+-
++ sdkp->min_xfer_blocks = get_unaligned_be16(&buffer[6]);
+ sdkp->max_xfer_blocks = get_unaligned_be32(&buffer[8]);
+ sdkp->opt_xfer_blocks = get_unaligned_be32(&buffer[12]);
+
+@@ -3214,6 +3211,29 @@ static void sd_read_cpr(struct scsi_disk *sdkp)
+ kfree(buffer);
+ }
+
++static bool sd_validate_min_xfer_size(struct scsi_disk *sdkp)
++{
++ struct scsi_device *sdp = sdkp->device;
++ unsigned int min_xfer_bytes =
++ logical_to_bytes(sdp, sdkp->min_xfer_blocks);
++
++ if (sdkp->min_xfer_blocks == 0)
++ return false;
++
++ if (min_xfer_bytes & (sdkp->physical_block_size - 1)) {
++ sd_first_printk(KERN_WARNING, sdkp,
++ "Preferred minimum I/O size %u bytes not a " \
++ "multiple of physical block size (%u bytes)\n",
++ min_xfer_bytes, sdkp->physical_block_size);
++ sdkp->min_xfer_blocks = 0;
++ return false;
++ }
++
++ sd_first_printk(KERN_INFO, sdkp, "Preferred minimum I/O size %u bytes\n",
++ min_xfer_bytes);
++ return true;
++}
++
+ /*
+ * Determine the device's preferred I/O size for reads and writes
+ * unless the reported value is unreasonably small, large, not a
+@@ -3225,6 +3245,8 @@ static bool sd_validate_opt_xfer_size(struct scsi_disk *sdkp,
+ struct scsi_device *sdp = sdkp->device;
+ unsigned int opt_xfer_bytes =
+ logical_to_bytes(sdp, sdkp->opt_xfer_blocks);
++ unsigned int min_xfer_bytes =
++ logical_to_bytes(sdp, sdkp->min_xfer_blocks);
+
+ if (sdkp->opt_xfer_blocks == 0)
+ return false;
+@@ -3261,6 +3283,15 @@ static bool sd_validate_opt_xfer_size(struct scsi_disk *sdkp,
+ return false;
+ }
+
++ if (min_xfer_bytes && opt_xfer_bytes & (min_xfer_bytes - 1)) {
++ sd_first_printk(KERN_WARNING, sdkp,
++ "Optimal transfer size %u bytes not a " \
++ "multiple of preferred minimum block " \
++ "size (%u bytes)\n",
++ opt_xfer_bytes, min_xfer_bytes);
++ return false;
++ }
++
+ sd_first_printk(KERN_INFO, sdkp, "Optimal transfer size %u bytes\n",
+ opt_xfer_bytes);
+ return true;
+@@ -3345,6 +3376,10 @@ static int sd_revalidate_disk(struct gendisk *disk)
+ dev_max = min_not_zero(dev_max, sdkp->max_xfer_blocks);
+ q->limits.max_dev_sectors = logical_to_sectors(sdp, dev_max);
+
++ if (sd_validate_min_xfer_size(sdkp))
++ blk_queue_io_min(sdkp->disk->queue,
++ logical_to_bytes(sdp, sdkp->min_xfer_blocks));
++
+ if (sd_validate_opt_xfer_size(sdkp, dev_max)) {
+ q->limits.io_opt = logical_to_bytes(sdp, sdkp->opt_xfer_blocks);
+ rw_max = logical_to_sectors(sdp, sdkp->opt_xfer_blocks);
+diff --git a/drivers/scsi/sd.h b/drivers/scsi/sd.h
+index 2e5932bde43d..f4fbca90e997 100644
+--- a/drivers/scsi/sd.h
++++ b/drivers/scsi/sd.h
+@@ -91,6 +91,7 @@ struct scsi_disk {
+ atomic_t openers;
+ sector_t capacity; /* size in logical blocks */
+ int max_retries;
++ u32 min_xfer_blocks;
+ u32 max_xfer_blocks;
+ u32 opt_xfer_blocks;
+ u32 max_ws_blocks;
+diff --git a/drivers/tty/Kconfig b/drivers/tty/Kconfig
+index cc30ff93e2e4..812116d48064 100644
+--- a/drivers/tty/Kconfig
++++ b/drivers/tty/Kconfig
+@@ -75,6 +75,19 @@ config VT_CONSOLE_SLEEP
+ def_bool y
+ depends on VT_CONSOLE && PM_SLEEP
+
++config NR_TTY_DEVICES
++ int "Maximum tty device number"
++ depends on VT
++ range 12 63
++ default 63
++ help
++ This option is used to change the number of tty devices in /dev.
++ The default value is 63. The lowest number you can set is 12,
++ 63 is also the upper limit so we don't overrun the serial
++ consoles.
++
++ If unsure, say 63.
++
+ config HW_CONSOLE
+ bool
+ depends on VT
+diff --git a/include/linux/pagemap.h b/include/linux/pagemap.h
+index 270bf5136c34..13207a15b326 100644
+--- a/include/linux/pagemap.h
++++ b/include/linux/pagemap.h
+@@ -1024,7 +1024,7 @@ struct readahead_control {
+ ._index = i, \
+ }
+
+-#define VM_READAHEAD_PAGES (SZ_128K / PAGE_SIZE)
++#define VM_READAHEAD_PAGES (SZ_8M / PAGE_SIZE)
+
+ void page_cache_ra_unbounded(struct readahead_control *,
+ unsigned long nr_to_read, unsigned long lookahead_count);
+diff --git a/include/linux/user_namespace.h b/include/linux/user_namespace.h
+index 33a4240e6a6f..82213f9c4c17 100644
+--- a/include/linux/user_namespace.h
++++ b/include/linux/user_namespace.h
+@@ -139,6 +139,8 @@ static inline void set_rlimit_ucount_max(struct user_namespace *ns,
+
+ #ifdef CONFIG_USER_NS
+
++extern int unprivileged_userns_clone;
++
+ static inline struct user_namespace *get_user_ns(struct user_namespace *ns)
+ {
+ if (ns)
+@@ -172,6 +174,8 @@ extern bool current_in_userns(const struct user_namespace *target_ns);
+ struct ns_common *ns_get_owner(struct ns_common *ns);
+ #else
+
++#define unprivileged_userns_clone 0
++
+ static inline struct user_namespace *get_user_ns(struct user_namespace *ns)
+ {
+ return &init_user_ns;
+diff --git a/include/uapi/linux/futex.h b/include/uapi/linux/futex.h
+index 71a5df8d2689..cf7fc988c24c 100644
+--- a/include/uapi/linux/futex.h
++++ b/include/uapi/linux/futex.h
+@@ -22,6 +22,7 @@
+ #define FUTEX_WAIT_REQUEUE_PI 11
+ #define FUTEX_CMP_REQUEUE_PI 12
+ #define FUTEX_LOCK_PI2 13
++#define FUTEX_WAIT_MULTIPLE 31
+
+ #define FUTEX_PRIVATE_FLAG 128
+ #define FUTEX_CLOCK_REALTIME 256
+@@ -68,6 +69,18 @@ struct futex_waitv {
+ __u32 __reserved;
+ };
+
++/**
++ * struct futex_wait_block - Block of futexes to be waited for
++ * @uaddr: User address of the futex
++ * @val: Futex value expected by userspace
++ * @bitset: Bitset for the optional bitmasked wakeup
++ */
++struct futex_wait_block {
++ __u32 __user *uaddr;
++ __u32 val;
++ __u32 bitset;
++};
++
+ /*
+ * Support for robust futexes: the kernel cleans up held futexes at
+ * thread exit time.
+diff --git a/include/uapi/linux/vt.h b/include/uapi/linux/vt.h
+index e9d39c48520a..3bceead8da40 100644
+--- a/include/uapi/linux/vt.h
++++ b/include/uapi/linux/vt.h
+@@ -3,12 +3,25 @@
+ #define _UAPI_LINUX_VT_H
+
+
++/*
++ * We will make this definition solely for the purpose of making packages
++ * such as splashutils build, because they can not understand that
++ * NR_TTY_DEVICES is defined in the kernel configuration.
++ */
++#ifndef CONFIG_NR_TTY_DEVICES
++#define CONFIG_NR_TTY_DEVICES 63
++#endif
++
+ /*
+ * These constants are also useful for user-level apps (e.g., VC
+ * resizing).
+ */
+ #define MIN_NR_CONSOLES 1 /* must be at least 1 */
+-#define MAX_NR_CONSOLES 63 /* serial lines start at 64 */
++/*
++ * NR_TTY_DEVICES:
++ * Value MUST be at least 12 and must never be higher then 63
++ */
++#define MAX_NR_CONSOLES CONFIG_NR_TTY_DEVICES /* serial lines start above this */
+ /* Note: the ioctl VT_GETSTATE does not work for
+ consoles 16 and higher (since it returns a short) */
+
+diff --git a/init/Kconfig b/init/Kconfig
+index e9119bf54b1f..a1a5a68aea07 100644
+--- a/init/Kconfig
++++ b/init/Kconfig
+@@ -1231,6 +1231,22 @@ config USER_NS
+
+ If unsure, say N.
+
++config USER_NS_UNPRIVILEGED
++ bool "Allow unprivileged users to create namespaces"
++ default y
++ depends on USER_NS
++ help
++ When disabled, unprivileged users will not be able to create
++ new namespaces. Allowing users to create their own namespaces
++ has been part of several recent local privilege escalation
++ exploits, so if you need user namespaces but are
++ paranoid^Wsecurity-conscious you want to disable this.
++
++ This setting can be overridden at runtime via the
++ kernel.unprivileged_userns_clone sysctl.
++
++ If unsure, say Y.
++
+ config PID_NS
+ bool "PID Namespaces"
+ default y
+diff --git a/kernel/Kconfig.hz b/kernel/Kconfig.hz
+index 38ef6d06888e..9948a1f1fb42 100644
+--- a/kernel/Kconfig.hz
++++ b/kernel/Kconfig.hz
+@@ -40,6 +40,41 @@ choice
+ on SMP and NUMA systems and exactly dividing by both PAL and
+ NTSC frame rates for video and multimedia work.
+
++ config HZ_500
++ bool "500 HZ"
++ help
++ 500 Hz is a balanced timer frequency. Provides fast interactivity
++ on desktops with good smoothness without increasing CPU power
++ consumption and sacrificing the battery life on laptops.
++
++ config HZ_600
++ bool "600 HZ"
++ help
++ 600 Hz is a balanced timer frequency. Provides fast interactivity
++ on desktops with good smoothness without increasing CPU power
++ consumption and sacrificing the battery life on laptops.
++
++ config HZ_750
++ bool "750 HZ"
++ help
++ 750 Hz is a balanced timer frequency. Provides fast interactivity
++ on desktops with good smoothness without increasing CPU power
++ consumption and sacrificing the battery life on laptops.
++
++ config HZ_600
++ bool "600 HZ"
++ help
++ 600 Hz is a balanced timer frequency. Provides fast interactivity
++ on desktops with good smoothness without increasing CPU power
++ consumption and sacrificing the battery life on laptops.
++
++ config HZ_750
++ bool "750 HZ"
++ help
++ 750 Hz is a balanced timer frequency. Provides fast interactivity
++ on desktops with good smoothness without increasing CPU power
++ consumption and sacrificing the battery life on laptops.
++
+ config HZ_1000
+ bool "1000 HZ"
+ help
+@@ -53,6 +88,9 @@ config HZ
+ default 100 if HZ_100
+ default 250 if HZ_250
+ default 300 if HZ_300
++ default 500 if HZ_500
++ default 600 if HZ_600
++ default 750 if HZ_750
+ default 1000 if HZ_1000
+
+ config SCHED_HRTICK
+diff --git a/kernel/cpu.c b/kernel/cpu.c
+index 407a2568f35e..cd7605204d4d 100644
+--- a/kernel/cpu.c
++++ b/kernel/cpu.c
+@@ -2616,6 +2616,7 @@ void init_cpu_possible(const struct cpumask *src)
+ void init_cpu_online(const struct cpumask *src)
+ {
+ cpumask_copy(&__cpu_online_mask, src);
++ atomic_set(&__num_online_cpus, cpumask_weight(cpu_online_mask));
+ }
+
+ void set_cpu_online(unsigned int cpu, bool online)
+diff --git a/kernel/fork.c b/kernel/fork.c
+index d75a528f7b21..cc89aa4d3a1f 100644
+--- a/kernel/fork.c
++++ b/kernel/fork.c
+@@ -98,6 +98,10 @@
+ #include <linux/io_uring.h>
+ #include <linux/bpf.h>
+
++#ifdef CONFIG_USER_NS
++#include <linux/user_namespace.h>
++#endif
++
+ #include <asm/pgalloc.h>
+ #include <linux/uaccess.h>
+ #include <asm/mmu_context.h>
+@@ -1922,6 +1926,10 @@ static __latent_entropy struct task_struct *copy_process(
+ if ((clone_flags & (CLONE_NEWUSER|CLONE_FS)) == (CLONE_NEWUSER|CLONE_FS))
+ return ERR_PTR(-EINVAL);
+
++ if ((clone_flags & CLONE_NEWUSER) && !unprivileged_userns_clone)
++ if (!capable(CAP_SYS_ADMIN))
++ return ERR_PTR(-EPERM);
++
+ /*
+ * Thread groups must share signals as well, and detached threads
+ * can only be started up within the thread group.
+@@ -3026,6 +3034,12 @@ int ksys_unshare(unsigned long unshare_flags)
+ if (unshare_flags & CLONE_NEWNS)
+ unshare_flags |= CLONE_FS;
+
++ if ((unshare_flags & CLONE_NEWUSER) && !unprivileged_userns_clone) {
++ err = -EPERM;
++ if (!capable(CAP_SYS_ADMIN))
++ goto bad_unshare_out;
++ }
++
+ err = check_unshare_flags(unshare_flags);
+ if (err)
+ goto bad_unshare_out;
+diff --git a/kernel/futex/syscalls.c b/kernel/futex/syscalls.c
+index 086a22d1adb7..c6f5f1e84e09 100644
+--- a/kernel/futex/syscalls.c
++++ b/kernel/futex/syscalls.c
+@@ -142,6 +142,7 @@ static __always_inline bool futex_cmd_has_timeout(u32 cmd)
+ case FUTEX_LOCK_PI2:
+ case FUTEX_WAIT_BITSET:
+ case FUTEX_WAIT_REQUEUE_PI:
++ case FUTEX_WAIT_MULTIPLE:
+ return true;
+ }
+ return false;
+@@ -154,13 +155,79 @@ futex_init_timeout(u32 cmd, u32 op, struct timespec64 *ts, ktime_t *t)
+ return -EINVAL;
+
+ *t = timespec64_to_ktime(*ts);
+- if (cmd == FUTEX_WAIT)
++ if (cmd == FUTEX_WAIT || cmd == FUTEX_WAIT_MULTIPLE)
+ *t = ktime_add_safe(ktime_get(), *t);
+ else if (cmd != FUTEX_LOCK_PI && !(op & FUTEX_CLOCK_REALTIME))
+ *t = timens_ktime_to_host(CLOCK_MONOTONIC, *t);
+ return 0;
+ }
+
++/**
++ * futex_read_wait_block - Read an array of futex_wait_block from userspace
++ * @uaddr: Userspace address of the block
++ * @count: Number of blocks to be read
++ *
++ * This function creates and allocate an array of futex_q (we zero it to
++ * initialize the fields) and then, for each futex_wait_block element from
++ * userspace, fill a futex_q element with proper values.
++ */
++inline struct futex_vector *futex_read_wait_block(u32 __user *uaddr, u32 count)
++{
++ unsigned int i;
++ struct futex_vector *futexv;
++ struct futex_wait_block fwb;
++ struct futex_wait_block __user *entry =
++ (struct futex_wait_block __user *)uaddr;
++
++ if (!count || count > FUTEX_WAITV_MAX)
++ return ERR_PTR(-EINVAL);
++
++ futexv = kcalloc(count, sizeof(*futexv), GFP_KERNEL);
++ if (!futexv)
++ return ERR_PTR(-ENOMEM);
++
++ for (i = 0; i < count; i++) {
++ if (copy_from_user(&fwb, &entry[i], sizeof(fwb))) {
++ kfree(futexv);
++ return ERR_PTR(-EFAULT);
++ }
++
++ futexv[i].w.flags = FUTEX_32;
++ futexv[i].w.val = fwb.val;
++ futexv[i].w.uaddr = (uintptr_t) (fwb.uaddr);
++ futexv[i].q = futex_q_init;
++ }
++
++ return futexv;
++}
++
++int futex_wait_multiple(struct futex_vector *vs, unsigned int count,
++ struct hrtimer_sleeper *to);
++
++int futex_opcode_31(ktime_t *abs_time, u32 __user *uaddr, int count)
++{
++ int ret;
++ struct futex_vector *vs;
++ struct hrtimer_sleeper *to = NULL, timeout;
++
++ to = futex_setup_timer(abs_time, &timeout, 0, 0);
++
++ vs = futex_read_wait_block(uaddr, count);
++
++ if (IS_ERR(vs))
++ return PTR_ERR(vs);
++
++ ret = futex_wait_multiple(vs, count, abs_time ? to : NULL);
++ kfree(vs);
++
++ if (to) {
++ hrtimer_cancel(&to->timer);
++ destroy_hrtimer_on_stack(&to->timer);
++ }
++
++ return ret;
++}
++
+ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
+ const struct __kernel_timespec __user *, utime,
+ u32 __user *, uaddr2, u32, val3)
+@@ -180,6 +247,9 @@ SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
+ tp = &t;
+ }
+
++ if (cmd == FUTEX_WAIT_MULTIPLE)
++ return futex_opcode_31(tp, uaddr, val);
++
+ return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
+ }
+
+@@ -370,6 +440,9 @@ SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,
+ tp = &t;
+ }
+
++ if (cmd == FUTEX_WAIT_MULTIPLE)
++ return futex_opcode_31(tp, uaddr, val);
++
+ return do_futex(uaddr, op, val, tp, uaddr2, (unsigned long)utime, val3);
+ }
+ #endif /* CONFIG_COMPAT_32BIT_TIME */
+diff --git a/kernel/sched/core.c b/kernel/sched/core.c
+index 848eaa0efe0e..afb671d71c59 100644
+--- a/kernel/sched/core.c
++++ b/kernel/sched/core.c
+@@ -383,7 +383,7 @@ sched_core_dequeue(struct rq *rq, struct task_struct *p, int flags) { }
+ * part of the period that we allow rt tasks to run in us.
+ * default: 0.95s
+ */
+-int sysctl_sched_rt_runtime = 950000;
++int sysctl_sched_rt_runtime = 980000;
+
+
+ /*
+diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
+index 5146163bfabb..47d9243bf250 100644
+--- a/kernel/sched/fair.c
++++ b/kernel/sched/fair.c
+@@ -35,8 +35,8 @@
+ *
+ * (default: 6ms * (1 + ilog(ncpus)), units: nanoseconds)
+ */
+-unsigned int sysctl_sched_latency = 6000000ULL;
+-static unsigned int normalized_sysctl_sched_latency = 6000000ULL;
++unsigned int sysctl_sched_latency = 4000000ULL;
++static unsigned int normalized_sysctl_sched_latency = 4000000ULL;
+
+ /*
+ * The initial- and re-scaling of tunables is configurable
+@@ -56,8 +56,8 @@ unsigned int sysctl_sched_tunable_scaling = SCHED_TUNABLESCALING_LOG;
+ *
+ * (default: 0.75 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ */
+-unsigned int sysctl_sched_min_granularity = 750000ULL;
+-static unsigned int normalized_sysctl_sched_min_granularity = 750000ULL;
++unsigned int sysctl_sched_min_granularity = 500000ULL;
++static unsigned int normalized_sysctl_sched_min_granularity = 500000ULL;
+
+ /*
+ * Minimal preemption granularity for CPU-bound SCHED_IDLE tasks.
+@@ -87,8 +87,8 @@ unsigned int sysctl_sched_child_runs_first __read_mostly;
+ *
+ * (default: 1 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ */
+-unsigned int sysctl_sched_wakeup_granularity = 1000000UL;
+-static unsigned int normalized_sysctl_sched_wakeup_granularity = 1000000UL;
++unsigned int sysctl_sched_wakeup_granularity = 800000UL;
++static unsigned int normalized_sysctl_sched_wakeup_granularity = 800000UL;
+
+ const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
+
+@@ -141,7 +141,7 @@ int __weak arch_asym_cpu_priority(int cpu)
+ *
+ * (default: 5 msec, units: microseconds)
+ */
+-unsigned int sysctl_sched_cfs_bandwidth_slice = 5000UL;
++unsigned int sysctl_sched_cfs_bandwidth_slice = 4000UL;
+ #endif
+
+ static inline void update_load_add(struct load_weight *lw, unsigned long inc)
+diff --git a/kernel/sysctl.c b/kernel/sysctl.c
+index 5ae443b2882e..bfbd398505d2 100644
+--- a/kernel/sysctl.c
++++ b/kernel/sysctl.c
+@@ -99,6 +99,9 @@
+ #ifdef CONFIG_PERF_EVENTS
+ static const int six_hundred_forty_kb = 640 * 1024;
+ #endif
++#ifdef CONFIG_USER_NS
++#include <linux/user_namespace.h>
++#endif
+
+ /* this is needed for the proc_doulongvec_minmax of vm_dirty_bytes */
+ static const unsigned long dirty_bytes_min = 2 * PAGE_SIZE;
+@@ -2005,6 +2008,15 @@ static struct ctl_table kern_table[] = {
+ .proc_handler = sysrq_sysctl_handler,
+ },
+ #endif
++#ifdef CONFIG_USER_NS
++ {
++ .procname = "unprivileged_userns_clone",
++ .data = &unprivileged_userns_clone,
++ .maxlen = sizeof(int),
++ .mode = 0644,
++ .proc_handler = proc_dointvec,
++ },
++#endif
+ #ifdef CONFIG_PROC_SYSCTL
+ {
+ .procname = "cad_pid",
+diff --git a/kernel/user_namespace.c b/kernel/user_namespace.c
+index 6b2e3ca7ee99..0253002184f1 100644
+--- a/kernel/user_namespace.c
++++ b/kernel/user_namespace.c
+@@ -21,6 +21,13 @@
+ #include <linux/bsearch.h>
+ #include <linux/sort.h>
+
++/* sysctl */
++#ifdef CONFIG_USER_NS_UNPRIVILEGED
++int unprivileged_userns_clone = 1;
++#else
++int unprivileged_userns_clone;
++#endif
++
+ static struct kmem_cache *user_ns_cachep __read_mostly;
+ static DEFINE_MUTEX(userns_state_mutex);
+
+diff --git a/mm/internal.h b/mm/internal.h
+index d80300392a19..96aa50e46119 100644
+--- a/mm/internal.h
++++ b/mm/internal.h
+@@ -248,6 +248,7 @@ extern void prep_compound_page(struct page *page, unsigned int order);
+ extern void post_alloc_hook(struct page *page, unsigned int order,
+ gfp_t gfp_flags);
+ extern int user_min_free_kbytes;
++extern atomic_long_t kswapd_waiters;
+
+ extern void free_unref_page(struct page *page, unsigned int order);
+ extern void free_unref_page_list(struct list_head *list);
+diff --git a/mm/page-writeback.c b/mm/page-writeback.c
+index 91d163f8d36b..9ba34ad877ae 100644
+--- a/mm/page-writeback.c
++++ b/mm/page-writeback.c
+@@ -70,7 +70,7 @@ static long ratelimit_pages = 32;
+ /*
+ * Start background writeback (via writeback threads) at this percentage
+ */
+-int dirty_background_ratio = 10;
++int dirty_background_ratio = 5;
+
+ /*
+ * dirty_background_bytes starts at 0 (disabled) so that it is a function of
+@@ -98,7 +98,7 @@ unsigned long vm_dirty_bytes;
+ /*
+ * The interval between `kupdate'-style writebacks
+ */
+-unsigned int dirty_writeback_interval = 5 * 100; /* centiseconds */
++unsigned int dirty_writeback_interval = 10 * 100; /* centiseconds */
+
+ EXPORT_SYMBOL_GPL(dirty_writeback_interval);
+
+diff --git a/mm/page_alloc.c b/mm/page_alloc.c
+index 3589febc6d31..9c51d9c8ca13 100644
+--- a/mm/page_alloc.c
++++ b/mm/page_alloc.c
+@@ -121,6 +121,8 @@ typedef int __bitwise fpi_t;
+ */
+ #define FPI_SKIP_KASAN_POISON ((__force fpi_t)BIT(2))
+
++atomic_long_t kswapd_waiters = ATOMIC_LONG_INIT(0);
++
+ /* prevent >1 _updater_ of zone percpu pageset ->high and ->batch fields */
+ static DEFINE_MUTEX(pcp_batch_high_lock);
+ #define MIN_PERCPU_PAGELIST_HIGH_FRACTION (8)
+@@ -341,7 +343,7 @@ compound_page_dtor * const compound_page_dtors[NR_COMPOUND_DTORS] = {
+
+ int min_free_kbytes = 1024;
+ int user_min_free_kbytes = -1;
+-int watermark_boost_factor __read_mostly = 15000;
++int watermark_boost_factor __read_mostly;
+ int watermark_scale_factor = 10;
+
+ static unsigned long nr_kernel_pages __initdata;
+@@ -2683,8 +2685,11 @@ static void steal_suitable_fallback(struct zone *zone, struct page *page,
+ * likelihood of future fallbacks. Wake kswapd now as the node
+ * may be balanced overall and kswapd will not wake naturally.
+ */
+- if (boost_watermark(zone) && (alloc_flags & ALLOC_KSWAPD))
+- set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
++ if (alloc_flags & ALLOC_KSWAPD) {
++ boost_watermark(zone);
++ if (zone->watermark_boost)
++ set_bit(ZONE_BOOSTED_WATERMARK, &zone->flags);
++ }
+
+ /* We are not allowed to try stealing from the whole block */
+ if (!whole_block)
+@@ -4878,6 +4883,7 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
+ int no_progress_loops;
+ unsigned int cpuset_mems_cookie;
+ int reserve_flags;
++ bool woke_kswapd = false;
+
+ /*
+ * We also sanity check to catch abuse of atomic reserves being used by
+@@ -4924,8 +4930,13 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
+ goto nopage;
+ }
+
+- if (alloc_flags & ALLOC_KSWAPD)
++ if (alloc_flags & ALLOC_KSWAPD) {
++ if (!woke_kswapd) {
++ atomic_long_inc(&kswapd_waiters);
++ woke_kswapd = true;
++ }
+ wake_all_kswapds(order, gfp_mask, ac);
++ }
+
+ /*
+ * The adjusted alloc_flags might result in immediate success, so try
+@@ -5130,9 +5141,12 @@ __alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
+ goto retry;
+ }
+ fail:
+- warn_alloc(gfp_mask, ac->nodemask,
+- "page allocation failure: order:%u", order);
+ got_pg:
++ if (woke_kswapd)
++ atomic_long_dec(&kswapd_waiters);
++ if (!page)
++ warn_alloc(gfp_mask, ac->nodemask,
++ "page allocation failure: order:%u", order);
+ return page;
+ }
+
+diff --git a/mm/vmscan.c b/mm/vmscan.c
+index 090bfb605ecf..236e119e2e6a 100644
+--- a/mm/vmscan.c
++++ b/mm/vmscan.c
+@@ -174,7 +174,7 @@ struct scan_control {
+ /*
+ * From 0 .. 200. Higher means more swappy.
+ */
+-int vm_swappiness = 60;
++int vm_swappiness = 30;
+
+ static void set_task_reclaim_state(struct task_struct *task,
+ struct reclaim_state *rs)
+@@ -4273,7 +4273,8 @@ static int balance_pgdat(pg_data_t *pgdat, int order, int highest_zoneidx)
+ __fs_reclaim_release(_THIS_IP_);
+ ret = try_to_freeze();
+ __fs_reclaim_acquire(_THIS_IP_);
+- if (ret || kthread_should_stop())
++ if (ret || kthread_should_stop() ||
++ !atomic_long_read(&kswapd_waiters))
+ break;
+
+ /*
+diff --git a/net/sched/Kconfig b/net/sched/Kconfig
+index 1e8ab4749c6c..75122fd6506b 100644
+--- a/net/sched/Kconfig
++++ b/net/sched/Kconfig
+@@ -474,6 +474,9 @@ choice
+ config DEFAULT_SFQ
+ bool "Stochastic Fair Queue" if NET_SCH_SFQ
+
++ config DEFAULT_CAKE
++ bool "Common Applications Kept Enhanced" if NET_SCH_CAKE
++
+ config DEFAULT_PFIFO_FAST
+ bool "Priority FIFO Fast"
+ endchoice
+@@ -485,6 +488,7 @@ config DEFAULT_NET_SCH
+ default "fq_codel" if DEFAULT_FQ_CODEL
+ default "fq_pie" if DEFAULT_FQ_PIE
+ default "sfq" if DEFAULT_SFQ
++ default "cake" if DEFAULT_CAKE
+ default "pfifo_fast"
+ endif
+
+--
+2.35.1
+
+From 7fc0e544875f4ba792f2720d1cdbb490d17561c2 Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sat, 29 Jan 2022 08:12:02 +0000
+Subject: [PATCH] pf-patches
+
+---
+ drivers/i2c/i2c-core-base.c | 2 +-
+ include/linux/irqdesc.h | 1 +
+ kernel/irq/irqdesc.c | 21 +++++++++++++++++++++
+ net/mac80211/airtime.c | 12 ++++++++++--
+ net/mac80211/rate.c | 12 +++++++++---
+ net/mac80211/rc80211_minstrel_ht.c | 7 ++++++-
+ 6 files changed, 48 insertions(+), 7 deletions(-)
+
+diff --git a/drivers/i2c/i2c-core-base.c b/drivers/i2c/i2c-core-base.c
+index 2c59dd748a49..121bbad56cfa 100644
+--- a/drivers/i2c/i2c-core-base.c
++++ b/drivers/i2c/i2c-core-base.c
+@@ -1424,7 +1424,7 @@ int i2c_handle_smbus_host_notify(struct i2c_adapter *adap, unsigned short addr)
+ if (irq <= 0)
+ return -ENXIO;
+
+- generic_handle_irq(irq);
++ generic_dispatch_irq(irq);
+
+ return 0;
+ }
+diff --git a/include/linux/irqdesc.h b/include/linux/irqdesc.h
+index 93d270ca0c56..fbec0c019e3b 100644
+--- a/include/linux/irqdesc.h
++++ b/include/linux/irqdesc.h
+@@ -160,6 +160,7 @@ static inline void generic_handle_irq_desc(struct irq_desc *desc)
+
+ int handle_irq_desc(struct irq_desc *desc);
+ int generic_handle_irq(unsigned int irq);
++int generic_dispatch_irq(unsigned int irq);
+
+ #ifdef CONFIG_IRQ_DOMAIN
+ /*
+diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
+index 2267e6527db3..3e129f2e19d1 100644
+--- a/kernel/irq/irqdesc.c
++++ b/kernel/irq/irqdesc.c
+@@ -681,6 +681,27 @@ int generic_handle_domain_irq(struct irq_domain *domain, unsigned int hwirq)
+ }
+ EXPORT_SYMBOL_GPL(generic_handle_domain_irq);
+
++/**
++ * generic_dispatch_irq - Dispatch an interrupt from an interrupt handler
++ * @irq: The irq number to handle
++ *
++ * A wrapper around generic_handle_irq() which ensures that interrupts are
++ * disabled when the primary handler of the dispatched irq is invoked.
++ * This is useful for interrupt handlers with dispatching to be safe for
++ * the forced threaded case.
++ */
++int generic_dispatch_irq(unsigned int irq)
++{
++ unsigned long flags;
++ int ret;
++
++ local_irq_save(flags);
++ ret = generic_handle_irq(irq);
++ local_irq_restore(flags);
++ return ret;
++}
++EXPORT_SYMBOL_GPL(generic_dispatch_irq);
++
+ /**
+ * generic_handle_domain_nmi - Invoke the handler for a HW nmi belonging
+ * to a domain.
+diff --git a/net/mac80211/airtime.c b/net/mac80211/airtime.c
+index 26d2f8ba7029..03fb8070bdae 100644
+--- a/net/mac80211/airtime.c
++++ b/net/mac80211/airtime.c
+@@ -453,9 +453,17 @@ static u32 ieee80211_get_rate_duration(struct ieee80211_hw *hw,
+ return 0;
+ }
+
+- if (WARN_ON_ONCE((status->encoding != RX_ENC_HE && streams > 4) ||
+- (status->encoding == RX_ENC_HE && streams > 8)))
++ if (unlikely(status->encoding != RX_ENC_HE && streams > 4)) {
++ pr_warn_once("%s: status->encoding != RX_ENC_HE (%u != %d) && streams > 4 (%d > 4)\n",
++ __func__, status->encoding, RX_ENC_HE, streams);
+ return 0;
++ }
++
++ if (unlikely(status->encoding == RX_ENC_HE && streams > 8)) {
++ pr_warn_once("%s: status->encoding == RX_ENC_HE (%u == %d) && streams > 8 (%d > 8)\n",
++ __func__, status->encoding, RX_ENC_HE, streams);
++ return 0;
++ }
+
+ duration = airtime_mcs_groups[group].duration[idx];
+ duration <<= airtime_mcs_groups[group].shift;
+diff --git a/net/mac80211/rate.c b/net/mac80211/rate.c
+index 8c6416129d5b..810ea5cd6681 100644
+--- a/net/mac80211/rate.c
++++ b/net/mac80211/rate.c
+@@ -664,7 +664,9 @@ static void rate_fixup_ratelist(struct ieee80211_vif *vif,
+ * needs to be fixed.
+ */
+ if (rates[i].flags & IEEE80211_TX_RC_MCS) {
+- WARN_ON(rates[i].idx > 76);
++ if (unlikely(rates[i].idx > 76))
++ pr_warn("%s: rates[%d].idx == %d > 76\n",
++ __func__, i, rates[i].idx);
+
+ if (!(rates[i].flags & IEEE80211_TX_RC_USE_RTS_CTS) &&
+ info->control.use_cts_prot)
+@@ -674,7 +676,9 @@ static void rate_fixup_ratelist(struct ieee80211_vif *vif,
+ }
+
+ if (rates[i].flags & IEEE80211_TX_RC_VHT_MCS) {
+- WARN_ON(ieee80211_rate_get_vht_mcs(&rates[i]) > 9);
++ if (unlikely(ieee80211_rate_get_vht_mcs(&rates[i]) > 9))
++ pr_warn("%s: ieee80211_rate_get_vht_mcs(&rates[%d]) == %d > 9\n",
++ __func__, i, ieee80211_rate_get_vht_mcs(&rates[i]));
+ continue;
+ }
+
+@@ -685,7 +689,9 @@ static void rate_fixup_ratelist(struct ieee80211_vif *vif,
+ }
+
+ /* RC is busted */
+- if (WARN_ON_ONCE(rates[i].idx >= sband->n_bitrates)) {
++ if (unlikely(rates[i].idx >= sband->n_bitrates)) {
++ pr_warn_once("%s: rates[%d].idx == %d >= sband->n_bitrates == %d\n",
++ __func__, i, rates[i].idx, sband->n_bitrates);
+ rates[i].idx = -1;
+ continue;
+ }
+diff --git a/net/mac80211/rc80211_minstrel_ht.c b/net/mac80211/rc80211_minstrel_ht.c
+index 9c3b7fc377c1..a65d55bbb46b 100644
+--- a/net/mac80211/rc80211_minstrel_ht.c
++++ b/net/mac80211/rc80211_minstrel_ht.c
+@@ -438,8 +438,13 @@ minstrel_ht_get_tp_avg(struct minstrel_ht_sta *mi, int group, int rate,
+
+ if (minstrel_ht_is_legacy_group(group))
+ overhead = mi->overhead_legacy;
+- else
++ else {
+ ampdu_len = minstrel_ht_avg_ampdu_len(mi);
++ if (unlikely(!ampdu_len)) {
++ pr_err_once("%s: ampdu_len == 0!\n", __func__);
++ ampdu_len = 1;
++ }
++ }
+
+ nsecs = 1000 * overhead / ampdu_len;
+ nsecs += minstrel_mcs_groups[group].duration[rate] <<
+--
+2.35.0
+
+From 1baca42af85c292893389760d4ee46c0e9a29b22 Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sun, 6 Feb 2022 20:21:57 +0000
+Subject: [PATCH] spf-anon
+
+---
+ arch/arm64/Kconfig | 1 +
+ arch/arm64/mm/fault.c | 62 ++++
+ arch/powerpc/Kconfig | 1 +
+ arch/powerpc/mm/fault.c | 64 ++++
+ arch/x86/Kconfig | 1 +
+ arch/x86/boot/compressed/kaslr.c | 2 +
+ arch/x86/mm/fault.c | 63 ++++
+ drivers/gpu/drm/i915/i915_gpu_error.c | 4 +-
+ include/linux/mm.h | 68 +++-
+ include/linux/mm_types.h | 33 +-
+ include/linux/mmap_lock.h | 109 ++++--
+ include/linux/mmu_notifier.h | 52 ++-
+ include/linux/percpu-rwsem.h | 13 +-
+ include/linux/vm_event.h | 111 ++++++
+ include/linux/vm_event_item.h | 25 ++
+ include/linux/vmstat.h | 95 +-----
+ kernel/fork.c | 18 +-
+ kernel/locking/percpu-rwsem.c | 32 ++
+ mm/Kconfig | 22 ++
+ mm/Kconfig.debug | 7 +
+ mm/debug.c | 1 +
+ mm/memory.c | 470 +++++++++++++++++++-------
+ mm/mmap.c | 13 +-
+ mm/vmstat.c | 25 ++
+ scripts/package/builddeb | 4 +-
+ 25 files changed, 1042 insertions(+), 254 deletions(-)
+ create mode 100644 include/linux/vm_event.h
+
+diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
+index cbcd42decb2a..15ba7cc583d3 100644
+--- a/arch/arm64/Kconfig
++++ b/arch/arm64/Kconfig
+@@ -222,6 +222,7 @@ config ARM64
+ select THREAD_INFO_IN_TASK
+ select HAVE_ARCH_USERFAULTFD_MINOR if USERFAULTFD
+ select TRACE_IRQFLAGS_SUPPORT
++ select ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT
+ help
+ ARM 64-bit (AArch64) Linux support.
+
+diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c
+index 77341b160aca..2598795f4e70 100644
+--- a/arch/arm64/mm/fault.c
++++ b/arch/arm64/mm/fault.c
+@@ -25,6 +25,7 @@
+ #include <linux/perf_event.h>
+ #include <linux/preempt.h>
+ #include <linux/hugetlb.h>
++#include <linux/vm_event_item.h>
+
+ #include <asm/acpi.h>
+ #include <asm/bug.h>
+@@ -524,6 +525,11 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
+ unsigned long vm_flags;
+ unsigned int mm_flags = FAULT_FLAG_DEFAULT;
+ unsigned long addr = untagged_addr(far);
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ struct vm_area_struct *vma;
++ struct vm_area_struct pvma;
++ unsigned long seq;
++#endif
+
+ if (kprobe_page_fault(regs, esr))
+ return 0;
+@@ -574,6 +580,59 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
+
+ perf_sw_event(PERF_COUNT_SW_PAGE_FAULTS, 1, regs, addr);
+
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ /*
++ * No need to try speculative faults for kernel or
++ * single threaded user space.
++ */
++ if (!(mm_flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1)
++ goto no_spf;
++
++ count_vm_event(SPF_ATTEMPT);
++ seq = mmap_seq_read_start(mm);
++ if (seq & 1) {
++ count_vm_spf_event(SPF_ABORT_ODD);
++ goto spf_abort;
++ }
++ rcu_read_lock();
++ vma = __find_vma(mm, addr);
++ if (!vma || vma->vm_start > addr) {
++ rcu_read_unlock();
++ count_vm_spf_event(SPF_ABORT_UNMAPPED);
++ goto spf_abort;
++ }
++ if (!vma_is_anonymous(vma)) {
++ rcu_read_unlock();
++ count_vm_spf_event(SPF_ABORT_NO_SPECULATE);
++ goto spf_abort;
++ }
++ pvma = *vma;
++ rcu_read_unlock();
++ if (!mmap_seq_read_check(mm, seq, SPF_ABORT_VMA_COPY))
++ goto spf_abort;
++ vma = &pvma;
++ if (!(vma->vm_flags & vm_flags)) {
++ count_vm_spf_event(SPF_ABORT_ACCESS_ERROR);
++ goto spf_abort;
++ }
++ fault = do_handle_mm_fault(vma, addr & PAGE_MASK,
++ mm_flags | FAULT_FLAG_SPECULATIVE, seq, regs);
++
++ /* Quick path to respond to signals */
++ if (fault_signal_pending(fault, regs)) {
++ if (!user_mode(regs))
++ goto no_context;
++ return 0;
++ }
++ if (!(fault & VM_FAULT_RETRY))
++ goto done;
++
++spf_abort:
++ count_vm_event(SPF_ABORT);
++no_spf:
++
++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
++
+ /*
+ * As per x86, we may deadlock here. However, since the kernel only
+ * validly references user space from well defined areas of the code,
+@@ -612,6 +671,9 @@ static int __kprobes do_page_fault(unsigned long far, unsigned int esr,
+ goto retry;
+ }
+ mmap_read_unlock(mm);
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++done:
++#endif
+
+ /*
+ * Handle the "normal" (no error) case first.
+diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig
+index b779603978e1..5f82bc7eee0b 100644
+--- a/arch/powerpc/Kconfig
++++ b/arch/powerpc/Kconfig
+@@ -144,6 +144,7 @@ config PPC
+ select ARCH_STACKWALK
+ select ARCH_SUPPORTS_ATOMIC_RMW
+ select ARCH_SUPPORTS_DEBUG_PAGEALLOC if PPC_BOOK3S || PPC_8xx || 40x
++ select ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT if PPC_BOOK3S_64
+ select ARCH_USE_BUILTIN_BSWAP
+ select ARCH_USE_CMPXCHG_LOCKREF if PPC64
+ select ARCH_USE_MEMTEST
+diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c
+index eb8ecd7343a9..3f039504e8fd 100644
+--- a/arch/powerpc/mm/fault.c
++++ b/arch/powerpc/mm/fault.c
+@@ -395,6 +395,10 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address,
+ int is_write = page_fault_is_write(error_code);
+ vm_fault_t fault, major = 0;
+ bool kprobe_fault = kprobe_page_fault(regs, 11);
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ struct vm_area_struct pvma;
++ unsigned long seq;
++#endif
+
+ if (unlikely(debugger_fault_handler(regs) || kprobe_fault))
+ return 0;
+@@ -451,6 +455,63 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address,
+ if (is_exec)
+ flags |= FAULT_FLAG_INSTRUCTION;
+
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ /*
++ * No need to try speculative faults for kernel or
++ * single threaded user space.
++ */
++ if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1)
++ goto no_spf;
++
++ count_vm_event(SPF_ATTEMPT);
++ seq = mmap_seq_read_start(mm);
++ if (seq & 1) {
++ count_vm_spf_event(SPF_ABORT_ODD);
++ goto spf_abort;
++ }
++ rcu_read_lock();
++ vma = __find_vma(mm, address);
++ if (!vma || vma->vm_start > address) {
++ rcu_read_unlock();
++ count_vm_spf_event(SPF_ABORT_UNMAPPED);
++ goto spf_abort;
++ }
++ if (!vma_is_anonymous(vma)) {
++ rcu_read_unlock();
++ count_vm_spf_event(SPF_ABORT_NO_SPECULATE);
++ goto spf_abort;
++ }
++ pvma = *vma;
++ rcu_read_unlock();
++ if (!mmap_seq_read_check(mm, seq, SPF_ABORT_VMA_COPY))
++ goto spf_abort;
++ vma = &pvma;
++#ifdef CONFIG_PPC_MEM_KEYS
++ if (unlikely(access_pkey_error(is_write, is_exec,
++ (error_code & DSISR_KEYFAULT), vma))) {
++ count_vm_spf_event(SPF_ABORT_ACCESS_ERROR);
++ goto spf_abort;
++ }
++#endif /* CONFIG_PPC_MEM_KEYS */
++ if (unlikely(access_error(is_write, is_exec, vma))) {
++ count_vm_spf_event(SPF_ABORT_ACCESS_ERROR);
++ goto spf_abort;
++ }
++ fault = do_handle_mm_fault(vma, address,
++ flags | FAULT_FLAG_SPECULATIVE, seq, regs);
++ major |= fault & VM_FAULT_MAJOR;
++
++ if (fault_signal_pending(fault, regs))
++ return user_mode(regs) ? 0 : SIGBUS;
++ if (!(fault & VM_FAULT_RETRY))
++ goto done;
++
++spf_abort:
++ count_vm_event(SPF_ABORT);
++no_spf:
++
++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
++
+ /* When running in the kernel we expect faults to occur only to
+ * addresses in user space. All other faults represent errors in the
+ * kernel and should generate an OOPS. Unfortunately, in the case of an
+@@ -522,6 +583,9 @@ static int ___do_page_fault(struct pt_regs *regs, unsigned long address,
+ }
+
+ mmap_read_unlock(current->mm);
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++done:
++#endif
+
+ if (unlikely(fault & VM_FAULT_ERROR))
+ return mm_fault_error(regs, address, fault);
+diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig
+index 9f5bd41bf660..3697b840a34c 100644
+--- a/arch/x86/Kconfig
++++ b/arch/x86/Kconfig
+@@ -34,6 +34,7 @@ config X86_64
+ select SWIOTLB
+ select ARCH_HAS_ELFCORE_COMPAT
+ select ZONE_DMA32
++ select ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT
+
+ config FORCE_DYNAMIC_FTRACE
+ def_bool y
+diff --git a/arch/x86/boot/compressed/kaslr.c b/arch/x86/boot/compressed/kaslr.c
+index 411b268bc0a2..dd3c2112a1e7 100644
+--- a/arch/x86/boot/compressed/kaslr.c
++++ b/arch/x86/boot/compressed/kaslr.c
+@@ -625,7 +625,9 @@ static bool process_mem_region(struct mem_vector *region,
+ unsigned long minimum,
+ unsigned long image_size)
+ {
++#if defined(CONFIG_MEMORY_HOTREMOVE) && defined(CONFIG_ACPI)
+ int i;
++#endif
+ /*
+ * If no immovable memory found, or MEMORY_HOTREMOVE disabled,
+ * use @region directly.
+diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c
+index d0074c6ed31a..a5a19561c319 100644
+--- a/arch/x86/mm/fault.c
++++ b/arch/x86/mm/fault.c
+@@ -1226,6 +1226,10 @@ void do_user_addr_fault(struct pt_regs *regs,
+ struct mm_struct *mm;
+ vm_fault_t fault;
+ unsigned int flags = FAULT_FLAG_DEFAULT;
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ struct vm_area_struct pvma;
++ unsigned long seq;
++#endif
+
+ tsk = current;
+ mm = tsk->mm;
+@@ -1323,6 +1327,62 @@ void do_user_addr_fault(struct pt_regs *regs,
+ }
+ #endif
+
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ /*
++ * No need to try speculative faults for kernel or
++ * single threaded user space.
++ */
++ if (!(flags & FAULT_FLAG_USER) || atomic_read(&mm->mm_users) == 1)
++ goto no_spf;
++
++ count_vm_event(SPF_ATTEMPT);
++ seq = mmap_seq_read_start(mm);
++ if (seq & 1) {
++ count_vm_spf_event(SPF_ABORT_ODD);
++ goto spf_abort;
++ }
++ rcu_read_lock();
++ vma = __find_vma(mm, address);
++ if (!vma || vma->vm_start > address) {
++ rcu_read_unlock();
++ count_vm_spf_event(SPF_ABORT_UNMAPPED);
++ goto spf_abort;
++ }
++ if (!vma_is_anonymous(vma)) {
++ rcu_read_unlock();
++ count_vm_spf_event(SPF_ABORT_NO_SPECULATE);
++ goto spf_abort;
++ }
++ pvma = *vma;
++ rcu_read_unlock();
++ if (!mmap_seq_read_check(mm, seq, SPF_ABORT_VMA_COPY))
++ goto spf_abort;
++ vma = &pvma;
++ if (unlikely(access_error(error_code, vma))) {
++ count_vm_spf_event(SPF_ABORT_ACCESS_ERROR);
++ goto spf_abort;
++ }
++ fault = do_handle_mm_fault(vma, address,
++ flags | FAULT_FLAG_SPECULATIVE, seq, regs);
++
++ if (!(fault & VM_FAULT_RETRY))
++ goto done;
++
++ /* Quick path to respond to signals */
++ if (fault_signal_pending(fault, regs)) {
++ if (!user_mode(regs))
++ kernelmode_fixup_or_oops(regs, error_code, address,
++ SIGBUS, BUS_ADRERR,
++ ARCH_DEFAULT_PKEY);
++ return;
++ }
++
++spf_abort:
++ count_vm_event(SPF_ABORT);
++no_spf:
++
++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
++
+ /*
+ * Kernel-mode access to the user address space should only occur
+ * on well-defined single instructions listed in the exception
+@@ -1419,6 +1479,9 @@ void do_user_addr_fault(struct pt_regs *regs,
+ }
+
+ mmap_read_unlock(mm);
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++done:
++#endif
+ if (likely(!(fault & VM_FAULT_ERROR)))
+ return;
+
+diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
+index 0633888a411e..0e1ddc5911a5 100644
+--- a/drivers/gpu/drm/i915/i915_gpu_error.c
++++ b/drivers/gpu/drm/i915/i915_gpu_error.c
+@@ -515,7 +515,7 @@ static void error_print_context(struct drm_i915_error_state_buf *m,
+ }
+
+ static struct i915_vma_coredump *
+-__find_vma(struct i915_vma_coredump *vma, const char *name)
++__i915_find_vma(struct i915_vma_coredump *vma, const char *name)
+ {
+ while (vma) {
+ if (strcmp(vma->name, name) == 0)
+@@ -529,7 +529,7 @@ __find_vma(struct i915_vma_coredump *vma, const char *name)
+ static struct i915_vma_coredump *
+ find_batch(const struct intel_engine_coredump *ee)
+ {
+- return __find_vma(ee->vma, "batch");
++ return __i915_find_vma(ee->vma, "batch");
+ }
+
+ static void error_print_engine(struct drm_i915_error_state_buf *m,
+diff --git a/include/linux/mm.h b/include/linux/mm.h
+index 213cc569b192..ad3b5aef9431 100644
+--- a/include/linux/mm.h
++++ b/include/linux/mm.h
+@@ -461,7 +461,8 @@ static inline bool fault_flag_allow_retry_first(enum fault_flag flags)
+ { FAULT_FLAG_USER, "USER" }, \
+ { FAULT_FLAG_REMOTE, "REMOTE" }, \
+ { FAULT_FLAG_INSTRUCTION, "INSTRUCTION" }, \
+- { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }
++ { FAULT_FLAG_INTERRUPTIBLE, "INTERRUPTIBLE" }, \
++ { FAULT_FLAG_SPECULATIVE, "SPECULATIVE" }
+
+ /*
+ * vm_fault is filled by the pagefault handler and passed to the vma's
+@@ -482,6 +483,10 @@ struct vm_fault {
+ };
+ enum fault_flag flags; /* FAULT_FLAG_xxx flags
+ * XXX: should really be 'const' */
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ unsigned long seq;
++ pmd_t orig_pmd;
++#endif
+ pmd_t *pmd; /* Pointer to pmd entry matching
+ * the 'address' */
+ pud_t *pud; /* Pointer to pud entry matching
+@@ -489,9 +494,11 @@ struct vm_fault {
+ */
+ union {
+ pte_t orig_pte; /* Value of PTE at the time of fault */
++#ifndef CONFIG_SPECULATIVE_PAGE_FAULT
+ pmd_t orig_pmd; /* Value of PMD at the time of fault,
+ * used by PMD fault only.
+ */
++#endif
+ };
+
+ struct page *cow_page; /* Page handler may use for COW fault */
+@@ -1857,9 +1864,15 @@ int generic_error_remove_page(struct address_space *mapping, struct page *page);
+ int invalidate_inode_page(struct page *page);
+
+ #ifdef CONFIG_MMU
+-extern vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
+- unsigned long address, unsigned int flags,
+- struct pt_regs *regs);
++extern vm_fault_t do_handle_mm_fault(struct vm_area_struct *vma,
++ unsigned long address, unsigned int flags,
++ unsigned long seq, struct pt_regs *regs);
++static inline vm_fault_t handle_mm_fault(struct vm_area_struct *vma,
++ unsigned long address, unsigned int flags,
++ struct pt_regs *regs)
++{
++ return do_handle_mm_fault(vma, address, flags, 0, regs);
++}
+ extern int fixup_user_fault(struct mm_struct *mm,
+ unsigned long address, unsigned int fault_flags,
+ bool *unlocked);
+@@ -2751,10 +2764,17 @@ extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
+ #endif
+
+ /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
+-extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
++extern struct vm_area_struct * __find_vma(struct mm_struct * mm, unsigned long addr);
+ extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
+ struct vm_area_struct **pprev);
+
++static inline
++struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
++{
++ mmap_assert_locked(mm);
++ return __find_vma(mm, addr);
++}
++
+ /**
+ * find_vma_intersection() - Look up the first VMA which intersects the interval
+ * @mm: The process address space.
+@@ -3381,5 +3401,43 @@ madvise_set_anon_name(struct mm_struct *mm, unsigned long start,
+ }
+ #endif
+
++#ifdef CONFIG_MMU
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++
++bool __pte_map_lock(struct vm_fault *vmf);
++
++static inline bool pte_map_lock(struct vm_fault *vmf)
++{
++ VM_BUG_ON(vmf->pte);
++ return __pte_map_lock(vmf);
++}
++
++static inline bool pte_spinlock(struct vm_fault *vmf)
++{
++ VM_BUG_ON(!vmf->pte);
++ return __pte_map_lock(vmf);
++}
++
++#else /* !CONFIG_SPECULATIVE_PAGE_FAULT */
++
++#define pte_map_lock(__vmf) \
++({ \
++ struct vm_fault *vmf = __vmf; \
++ vmf->pte = pte_offset_map_lock(vmf->vma->vm_mm, vmf->pmd, \
++ vmf->address, &vmf->ptl); \
++ true; \
++})
++
++#define pte_spinlock(__vmf) \
++({ \
++ struct vm_fault *vmf = __vmf; \
++ vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd); \
++ spin_lock(vmf->ptl); \
++ true; \
++})
++
++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
++#endif /* CONFIG_MMU */
++
+ #endif /* __KERNEL__ */
+ #endif /* _LINUX_MM_H */
+diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
+index 5140e5feb486..f4ba3d11cbbe 100644
+--- a/include/linux/mm_types.h
++++ b/include/linux/mm_types.h
+@@ -371,16 +371,27 @@ struct anon_vma_name {
+ * per VM-area/task. A VM area is any part of the process virtual memory
+ * space that has a special rule for the page-fault handlers (ie a shared
+ * library, the executable area etc).
++ *
++ * Note that speculative page faults make an on-stack copy of the VMA,
++ * so the structure size matters.
++ * (TODO - it would be preferable to copy only the required vma attributes
++ * rather than the entire vma).
+ */
+ struct vm_area_struct {
+ /* The first cache line has the info for VMA tree walking. */
+
+- unsigned long vm_start; /* Our start address within vm_mm. */
+- unsigned long vm_end; /* The first byte after our end address
+- within vm_mm. */
++ union {
++ struct {
++ /* VMA covers [vm_start; vm_end) addresses within mm */
++ unsigned long vm_start, vm_end;
+
+- /* linked list of VM areas per task, sorted by address */
+- struct vm_area_struct *vm_next, *vm_prev;
++ /* linked list of VMAs per task, sorted by address */
++ struct vm_area_struct *vm_next, *vm_prev;
++ };
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ struct rcu_head vm_rcu; /* Used for deferred freeing. */
++#endif
++ };
+
+ struct rb_node vm_rb;
+
+@@ -452,6 +463,7 @@ struct vm_area_struct {
+ } __randomize_layout;
+
+ struct kioctx_table;
++struct percpu_rw_semaphore;
+ struct mm_struct {
+ struct {
+ struct vm_area_struct *mmap; /* list of VMAs */
+@@ -524,6 +536,10 @@ struct mm_struct {
+ * cacheline.
+ */
+ struct rw_semaphore mmap_lock;
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ unsigned long mmap_seq;
++#endif
++
+
+ struct list_head mmlist; /* List of maybe swapped mm's. These
+ * are globally strung together off
+@@ -594,7 +610,10 @@ struct mm_struct {
+ struct file __rcu *exe_file;
+ #ifdef CONFIG_MMU_NOTIFIER
+ struct mmu_notifier_subscriptions *notifier_subscriptions;
+-#endif
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ struct percpu_rw_semaphore *mmu_notifier_lock;
++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
++#endif /* CONFIG_MMU_NOTIFIER */
+ #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
+ pgtable_t pmd_huge_pte; /* protected by page_table_lock */
+ #endif
+@@ -791,6 +810,7 @@ typedef struct {
+ * @FAULT_FLAG_REMOTE: The fault is not for current task/mm.
+ * @FAULT_FLAG_INSTRUCTION: The fault was during an instruction fetch.
+ * @FAULT_FLAG_INTERRUPTIBLE: The fault can be interrupted by non-fatal signals.
++ * @FAULT_FLAG_SPECULATIVE: The fault is handled without holding the mmap lock.
+ *
+ * About @FAULT_FLAG_ALLOW_RETRY and @FAULT_FLAG_TRIED: we can specify
+ * whether we would allow page faults to retry by specifying these two
+@@ -822,6 +842,7 @@ enum fault_flag {
+ FAULT_FLAG_REMOTE = 1 << 7,
+ FAULT_FLAG_INSTRUCTION = 1 << 8,
+ FAULT_FLAG_INTERRUPTIBLE = 1 << 9,
++ FAULT_FLAG_SPECULATIVE = 1 << 10,
+ };
+
+ #endif /* _LINUX_MM_TYPES_H */
+diff --git a/include/linux/mmap_lock.h b/include/linux/mmap_lock.h
+index 96e113e23d04..747805ce07b8 100644
+--- a/include/linux/mmap_lock.h
++++ b/include/linux/mmap_lock.h
+@@ -7,9 +7,18 @@
+ #include <linux/rwsem.h>
+ #include <linux/tracepoint-defs.h>
+ #include <linux/types.h>
++#include <linux/vm_event.h>
+
+-#define MMAP_LOCK_INITIALIZER(name) \
+- .mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock),
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++#define MMAP_LOCK_SEQ_INITIALIZER(name) \
++ .mmap_seq = 0,
++#else
++#define MMAP_LOCK_SEQ_INITIALIZER(name)
++#endif
++
++#define MMAP_LOCK_INITIALIZER(name) \
++ .mmap_lock = __RWSEM_INITIALIZER((name).mmap_lock), \
++ MMAP_LOCK_SEQ_INITIALIZER(name)
+
+ DECLARE_TRACEPOINT(mmap_lock_start_locking);
+ DECLARE_TRACEPOINT(mmap_lock_acquire_returned);
+@@ -63,13 +72,66 @@ static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)
+ static inline void mmap_init_lock(struct mm_struct *mm)
+ {
+ init_rwsem(&mm->mmap_lock);
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ mm->mmap_seq = 0;
++#endif
++}
++
++static inline void __mmap_seq_write_lock(struct mm_struct *mm)
++{
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ VM_BUG_ON_MM(mm->mmap_seq & 1, mm);
++ mm->mmap_seq++;
++ smp_wmb();
++#endif
++}
++
++static inline void __mmap_seq_write_unlock(struct mm_struct *mm)
++{
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ smp_wmb();
++ mm->mmap_seq++;
++ VM_BUG_ON_MM(mm->mmap_seq & 1, mm);
++#endif
++}
++
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++static inline unsigned long mmap_seq_read_start(struct mm_struct *mm)
++{
++ unsigned long seq;
++
++ seq = READ_ONCE(mm->mmap_seq);
++ smp_rmb();
++ return seq;
++}
++
++static inline bool __mmap_seq_read_check(struct mm_struct *mm,
++ unsigned long seq)
++{
++ smp_rmb();
++ return seq == READ_ONCE(mm->mmap_seq);
++}
++
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT_STATS
++static inline bool mmap_seq_read_check(struct mm_struct *mm, unsigned long seq,
++ enum vm_event_item fail_event)
++{
++ if (__mmap_seq_read_check(mm, seq))
++ return true;
++ count_vm_event(fail_event);
++ return false;
+ }
++#else
++#define mmap_seq_read_check(mm, seq, fail) __mmap_seq_read_check(mm, seq)
++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT_STATS */
++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
+
+ static inline void mmap_write_lock(struct mm_struct *mm)
+ {
+ __mmap_lock_trace_start_locking(mm, true);
+ down_write(&mm->mmap_lock);
+ __mmap_lock_trace_acquire_returned(mm, true, true);
++ __mmap_seq_write_lock(mm);
+ }
+
+ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)
+@@ -77,37 +139,44 @@ static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)
+ __mmap_lock_trace_start_locking(mm, true);
+ down_write_nested(&mm->mmap_lock, subclass);
+ __mmap_lock_trace_acquire_returned(mm, true, true);
++ __mmap_seq_write_lock(mm);
+ }
+
+ static inline int mmap_write_lock_killable(struct mm_struct *mm)
+ {
+- int ret;
++ int error;
+
+ __mmap_lock_trace_start_locking(mm, true);
+- ret = down_write_killable(&mm->mmap_lock);
+- __mmap_lock_trace_acquire_returned(mm, true, ret == 0);
+- return ret;
++ error = down_write_killable(&mm->mmap_lock);
++ __mmap_lock_trace_acquire_returned(mm, true, !error);
++ if (likely(!error))
++ __mmap_seq_write_lock(mm);
++ return error;
+ }
+
+ static inline bool mmap_write_trylock(struct mm_struct *mm)
+ {
+- bool ret;
++ bool ok;
+
+ __mmap_lock_trace_start_locking(mm, true);
+- ret = down_write_trylock(&mm->mmap_lock) != 0;
+- __mmap_lock_trace_acquire_returned(mm, true, ret);
+- return ret;
++ ok = down_write_trylock(&mm->mmap_lock) != 0;
++ __mmap_lock_trace_acquire_returned(mm, true, ok);
++ if (likely(ok))
++ __mmap_seq_write_lock(mm);
++ return ok;
+ }
+
+ static inline void mmap_write_unlock(struct mm_struct *mm)
+ {
+ __mmap_lock_trace_released(mm, true);
++ __mmap_seq_write_unlock(mm);
+ up_write(&mm->mmap_lock);
+ }
+
+ static inline void mmap_write_downgrade(struct mm_struct *mm)
+ {
+ __mmap_lock_trace_acquire_returned(mm, false, true);
++ __mmap_seq_write_unlock(mm);
+ downgrade_write(&mm->mmap_lock);
+ }
+
+@@ -120,22 +189,22 @@ static inline void mmap_read_lock(struct mm_struct *mm)
+
+ static inline int mmap_read_lock_killable(struct mm_struct *mm)
+ {
+- int ret;
++ int error;
+
+ __mmap_lock_trace_start_locking(mm, false);
+- ret = down_read_killable(&mm->mmap_lock);
+- __mmap_lock_trace_acquire_returned(mm, false, ret == 0);
+- return ret;
++ error = down_read_killable(&mm->mmap_lock);
++ __mmap_lock_trace_acquire_returned(mm, false, !error);
++ return error;
+ }
+
+ static inline bool mmap_read_trylock(struct mm_struct *mm)
+ {
+- bool ret;
++ bool ok;
+
+ __mmap_lock_trace_start_locking(mm, false);
+- ret = down_read_trylock(&mm->mmap_lock) != 0;
+- __mmap_lock_trace_acquire_returned(mm, false, ret);
+- return ret;
++ ok = down_read_trylock(&mm->mmap_lock) != 0;
++ __mmap_lock_trace_acquire_returned(mm, false, ok);
++ return ok;
+ }
+
+ static inline void mmap_read_unlock(struct mm_struct *mm)
+@@ -162,9 +231,9 @@ static inline void mmap_assert_write_locked(struct mm_struct *mm)
+ VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm);
+ }
+
+-static inline int mmap_lock_is_contended(struct mm_struct *mm)
++static inline bool mmap_lock_is_contended(struct mm_struct *mm)
+ {
+- return rwsem_is_contended(&mm->mmap_lock);
++ return rwsem_is_contended(&mm->mmap_lock) != 0;
+ }
+
+ #endif /* _LINUX_MMAP_LOCK_H */
+diff --git a/include/linux/mmu_notifier.h b/include/linux/mmu_notifier.h
+index 45fc2c81e370..d0430410fdd8 100644
+--- a/include/linux/mmu_notifier.h
++++ b/include/linux/mmu_notifier.h
+@@ -6,6 +6,8 @@
+ #include <linux/spinlock.h>
+ #include <linux/mm_types.h>
+ #include <linux/mmap_lock.h>
++#include <linux/percpu-rwsem.h>
++#include <linux/slab.h>
+ #include <linux/srcu.h>
+ #include <linux/interval_tree.h>
+
+@@ -499,15 +501,35 @@ static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
+ __mmu_notifier_invalidate_range(mm, start, end);
+ }
+
+-static inline void mmu_notifier_subscriptions_init(struct mm_struct *mm)
++static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)
+ {
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ mm->mmu_notifier_lock = kzalloc(sizeof(struct percpu_rw_semaphore), GFP_KERNEL);
++ if (!mm->mmu_notifier_lock)
++ return false;
++ if (percpu_init_rwsem(mm->mmu_notifier_lock)) {
++ kfree(mm->mmu_notifier_lock);
++ return false;
++ }
++#endif
++
+ mm->notifier_subscriptions = NULL;
++ return true;
+ }
+
+ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
+ {
+ if (mm_has_notifiers(mm))
+ __mmu_notifier_subscriptions_destroy(mm);
++
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ if (!in_atomic()) {
++ percpu_free_rwsem(mm->mmu_notifier_lock);
++ kfree(mm->mmu_notifier_lock);
++ } else {
++ percpu_rwsem_async_destroy(mm->mmu_notifier_lock);
++ }
++#endif
+ }
+
+
+@@ -724,8 +746,9 @@ static inline void mmu_notifier_invalidate_range(struct mm_struct *mm,
+ {
+ }
+
+-static inline void mmu_notifier_subscriptions_init(struct mm_struct *mm)
++static inline bool mmu_notifier_subscriptions_init(struct mm_struct *mm)
+ {
++ return true;
+ }
+
+ static inline void mmu_notifier_subscriptions_destroy(struct mm_struct *mm)
+@@ -749,4 +772,29 @@ static inline void mmu_notifier_synchronize(void)
+
+ #endif /* CONFIG_MMU_NOTIFIER */
+
++#if defined(CONFIG_MMU_NOTIFIER) && defined(CONFIG_SPECULATIVE_PAGE_FAULT)
++
++static inline bool mmu_notifier_trylock(struct mm_struct *mm)
++{
++ return percpu_down_read_trylock(mm->mmu_notifier_lock);
++}
++
++static inline void mmu_notifier_unlock(struct mm_struct *mm)
++{
++ percpu_up_read(mm->mmu_notifier_lock);
++}
++
++#else
++
++static inline bool mmu_notifier_trylock(struct mm_struct *mm)
++{
++ return true;
++}
++
++static inline void mmu_notifier_unlock(struct mm_struct *mm)
++{
++}
++
++#endif
++
+ #endif /* _LINUX_MMU_NOTIFIER_H */
+diff --git a/include/linux/percpu-rwsem.h b/include/linux/percpu-rwsem.h
+index 5fda40f97fe9..bf1668fc9c5e 100644
+--- a/include/linux/percpu-rwsem.h
++++ b/include/linux/percpu-rwsem.h
+@@ -13,7 +13,14 @@ struct percpu_rw_semaphore {
+ struct rcu_sync rss;
+ unsigned int __percpu *read_count;
+ struct rcuwait writer;
+- wait_queue_head_t waiters;
++ /*
++ * destroy_list_entry is used during object destruction when waiters
++ * can't be used, therefore reusing the same space.
++ */
++ union {
++ wait_queue_head_t waiters;
++ struct list_head destroy_list_entry;
++ };
+ atomic_t block;
+ #ifdef CONFIG_DEBUG_LOCK_ALLOC
+ struct lockdep_map dep_map;
+@@ -127,8 +134,12 @@ extern void percpu_up_write(struct percpu_rw_semaphore *);
+ extern int __percpu_init_rwsem(struct percpu_rw_semaphore *,
+ const char *, struct lock_class_key *);
+
++/* Can't be called in atomic context. */
+ extern void percpu_free_rwsem(struct percpu_rw_semaphore *);
+
++/* Invokes percpu_free_rwsem and frees the semaphore from a worker thread. */
++extern void percpu_rwsem_async_destroy(struct percpu_rw_semaphore *sem);
++
+ #define percpu_init_rwsem(sem) \
+ ({ \
+ static struct lock_class_key rwsem_key; \
+diff --git a/include/linux/vm_event.h b/include/linux/vm_event.h
+new file mode 100644
+index 000000000000..689a21387dad
+--- /dev/null
++++ b/include/linux/vm_event.h
+@@ -0,0 +1,111 @@
++/* SPDX-License-Identifier: GPL-2.0 */
++#ifndef _LINUX_VM_EVENT_H
++#define _LINUX_VM_EVENT_H
++
++#include <linux/types.h>
++#include <linux/percpu.h>
++#include <linux/mmzone.h>
++#include <linux/vm_event_item.h>
++#include <linux/atomic.h>
++
++#ifdef CONFIG_VM_EVENT_COUNTERS
++/*
++ * Light weight per cpu counter implementation.
++ *
++ * Counters should only be incremented and no critical kernel component
++ * should rely on the counter values.
++ *
++ * Counters are handled completely inline. On many platforms the code
++ * generated will simply be the increment of a global address.
++ */
++
++struct vm_event_state {
++ unsigned long event[NR_VM_EVENT_ITEMS];
++};
++
++DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
++
++/*
++ * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
++ * local_irq_disable overhead.
++ */
++static inline void __count_vm_event(enum vm_event_item item)
++{
++ raw_cpu_inc(vm_event_states.event[item]);
++}
++
++static inline void count_vm_event(enum vm_event_item item)
++{
++ this_cpu_inc(vm_event_states.event[item]);
++}
++
++static inline void __count_vm_events(enum vm_event_item item, long delta)
++{
++ raw_cpu_add(vm_event_states.event[item], delta);
++}
++
++static inline void count_vm_events(enum vm_event_item item, long delta)
++{
++ this_cpu_add(vm_event_states.event[item], delta);
++}
++
++extern void all_vm_events(unsigned long *);
++
++extern void vm_events_fold_cpu(int cpu);
++
++#else
++
++/* Disable counters */
++static inline void count_vm_event(enum vm_event_item item)
++{
++}
++static inline void count_vm_events(enum vm_event_item item, long delta)
++{
++}
++static inline void __count_vm_event(enum vm_event_item item)
++{
++}
++static inline void __count_vm_events(enum vm_event_item item, long delta)
++{
++}
++static inline void all_vm_events(unsigned long *ret)
++{
++}
++static inline void vm_events_fold_cpu(int cpu)
++{
++}
++
++#endif /* CONFIG_VM_EVENT_COUNTERS */
++
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT_STATS
++#define count_vm_spf_event(x) count_vm_event(x)
++#else
++#define count_vm_spf_event(x) do {} while (0)
++#endif
++
++#ifdef CONFIG_NUMA_BALANCING
++#define count_vm_numa_event(x) count_vm_event(x)
++#define count_vm_numa_events(x, y) count_vm_events(x, y)
++#else
++#define count_vm_numa_event(x) do {} while (0)
++#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
++#endif /* CONFIG_NUMA_BALANCING */
++
++#ifdef CONFIG_DEBUG_TLBFLUSH
++#define count_vm_tlb_event(x) count_vm_event(x)
++#define count_vm_tlb_events(x, y) count_vm_events(x, y)
++#else
++#define count_vm_tlb_event(x) do {} while (0)
++#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
++#endif
++
++#ifdef CONFIG_DEBUG_VM_VMACACHE
++#define count_vm_vmacache_event(x) count_vm_event(x)
++#else
++#define count_vm_vmacache_event(x) do {} while (0)
++#endif
++
++#define __count_zid_vm_events(item, zid, delta) \
++ __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
++
++#endif /* _LINUX_VM_EVENT_H */
+diff --git a/include/linux/vm_event_item.h b/include/linux/vm_event_item.h
+index 7b2363388bfa..0390b81b1e71 100644
+--- a/include/linux/vm_event_item.h
++++ b/include/linux/vm_event_item.h
+@@ -133,6 +133,31 @@ enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT,
+ #ifdef CONFIG_X86
+ DIRECT_MAP_LEVEL2_SPLIT,
+ DIRECT_MAP_LEVEL3_SPLIT,
++#endif
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ SPF_ATTEMPT,
++ SPF_ABORT,
++#endif
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT_STATS
++ SPF_ABORT_ODD,
++ SPF_ABORT_UNMAPPED,
++ SPF_ABORT_NO_SPECULATE,
++ SPF_ABORT_VMA_COPY,
++ SPF_ABORT_ACCESS_ERROR,
++ SPF_ABORT_PUD,
++ SPF_ABORT_PMD,
++ SPF_ABORT_ANON_VMA,
++ SPF_ABORT_PTE_MAP_LOCK_SEQ1,
++ SPF_ABORT_PTE_MAP_LOCK_PMD,
++ SPF_ABORT_PTE_MAP_LOCK_PTL,
++ SPF_ABORT_PTE_MAP_LOCK_SEQ2,
++ SPF_ABORT_USERFAULTFD,
++ SPF_ABORT_FAULT,
++ SPF_ABORT_SWAP,
++ SPF_ATTEMPT_ANON,
++ SPF_ATTEMPT_NUMA,
++ SPF_ATTEMPT_PTE,
++ SPF_ATTEMPT_WP,
+ #endif
+ NR_VM_EVENT_ITEMS
+ };
+diff --git a/include/linux/vmstat.h b/include/linux/vmstat.h
+index bfe38869498d..7c3c892ce89a 100644
+--- a/include/linux/vmstat.h
++++ b/include/linux/vmstat.h
+@@ -6,6 +6,7 @@
+ #include <linux/percpu.h>
+ #include <linux/mmzone.h>
+ #include <linux/vm_event_item.h>
++#include <linux/vm_event.h>
+ #include <linux/atomic.h>
+ #include <linux/static_key.h>
+ #include <linux/mmdebug.h>
+@@ -40,100 +41,6 @@ enum writeback_stat_item {
+ NR_VM_WRITEBACK_STAT_ITEMS,
+ };
+
+-#ifdef CONFIG_VM_EVENT_COUNTERS
+-/*
+- * Light weight per cpu counter implementation.
+- *
+- * Counters should only be incremented and no critical kernel component
+- * should rely on the counter values.
+- *
+- * Counters are handled completely inline. On many platforms the code
+- * generated will simply be the increment of a global address.
+- */
+-
+-struct vm_event_state {
+- unsigned long event[NR_VM_EVENT_ITEMS];
+-};
+-
+-DECLARE_PER_CPU(struct vm_event_state, vm_event_states);
+-
+-/*
+- * vm counters are allowed to be racy. Use raw_cpu_ops to avoid the
+- * local_irq_disable overhead.
+- */
+-static inline void __count_vm_event(enum vm_event_item item)
+-{
+- raw_cpu_inc(vm_event_states.event[item]);
+-}
+-
+-static inline void count_vm_event(enum vm_event_item item)
+-{
+- this_cpu_inc(vm_event_states.event[item]);
+-}
+-
+-static inline void __count_vm_events(enum vm_event_item item, long delta)
+-{
+- raw_cpu_add(vm_event_states.event[item], delta);
+-}
+-
+-static inline void count_vm_events(enum vm_event_item item, long delta)
+-{
+- this_cpu_add(vm_event_states.event[item], delta);
+-}
+-
+-extern void all_vm_events(unsigned long *);
+-
+-extern void vm_events_fold_cpu(int cpu);
+-
+-#else
+-
+-/* Disable counters */
+-static inline void count_vm_event(enum vm_event_item item)
+-{
+-}
+-static inline void count_vm_events(enum vm_event_item item, long delta)
+-{
+-}
+-static inline void __count_vm_event(enum vm_event_item item)
+-{
+-}
+-static inline void __count_vm_events(enum vm_event_item item, long delta)
+-{
+-}
+-static inline void all_vm_events(unsigned long *ret)
+-{
+-}
+-static inline void vm_events_fold_cpu(int cpu)
+-{
+-}
+-
+-#endif /* CONFIG_VM_EVENT_COUNTERS */
+-
+-#ifdef CONFIG_NUMA_BALANCING
+-#define count_vm_numa_event(x) count_vm_event(x)
+-#define count_vm_numa_events(x, y) count_vm_events(x, y)
+-#else
+-#define count_vm_numa_event(x) do {} while (0)
+-#define count_vm_numa_events(x, y) do { (void)(y); } while (0)
+-#endif /* CONFIG_NUMA_BALANCING */
+-
+-#ifdef CONFIG_DEBUG_TLBFLUSH
+-#define count_vm_tlb_event(x) count_vm_event(x)
+-#define count_vm_tlb_events(x, y) count_vm_events(x, y)
+-#else
+-#define count_vm_tlb_event(x) do {} while (0)
+-#define count_vm_tlb_events(x, y) do { (void)(y); } while (0)
+-#endif
+-
+-#ifdef CONFIG_DEBUG_VM_VMACACHE
+-#define count_vm_vmacache_event(x) count_vm_event(x)
+-#else
+-#define count_vm_vmacache_event(x) do {} while (0)
+-#endif
+-
+-#define __count_zid_vm_events(item, zid, delta) \
+- __count_vm_events(item##_NORMAL - ZONE_NORMAL + zid, delta)
+-
+ /*
+ * Zone and node-based page accounting with per cpu differentials.
+ */
+diff --git a/kernel/fork.c b/kernel/fork.c
+index d75a528f7b21..34600fe86743 100644
+--- a/kernel/fork.c
++++ b/kernel/fork.c
+@@ -371,9 +371,24 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
+ return new;
+ }
+
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++static void __vm_area_free(struct rcu_head *head)
++{
++ struct vm_area_struct *vma = container_of(head, struct vm_area_struct,
++ vm_rcu);
++ kmem_cache_free(vm_area_cachep, vma);
++}
++#endif
++
+ void vm_area_free(struct vm_area_struct *vma)
+ {
+ free_vma_anon_name(vma);
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ if (atomic_read(&vma->vm_mm->mm_users) > 1) {
++ call_rcu(&vma->vm_rcu, __vm_area_free);
++ return;
++ }
++#endif
+ kmem_cache_free(vm_area_cachep, vma);
+ }
+
+@@ -1056,7 +1071,8 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p,
+ mm_init_owner(mm, p);
+ mm_init_pasid(mm);
+ RCU_INIT_POINTER(mm->exe_file, NULL);
+- mmu_notifier_subscriptions_init(mm);
++ if (!mmu_notifier_subscriptions_init(mm))
++ goto fail_nopgd;
+ init_tlb_flush_pending(mm);
+ #if defined(CONFIG_TRANSPARENT_HUGEPAGE) && !USE_SPLIT_PMD_PTLOCKS
+ mm->pmd_huge_pte = NULL;
+diff --git a/kernel/locking/percpu-rwsem.c b/kernel/locking/percpu-rwsem.c
+index 70a32a576f3f..a3d37bf83c60 100644
+--- a/kernel/locking/percpu-rwsem.c
++++ b/kernel/locking/percpu-rwsem.c
+@@ -7,6 +7,7 @@
+ #include <linux/rcupdate.h>
+ #include <linux/sched.h>
+ #include <linux/sched/task.h>
++#include <linux/slab.h>
+ #include <linux/errno.h>
+
+ int __percpu_init_rwsem(struct percpu_rw_semaphore *sem,
+@@ -268,3 +269,34 @@ void percpu_up_write(struct percpu_rw_semaphore *sem)
+ rcu_sync_exit(&sem->rss);
+ }
+ EXPORT_SYMBOL_GPL(percpu_up_write);
++
++static LIST_HEAD(destroy_list);
++static DEFINE_SPINLOCK(destroy_list_lock);
++
++static void destroy_list_workfn(struct work_struct *work)
++{
++ struct percpu_rw_semaphore *sem, *sem2;
++ LIST_HEAD(to_destroy);
++
++ spin_lock(&destroy_list_lock);
++ list_splice_init(&destroy_list, &to_destroy);
++ spin_unlock(&destroy_list_lock);
++
++ if (list_empty(&to_destroy))
++ return;
++
++ list_for_each_entry_safe(sem, sem2, &to_destroy, destroy_list_entry) {
++ percpu_free_rwsem(sem);
++ kfree(sem);
++ }
++}
++
++static DECLARE_WORK(destroy_list_work, destroy_list_workfn);
++
++void percpu_rwsem_async_destroy(struct percpu_rw_semaphore *sem)
++{
++ spin_lock(&destroy_list_lock);
++ list_add_tail(&sem->destroy_list_entry, &destroy_list);
++ spin_unlock(&destroy_list_lock);
++ schedule_work(&destroy_list_work);
++}
+diff --git a/mm/Kconfig b/mm/Kconfig
+index 3326ee3903f3..d304fca0f293 100644
+--- a/mm/Kconfig
++++ b/mm/Kconfig
+@@ -894,4 +894,26 @@ config ANON_VMA_NAME
+
+ source "mm/damon/Kconfig"
+
++config ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT
++ def_bool n
++
++config SPECULATIVE_PAGE_FAULT
++ bool "Speculative page faults"
++ default y
++ depends on ARCH_SUPPORTS_SPECULATIVE_PAGE_FAULT && MMU && SMP
++ help
++ Try to handle user space page faults without holding the mmap lock.
++
++ Instead of blocking writers through the use of mmap lock,
++ the page fault handler merely verifies, at the end of the page
++ fault, that no writers have been running concurrently with it.
++
++ In high concurrency situations, the speculative fault handler
++ gains a throughput advantage by avoiding having to update the
++ mmap lock reader count.
++
++ If the check fails due to a concurrent writer, or due to hitting
++ an unsupported case, the fault handler falls back to classical
++ processing using the mmap read lock.
++
+ endmenu
+diff --git a/mm/Kconfig.debug b/mm/Kconfig.debug
+index 5bd5bb097252..73b61cc95562 100644
+--- a/mm/Kconfig.debug
++++ b/mm/Kconfig.debug
+@@ -174,3 +174,10 @@ config PTDUMP_DEBUGFS
+ kernel.
+
+ If in doubt, say N.
++
++config SPECULATIVE_PAGE_FAULT_STATS
++ bool "Additional statistics for speculative page faults"
++ depends on SPECULATIVE_PAGE_FAULT
++ help
++ Additional statistics for speculative page faults.
++ If in doubt, say N.
+diff --git a/mm/debug.c b/mm/debug.c
+index bc9ac87f0e08..40d3f358b75c 100644
+--- a/mm/debug.c
++++ b/mm/debug.c
+@@ -220,6 +220,7 @@ void dump_mm(const struct mm_struct *mm)
+ mm->def_flags, &mm->def_flags
+ );
+ }
++EXPORT_SYMBOL(dump_mm);
+
+ static bool page_init_poisoning __read_mostly = true;
+
+diff --git a/mm/memory.c b/mm/memory.c
+index c125c4969913..a5754309eaae 100644
+--- a/mm/memory.c
++++ b/mm/memory.c
+@@ -2725,6 +2725,100 @@ int apply_to_existing_page_range(struct mm_struct *mm, unsigned long addr,
+ }
+ EXPORT_SYMBOL_GPL(apply_to_existing_page_range);
+
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++
++/*
++ * speculative_page_walk_begin() ... speculative_page_walk_end() protects
++ * against races with page table reclamation.
++ *
++ * This is similar to what fast GUP does, but fast GUP also needs to
++ * protect against races with THP page splitting, so it always needs
++ * to disable interrupts.
++ * Speculative page faults only need to protect against page table reclamation,
++ * so rcu_read_lock() is sufficient in the MMU_GATHER_RCU_TABLE_FREE case.
++ */
++#ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
++#define speculative_page_walk_begin() rcu_read_lock()
++#define speculative_page_walk_end() rcu_read_unlock()
++#else
++#define speculative_page_walk_begin() local_irq_disable()
++#define speculative_page_walk_end() local_irq_enable()
++#endif
++
++bool __pte_map_lock(struct vm_fault *vmf)
++{
++#ifdef CONFIG_TRANSPARENT_HUGEPAGE
++ pmd_t pmdval;
++#endif
++ pte_t *pte = vmf->pte;
++ spinlock_t *ptl;
++
++ if (!(vmf->flags & FAULT_FLAG_SPECULATIVE)) {
++ vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
++ if (!pte)
++ vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
++ spin_lock(vmf->ptl);
++ return true;
++ }
++
++ speculative_page_walk_begin();
++ if (!mmap_seq_read_check(vmf->vma->vm_mm, vmf->seq,
++ SPF_ABORT_PTE_MAP_LOCK_SEQ1))
++ goto fail;
++ /*
++ * The mmap sequence count check guarantees that the page
++ * tables are still valid at that point, and
++ * speculative_page_walk_begin() ensures that they stay around.
++ */
++#ifdef CONFIG_TRANSPARENT_HUGEPAGE
++ /*
++ * We check if the pmd value is still the same to ensure that there
++ * is not a huge collapse operation in progress in our back.
++ */
++ pmdval = READ_ONCE(*vmf->pmd);
++ if (!pmd_same(pmdval, vmf->orig_pmd)) {
++ count_vm_spf_event(SPF_ABORT_PTE_MAP_LOCK_PMD);
++ goto fail;
++ }
++#endif
++ ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
++ if (!pte)
++ pte = pte_offset_map(vmf->pmd, vmf->address);
++ /*
++ * Try locking the page table.
++ *
++ * Note that we might race against zap_pte_range() which
++ * invalidates TLBs while holding the page table lock.
++ * We are still under the speculative_page_walk_begin() section,
++ * and zap_pte_range() could thus deadlock with us if we tried
++ * using spin_lock() here.
++ *
++ * We also don't want to retry until spin_trylock() succeeds,
++ * because of the starvation potential against a stream of lockers.
++ */
++ if (unlikely(!spin_trylock(ptl))) {
++ count_vm_spf_event(SPF_ABORT_PTE_MAP_LOCK_PTL);
++ goto fail;
++ }
++ if (!mmap_seq_read_check(vmf->vma->vm_mm, vmf->seq,
++ SPF_ABORT_PTE_MAP_LOCK_SEQ2))
++ goto unlock_fail;
++ speculative_page_walk_end();
++ vmf->pte = pte;
++ vmf->ptl = ptl;
++ return true;
++
++unlock_fail:
++ spin_unlock(ptl);
++fail:
++ if (pte)
++ pte_unmap(pte);
++ speculative_page_walk_end();
++ return false;
++}
++
++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
++
+ /*
+ * handle_pte_fault chooses page fault handler according to an entry which was
+ * read non-atomically. Before making any commitment, on those architectures
+@@ -2999,20 +3093,28 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
+ pte_t entry;
+ int page_copied = 0;
+ struct mmu_notifier_range range;
++ vm_fault_t ret = VM_FAULT_OOM;
+
+- if (unlikely(anon_vma_prepare(vma)))
+- goto oom;
++ if (unlikely(!vma->anon_vma)) {
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
++ count_vm_spf_event(SPF_ABORT_ANON_VMA);
++ ret = VM_FAULT_RETRY;
++ goto out;
++ }
++ if (__anon_vma_prepare(vma))
++ goto out;
++ }
+
+ if (is_zero_pfn(pte_pfn(vmf->orig_pte))) {
+ new_page = alloc_zeroed_user_highpage_movable(vma,
+ vmf->address);
+ if (!new_page)
+- goto oom;
++ goto out;
+ } else {
+ new_page = alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma,
+ vmf->address);
+ if (!new_page)
+- goto oom;
++ goto out;
+
+ if (!cow_user_page(new_page, old_page, vmf)) {
+ /*
+@@ -3029,11 +3131,16 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
+ }
+
+ if (mem_cgroup_charge(page_folio(new_page), mm, GFP_KERNEL))
+- goto oom_free_new;
++ goto out_free_new;
+ cgroup_throttle_swaprate(new_page, GFP_KERNEL);
+
+ __SetPageUptodate(new_page);
+
++ if ((vmf->flags & FAULT_FLAG_SPECULATIVE) &&
++ !mmu_notifier_trylock(mm)) {
++ ret = VM_FAULT_RETRY;
++ goto out_free_new;
++ }
+ mmu_notifier_range_init(&range, MMU_NOTIFY_CLEAR, 0, vma, mm,
+ vmf->address & PAGE_MASK,
+ (vmf->address & PAGE_MASK) + PAGE_SIZE);
+@@ -3042,7 +3149,11 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
+ /*
+ * Re-check the pte - we dropped the lock
+ */
+- vmf->pte = pte_offset_map_lock(mm, vmf->pmd, vmf->address, &vmf->ptl);
++ if (!pte_map_lock(vmf)) {
++ ret = VM_FAULT_RETRY;
++ /* put_page() will uncharge the page */
++ goto out_notify;
++ }
+ if (likely(pte_same(*vmf->pte, vmf->orig_pte))) {
+ if (old_page) {
+ if (!PageAnon(old_page)) {
+@@ -3117,6 +3228,8 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
+ * the above ptep_clear_flush_notify() did already call it.
+ */
+ mmu_notifier_invalidate_range_only_end(&range);
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)
++ mmu_notifier_unlock(mm);
+ if (old_page) {
+ /*
+ * Don't let another task, with possibly unlocked vma,
+@@ -3133,12 +3246,16 @@ static vm_fault_t wp_page_copy(struct vm_fault *vmf)
+ put_page(old_page);
+ }
+ return page_copied ? VM_FAULT_WRITE : 0;
+-oom_free_new:
++out_notify:
++ mmu_notifier_invalidate_range_only_end(&range);
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)
++ mmu_notifier_unlock(mm);
++out_free_new:
+ put_page(new_page);
+-oom:
++out:
+ if (old_page)
+ put_page(old_page);
+- return VM_FAULT_OOM;
++ return ret;
+ }
+
+ /**
+@@ -3183,6 +3300,7 @@ static vm_fault_t wp_pfn_shared(struct vm_fault *vmf)
+ {
+ struct vm_area_struct *vma = vmf->vma;
+
++ VM_BUG_ON(vmf->flags & FAULT_FLAG_SPECULATIVE);
+ if (vma->vm_ops && vma->vm_ops->pfn_mkwrite) {
+ vm_fault_t ret;
+
+@@ -3203,6 +3321,8 @@ static vm_fault_t wp_page_shared(struct vm_fault *vmf)
+ struct vm_area_struct *vma = vmf->vma;
+ vm_fault_t ret = VM_FAULT_WRITE;
+
++ VM_BUG_ON(vmf->flags & FAULT_FLAG_SPECULATIVE);
++
+ get_page(vmf->page);
+
+ if (vma->vm_ops && vma->vm_ops->page_mkwrite) {
+@@ -3254,8 +3374,15 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)
+ {
+ struct vm_area_struct *vma = vmf->vma;
+
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)
++ count_vm_spf_event(SPF_ATTEMPT_WP);
++
+ if (userfaultfd_pte_wp(vma, *vmf->pte)) {
+ pte_unmap_unlock(vmf->pte, vmf->ptl);
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
++ count_vm_spf_event(SPF_ABORT_USERFAULTFD);
++ return VM_FAULT_RETRY;
++ }
+ return handle_userfault(vmf, VM_UFFD_WP);
+ }
+
+@@ -3281,6 +3408,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)
+ return wp_pfn_shared(vmf);
+
+ pte_unmap_unlock(vmf->pte, vmf->ptl);
++ vmf->pte = NULL;
+ return wp_page_copy(vmf);
+ }
+
+@@ -3319,6 +3447,7 @@ static vm_fault_t do_wp_page(struct vm_fault *vmf)
+ get_page(vmf->page);
+
+ pte_unmap_unlock(vmf->pte, vmf->ptl);
++ vmf->pte = NULL;
+ return wp_page_copy(vmf);
+ }
+
+@@ -3501,6 +3630,12 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
+ vm_fault_t ret = 0;
+ void *shadow = NULL;
+
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
++ pte_unmap(vmf->pte);
++ count_vm_spf_event(SPF_ABORT_SWAP);
++ return VM_FAULT_RETRY;
++ }
++
+ if (!pte_unmap_same(vmf))
+ goto out;
+
+@@ -3726,10 +3861,13 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
+ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
+ {
+ struct vm_area_struct *vma = vmf->vma;
+- struct page *page;
++ struct page *page = NULL;
+ vm_fault_t ret = 0;
+ pte_t entry;
+
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)
++ count_vm_spf_event(SPF_ATTEMPT_ANON);
++
+ /* File mapping without ->vm_ops ? */
+ if (vma->vm_flags & VM_SHARED)
+ return VM_FAULT_SIGBUS;
+@@ -3747,7 +3885,7 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
+ if (pte_alloc(vma->vm_mm, vmf->pmd))
+ return VM_FAULT_OOM;
+
+- /* See comment in handle_pte_fault() */
++ /* See comment in __handle_mm_fault() */
+ if (unlikely(pmd_trans_unstable(vmf->pmd)))
+ return 0;
+
+@@ -3756,78 +3894,80 @@ static vm_fault_t do_anonymous_page(struct vm_fault *vmf)
+ !mm_forbids_zeropage(vma->vm_mm)) {
+ entry = pte_mkspecial(pfn_pte(my_zero_pfn(vmf->address),
+ vma->vm_page_prot));
+- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd,
+- vmf->address, &vmf->ptl);
+- if (!pte_none(*vmf->pte)) {
+- update_mmu_tlb(vma, vmf->address, vmf->pte);
+- goto unlock;
+- }
+- ret = check_stable_address_space(vma->vm_mm);
+- if (ret)
+- goto unlock;
+- /* Deliver the page fault to userland, check inside PT lock */
+- if (userfaultfd_missing(vma)) {
+- pte_unmap_unlock(vmf->pte, vmf->ptl);
+- return handle_userfault(vmf, VM_UFFD_MISSING);
++ } else {
++ /* Allocate our own private page. */
++ if (unlikely(!vma->anon_vma)) {
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
++ count_vm_spf_event(SPF_ABORT_ANON_VMA);
++ return VM_FAULT_RETRY;
++ }
++ if (__anon_vma_prepare(vma))
++ goto oom;
+ }
+- goto setpte;
+- }
+-
+- /* Allocate our own private page. */
+- if (unlikely(anon_vma_prepare(vma)))
+- goto oom;
+- page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
+- if (!page)
+- goto oom;
++ page = alloc_zeroed_user_highpage_movable(vma, vmf->address);
++ if (!page)
++ goto oom;
+
+- if (mem_cgroup_charge(page_folio(page), vma->vm_mm, GFP_KERNEL))
+- goto oom_free_page;
+- cgroup_throttle_swaprate(page, GFP_KERNEL);
++ if (mem_cgroup_charge(page_folio(page), vma->vm_mm, GFP_KERNEL))
++ goto oom_free_page;
++ cgroup_throttle_swaprate(page, GFP_KERNEL);
+
+- /*
+- * The memory barrier inside __SetPageUptodate makes sure that
+- * preceding stores to the page contents become visible before
+- * the set_pte_at() write.
+- */
+- __SetPageUptodate(page);
++ /*
++ * The memory barrier inside __SetPageUptodate makes sure that
++ * preceding stores to the page contents become visible before
++ * the set_pte_at() write.
++ */
++ __SetPageUptodate(page);
+
+- entry = mk_pte(page, vma->vm_page_prot);
+- entry = pte_sw_mkyoung(entry);
+- if (vma->vm_flags & VM_WRITE)
+- entry = pte_mkwrite(pte_mkdirty(entry));
++ entry = mk_pte(page, vma->vm_page_prot);
++ entry = pte_sw_mkyoung(entry);
++ if (vma->vm_flags & VM_WRITE)
++ entry = pte_mkwrite(pte_mkdirty(entry));
++ }
+
+- vmf->pte = pte_offset_map_lock(vma->vm_mm, vmf->pmd, vmf->address,
+- &vmf->ptl);
+- if (!pte_none(*vmf->pte)) {
+- update_mmu_cache(vma, vmf->address, vmf->pte);
++ if (!pte_map_lock(vmf)) {
++ ret = VM_FAULT_RETRY;
+ goto release;
+ }
++ if (!pte_none(*vmf->pte)) {
++ update_mmu_tlb(vma, vmf->address, vmf->pte);
++ goto unlock;
++ }
+
+ ret = check_stable_address_space(vma->vm_mm);
+ if (ret)
+- goto release;
++ goto unlock;
+
+ /* Deliver the page fault to userland, check inside PT lock */
+ if (userfaultfd_missing(vma)) {
+ pte_unmap_unlock(vmf->pte, vmf->ptl);
+- put_page(page);
++ if (page)
++ put_page(page);
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE) {
++ count_vm_spf_event(SPF_ABORT_USERFAULTFD);
++ return VM_FAULT_RETRY;
++ }
+ return handle_userfault(vmf, VM_UFFD_MISSING);
+ }
+
+- inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
+- page_add_new_anon_rmap(page, vma, vmf->address, false);
+- lru_cache_add_inactive_or_unevictable(page, vma);
+-setpte:
++ if (page) {
++ inc_mm_counter_fast(vma->vm_mm, MM_ANONPAGES);
++ page_add_new_anon_rmap(page, vma, vmf->address, false);
++ lru_cache_add_inactive_or_unevictable(page, vma);
++ }
++
+ set_pte_at(vma->vm_mm, vmf->address, vmf->pte, entry);
+
+ /* No need to invalidate - it was non-present before */
+ update_mmu_cache(vma, vmf->address, vmf->pte);
++ pte_unmap_unlock(vmf->pte, vmf->ptl);
++ return 0;
+ unlock:
+ pte_unmap_unlock(vmf->pte, vmf->ptl);
+- return ret;
+ release:
+- put_page(page);
+- goto unlock;
++ if (page)
++ put_page(page);
++ return ret;
+ oom_free_page:
+ put_page(page);
+ oom:
+@@ -4051,7 +4191,7 @@ vm_fault_t finish_fault(struct vm_fault *vmf)
+ return VM_FAULT_OOM;
+ }
+
+- /* See comment in handle_pte_fault() */
++ /* See comment in __handle_mm_fault() */
+ if (pmd_devmap_trans_unstable(vmf->pmd))
+ return 0;
+
+@@ -4279,6 +4419,8 @@ static vm_fault_t do_fault(struct vm_fault *vmf)
+ struct mm_struct *vm_mm = vma->vm_mm;
+ vm_fault_t ret;
+
++ VM_BUG_ON(vmf->flags & FAULT_FLAG_SPECULATIVE);
++
+ /*
+ * The VMA was not fully populated on mmap() or missing VM_DONTEXPAND
+ */
+@@ -4348,13 +4490,16 @@ static vm_fault_t do_numa_page(struct vm_fault *vmf)
+ bool was_writable = pte_savedwrite(vmf->orig_pte);
+ int flags = 0;
+
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)
++ count_vm_spf_event(SPF_ATTEMPT_NUMA);
++
+ /*
+ * The "pte" at this point cannot be used safely without
+ * validation through pte_unmap_same(). It's of NUMA type but
+ * the pfn may be screwed if the read is non atomic.
+ */
+- vmf->ptl = pte_lockptr(vma->vm_mm, vmf->pmd);
+- spin_lock(vmf->ptl);
++ if (!pte_spinlock(vmf))
++ return VM_FAULT_RETRY;
+ if (unlikely(!pte_same(*vmf->pte, vmf->orig_pte))) {
+ pte_unmap_unlock(vmf->pte, vmf->ptl);
+ goto out;
+@@ -4516,53 +4661,6 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
+ {
+ pte_t entry;
+
+- if (unlikely(pmd_none(*vmf->pmd))) {
+- /*
+- * Leave __pte_alloc() until later: because vm_ops->fault may
+- * want to allocate huge page, and if we expose page table
+- * for an instant, it will be difficult to retract from
+- * concurrent faults and from rmap lookups.
+- */
+- vmf->pte = NULL;
+- } else {
+- /*
+- * If a huge pmd materialized under us just retry later. Use
+- * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead
+- * of pmd_trans_huge() to ensure the pmd didn't become
+- * pmd_trans_huge under us and then back to pmd_none, as a
+- * result of MADV_DONTNEED running immediately after a huge pmd
+- * fault in a different thread of this mm, in turn leading to a
+- * misleading pmd_trans_huge() retval. All we have to ensure is
+- * that it is a regular pmd that we can walk with
+- * pte_offset_map() and we can do that through an atomic read
+- * in C, which is what pmd_trans_unstable() provides.
+- */
+- if (pmd_devmap_trans_unstable(vmf->pmd))
+- return 0;
+- /*
+- * A regular pmd is established and it can't morph into a huge
+- * pmd from under us anymore at this point because we hold the
+- * mmap_lock read mode and khugepaged takes it in write mode.
+- * So now it's safe to run pte_offset_map().
+- */
+- vmf->pte = pte_offset_map(vmf->pmd, vmf->address);
+- vmf->orig_pte = *vmf->pte;
+-
+- /*
+- * some architectures can have larger ptes than wordsize,
+- * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
+- * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic
+- * accesses. The code below just needs a consistent view
+- * for the ifs and we later double check anyway with the
+- * ptl lock held. So here a barrier will do.
+- */
+- barrier();
+- if (pte_none(vmf->orig_pte)) {
+- pte_unmap(vmf->pte);
+- vmf->pte = NULL;
+- }
+- }
+-
+ if (!vmf->pte) {
+ if (vma_is_anonymous(vmf->vma))
+ return do_anonymous_page(vmf);
+@@ -4576,8 +4674,11 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
+ if (pte_protnone(vmf->orig_pte) && vma_is_accessible(vmf->vma))
+ return do_numa_page(vmf);
+
+- vmf->ptl = pte_lockptr(vmf->vma->vm_mm, vmf->pmd);
+- spin_lock(vmf->ptl);
++ if (vmf->flags & FAULT_FLAG_SPECULATIVE)
++ count_vm_spf_event(SPF_ATTEMPT_PTE);
++
++ if (!pte_spinlock(vmf))
++ return VM_FAULT_RETRY;
+ entry = vmf->orig_pte;
+ if (unlikely(!pte_same(*vmf->pte, entry))) {
+ update_mmu_tlb(vmf->vma, vmf->address, vmf->pte);
+@@ -4617,7 +4718,7 @@ static vm_fault_t handle_pte_fault(struct vm_fault *vmf)
+ * return value. See filemap_fault() and __folio_lock_or_retry().
+ */
+ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
+- unsigned long address, unsigned int flags)
++ unsigned long address, unsigned int flags, unsigned long seq)
+ {
+ struct vm_fault vmf = {
+ .vma = vma,
+@@ -4632,6 +4733,87 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
+ p4d_t *p4d;
+ vm_fault_t ret;
+
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ if (flags & FAULT_FLAG_SPECULATIVE) {
++ pgd_t pgdval;
++ p4d_t p4dval;
++ pud_t pudval;
++
++ vmf.seq = seq;
++
++ speculative_page_walk_begin();
++ pgd = pgd_offset(mm, address);
++ pgdval = READ_ONCE(*pgd);
++ if (pgd_none(pgdval) || unlikely(pgd_bad(pgdval))) {
++ count_vm_spf_event(SPF_ABORT_PUD);
++ goto spf_fail;
++ }
++
++ p4d = p4d_offset(pgd, address);
++ p4dval = READ_ONCE(*p4d);
++ if (p4d_none(p4dval) || unlikely(p4d_bad(p4dval))) {
++ count_vm_spf_event(SPF_ABORT_PUD);
++ goto spf_fail;
++ }
++
++ vmf.pud = pud_offset(p4d, address);
++ pudval = READ_ONCE(*vmf.pud);
++ if (pud_none(pudval) || unlikely(pud_bad(pudval)) ||
++ unlikely(pud_trans_huge(pudval)) ||
++ unlikely(pud_devmap(pudval))) {
++ count_vm_spf_event(SPF_ABORT_PUD);
++ goto spf_fail;
++ }
++
++ vmf.pmd = pmd_offset(vmf.pud, address);
++ vmf.orig_pmd = READ_ONCE(*vmf.pmd);
++
++ /*
++ * pmd_none could mean that a hugepage collapse is in
++ * progress in our back as collapse_huge_page() mark
++ * it before invalidating the pte (which is done once
++ * the IPI is catched by all CPU and we have interrupt
++ * disabled). For this reason we cannot handle THP in
++ * a speculative way since we can't safely identify an
++ * in progress collapse operation done in our back on
++ * that PMD.
++ */
++ if (unlikely(pmd_none(vmf.orig_pmd) ||
++ is_swap_pmd(vmf.orig_pmd) ||
++ pmd_trans_huge(vmf.orig_pmd) ||
++ pmd_devmap(vmf.orig_pmd))) {
++ count_vm_spf_event(SPF_ABORT_PMD);
++ goto spf_fail;
++ }
++
++ /*
++ * The above does not allocate/instantiate page-tables because
++ * doing so would lead to the possibility of instantiating
++ * page-tables after free_pgtables() -- and consequently
++ * leaking them.
++ *
++ * The result is that we take at least one non-speculative
++ * fault per PMD in order to instantiate it.
++ */
++
++ vmf.pte = pte_offset_map(vmf.pmd, address);
++ vmf.orig_pte = READ_ONCE(*vmf.pte);
++ barrier();
++ if (pte_none(vmf.orig_pte)) {
++ pte_unmap(vmf.pte);
++ vmf.pte = NULL;
++ }
++
++ speculative_page_walk_end();
++
++ return handle_pte_fault(&vmf);
++
++ spf_fail:
++ speculative_page_walk_end();
++ return VM_FAULT_RETRY;
++ }
++#endif /* CONFIG_SPECULATIVE_PAGE_FAULT */
++
+ pgd = pgd_offset(mm, address);
+ p4d = p4d_alloc(mm, pgd, address);
+ if (!p4d)
+@@ -4702,6 +4884,53 @@ static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma,
+ }
+ }
+
++ if (unlikely(pmd_none(*vmf.pmd))) {
++ /*
++ * Leave __pte_alloc() until later: because vm_ops->fault may
++ * want to allocate huge page, and if we expose page table
++ * for an instant, it will be difficult to retract from
++ * concurrent faults and from rmap lookups.
++ */
++ vmf.pte = NULL;
++ } else {
++ /*
++ * If a huge pmd materialized under us just retry later. Use
++ * pmd_trans_unstable() via pmd_devmap_trans_unstable() instead
++ * of pmd_trans_huge() to ensure the pmd didn't become
++ * pmd_trans_huge under us and then back to pmd_none, as a
++ * result of MADV_DONTNEED running immediately after a huge pmd
++ * fault in a different thread of this mm, in turn leading to a
++ * misleading pmd_trans_huge() retval. All we have to ensure is
++ * that it is a regular pmd that we can walk with
++ * pte_offset_map() and we can do that through an atomic read
++ * in C, which is what pmd_trans_unstable() provides.
++ */
++ if (pmd_devmap_trans_unstable(vmf.pmd))
++ return 0;
++ /*
++ * A regular pmd is established and it can't morph into a huge
++ * pmd from under us anymore at this point because we hold the
++ * mmap_lock read mode and khugepaged takes it in write mode.
++ * So now it's safe to run pte_offset_map().
++ */
++ vmf.pte = pte_offset_map(vmf.pmd, vmf.address);
++ vmf.orig_pte = *vmf.pte;
++
++ /*
++ * some architectures can have larger ptes than wordsize,
++ * e.g.ppc44x-defconfig has CONFIG_PTE_64BIT=y and
++ * CONFIG_32BIT=y, so READ_ONCE cannot guarantee atomic
++ * accesses. The code below just needs a consistent view
++ * for the ifs and we later double check anyway with the
++ * ptl lock held. So here a barrier will do.
++ */
++ barrier();
++ if (pte_none(vmf.orig_pte)) {
++ pte_unmap(vmf.pte);
++ vmf.pte = NULL;
++ }
++ }
++
+ return handle_pte_fault(&vmf);
+ }
+
+@@ -4772,11 +5001,14 @@ static inline void mm_account_fault(struct pt_regs *regs,
+ * The mmap_lock may have been released depending on flags and our
+ * return value. See filemap_fault() and __folio_lock_or_retry().
+ */
+-vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
+- unsigned int flags, struct pt_regs *regs)
++vm_fault_t do_handle_mm_fault(struct vm_area_struct *vma,
++ unsigned long address, unsigned int flags,
++ unsigned long seq, struct pt_regs *regs)
+ {
+ vm_fault_t ret;
+
++ VM_BUG_ON((flags & FAULT_FLAG_SPECULATIVE) && !vma_is_anonymous(vma));
++
+ __set_current_state(TASK_RUNNING);
+
+ count_vm_event(PGFAULT);
+@@ -4797,10 +5029,12 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
+ if (flags & FAULT_FLAG_USER)
+ mem_cgroup_enter_user_fault();
+
+- if (unlikely(is_vm_hugetlb_page(vma)))
++ if (unlikely(is_vm_hugetlb_page(vma))) {
++ VM_BUG_ON(flags & FAULT_FLAG_SPECULATIVE);
+ ret = hugetlb_fault(vma->vm_mm, vma, address, flags);
+- else
+- ret = __handle_mm_fault(vma, address, flags);
++ } else {
++ ret = __handle_mm_fault(vma, address, flags, seq);
++ }
+
+ if (flags & FAULT_FLAG_USER) {
+ mem_cgroup_exit_user_fault();
+@@ -4818,7 +5052,7 @@ vm_fault_t handle_mm_fault(struct vm_area_struct *vma, unsigned long address,
+
+ return ret;
+ }
+-EXPORT_SYMBOL_GPL(handle_mm_fault);
++EXPORT_SYMBOL_GPL(do_handle_mm_fault);
+
+ #ifndef __PAGETABLE_P4D_FOLDED
+ /*
+diff --git a/mm/mmap.c b/mm/mmap.c
+index 1e8fdb0b51ed..a67c3600d995 100644
+--- a/mm/mmap.c
++++ b/mm/mmap.c
+@@ -2276,12 +2276,11 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
+ EXPORT_SYMBOL(get_unmapped_area);
+
+ /* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
+-struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
++struct vm_area_struct *__find_vma(struct mm_struct *mm, unsigned long addr)
+ {
+ struct rb_node *rb_node;
+ struct vm_area_struct *vma;
+
+- mmap_assert_locked(mm);
+ /* Check the cache first. */
+ vma = vmacache_find(mm, addr);
+ if (likely(vma))
+@@ -2308,7 +2307,7 @@ struct vm_area_struct *find_vma(struct mm_struct *mm, unsigned long addr)
+ return vma;
+ }
+
+-EXPORT_SYMBOL(find_vma);
++EXPORT_SYMBOL(__find_vma);
+
+ /*
+ * Same as find_vma, but also return a pointer to the previous VMA in *pprev.
+@@ -3593,6 +3592,10 @@ int mm_take_all_locks(struct mm_struct *mm)
+
+ mutex_lock(&mm_all_locks_mutex);
+
++#if defined(CONFIG_MMU_NOTIFIER) && defined(CONFIG_SPECULATIVE_PAGE_FAULT)
++ percpu_down_write(mm->mmu_notifier_lock);
++#endif
++
+ for (vma = mm->mmap; vma; vma = vma->vm_next) {
+ if (signal_pending(current))
+ goto out_unlock;
+@@ -3680,6 +3683,10 @@ void mm_drop_all_locks(struct mm_struct *mm)
+ vm_unlock_mapping(vma->vm_file->f_mapping);
+ }
+
++#if defined(CONFIG_MMU_NOTIFIER) && defined(CONFIG_SPECULATIVE_PAGE_FAULT)
++ percpu_up_write(mm->mmu_notifier_lock);
++#endif
++
+ mutex_unlock(&mm_all_locks_mutex);
+ }
+
+diff --git a/mm/vmstat.c b/mm/vmstat.c
+index 4057372745d0..20ac17cf582a 100644
+--- a/mm/vmstat.c
++++ b/mm/vmstat.c
+@@ -1390,6 +1390,31 @@ const char * const vmstat_text[] = {
+ "direct_map_level2_splits",
+ "direct_map_level3_splits",
+ #endif
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT
++ "spf_attempt",
++ "spf_abort",
++#endif
++#ifdef CONFIG_SPECULATIVE_PAGE_FAULT_STATS
++ "SPF_ABORT_ODD",
++ "SPF_ABORT_UNMAPPED",
++ "SPF_ABORT_NO_SPECULATE",
++ "SPF_ABORT_VMA_COPY",
++ "SPF_ABORT_ACCESS_ERROR",
++ "SPF_ABORT_PUD",
++ "SPF_ABORT_PMD",
++ "SPF_ABORT_ANON_VMA",
++ "SPF_ABORT_PTE_MAP_LOCK_SEQ1",
++ "SPF_ABORT_PTE_MAP_LOCK_PMD",
++ "SPF_ABORT_PTE_MAP_LOCK_PTL",
++ "SPF_ABORT_PTE_MAP_LOCK_SEQ2",
++ "SPF_ABORT_USERFAULTFD",
++ "SPF_ABORT_FAULT",
++ "SPF_ABORT_SWAP",
++ "SPF_ATTEMPT_ANON",
++ "SPF_ATTEMPT_NUMA",
++ "SPF_ATTEMPT_PTE",
++ "SPF_ATTEMPT_WP",
++#endif
+ #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */
+ };
+ #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */
+diff --git a/scripts/package/builddeb b/scripts/package/builddeb
+index 91a502bb97e8..b83a4a09a75d 100755
+--- a/scripts/package/builddeb
++++ b/scripts/package/builddeb
+@@ -50,7 +50,7 @@ create_package() {
+
+ # Create the package
+ dpkg-gencontrol -p$pname -P"$pdir"
+- dpkg-deb $dpkg_deb_opts ${KDEB_COMPRESS:+-Z$KDEB_COMPRESS} --build "$pdir" ..
++ dpkg-deb $dpkg_deb_opts ${KDEB_COMPRESS:+-Z$KDEB_COMPRESS} --build "$pdir" "${UPLOAD_DIR:-..}"
+ }
+
+ deploy_kernel_headers () {
+@@ -208,7 +208,7 @@ EOF
+ chmod 755 "$tmpdir/DEBIAN/$script"
+ done
+
+-if [ "$ARCH" != "um" ]; then
++if [ "$ARCH" != "um" -a -z "$NO_HEADER_PACKAGES" ]; then
+ if is_enabled CONFIG_MODULES; then
+ deploy_kernel_headers debian/linux-headers
+ create_package linux-headers-$version debian/linux-headers
+--
+2.35.1
+
+From 7e908601be64f8331e3b3f18c25f98c991d03c6e Mon Sep 17 00:00:00 2001
+From: "P. Jung" <[email protected]>
+Date: Sun, 6 Feb 2022 20:40:19 +0000
+Subject: [PATCH] zstd-patches
+
+Signed-off-by: P. Jung <[email protected]>
+---
+ arch/arm/boot/deflate_xip_data.sh | 2 +-
+ arch/powerpc/boot/wrapper | 2 +-
+ arch/x86/boot/compressed/Makefile | 5 +
+ include/linux/zstd_lib.h | 4 +-
+ init/Kconfig | 44 ++++++
+ lib/zstd/compress/zstd_double_fast.c | 61 ++++++--
+ lib/zstd/compress/zstd_fast.c | 69 +++++++--
+ lib/zstd/compress/zstd_lazy.c | 223 +++++++++++----------------
+ scripts/Makefile.lib | 24 +--
+ scripts/Makefile.modinst | 7 +-
+ scripts/file-size.pl | 8 +
+ scripts/file-size.sh | 4 -
+ scripts/link-vmlinux.sh | 4 +-
+ 13 files changed, 274 insertions(+), 183 deletions(-)
+ create mode 100755 scripts/file-size.pl
+ delete mode 100755 scripts/file-size.sh
+
+diff --git a/arch/arm/boot/deflate_xip_data.sh b/arch/arm/boot/deflate_xip_data.sh
+index 304495c3c2c5..14cfa2babb93 100755
+--- a/arch/arm/boot/deflate_xip_data.sh
++++ b/arch/arm/boot/deflate_xip_data.sh
+@@ -43,7 +43,7 @@ data_start=$(($__data_loc - $base_offset))
+ data_end=$(($_edata_loc - $base_offset))
+
+ # Make sure data occupies the last part of the file.
+-file_end=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" "$XIPIMAGE")
++file_end=$(${PERL} "${srctree}/scripts/file-size.pl" "$XIPIMAGE")
+ if [ "$file_end" != "$data_end" ]; then
+ printf "end of xipImage doesn't match with _edata_loc (%#x vs %#x)\n" \
+ $(($file_end + $base_offset)) $_edata_loc 1>&2
+diff --git a/arch/powerpc/boot/wrapper b/arch/powerpc/boot/wrapper
+index 9184eda780fd..9f9ee8613432 100755
+--- a/arch/powerpc/boot/wrapper
++++ b/arch/powerpc/boot/wrapper
+@@ -380,7 +380,7 @@ vmz="$tmpdir/`basename \"$kernel\"`.$ext"
+
+ # Calculate the vmlinux.strip size
+ ${CROSS}objcopy $objflags "$kernel" "$vmz.$$"
+-strip_size=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" "$vmz.$$")
++strip_size=$(${PERL} "${srctree}/scripts/file-size.pl" "$vmz.$$")
+
+ if [ -z "$cacheit" -o ! -f "$vmz$compression" -o "$vmz$compression" -ot "$kernel" ]; then
+ # recompress the image if we need to
+diff --git a/arch/x86/boot/compressed/Makefile b/arch/x86/boot/compressed/Makefile
+index 6115274fe10f..5f04502554e2 100644
+--- a/arch/x86/boot/compressed/Makefile
++++ b/arch/x86/boot/compressed/Makefile
+@@ -135,8 +135,13 @@ $(obj)/vmlinux.bin.lzo: $(vmlinux.bin.all-y) FORCE
+ $(call if_changed,lzo_with_size)
+ $(obj)/vmlinux.bin.lz4: $(vmlinux.bin.all-y) FORCE
+ $(call if_changed,lz4_with_size)
++ifdef CONFIG_KERNEL_ZSTD_ULTRA
+ $(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE
+ $(call if_changed,zstd22_with_size)
++else
++$(obj)/vmlinux.bin.zst: $(vmlinux.bin.all-y) FORCE
++ $(call if_changed,zstd)
++endif
+
+ suffix-$(CONFIG_KERNEL_GZIP) := gz
+ suffix-$(CONFIG_KERNEL_BZIP2) := bz2
+diff --git a/include/linux/zstd_lib.h b/include/linux/zstd_lib.h
+index b8c7dbf98390..6b91758b61af 100644
+--- a/include/linux/zstd_lib.h
++++ b/include/linux/zstd_lib.h
+@@ -1330,7 +1330,7 @@ ZSTDLIB_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs,
+
+ /*! ZSTD_mergeBlockDelimiters() :
+ * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals
+- * by merging them into into the literals of the next sequence.
++ * by merging them into the literals of the next sequence.
+ *
+ * As such, the final generated result has no explicit representation of block boundaries,
+ * and the final last literals segment is not represented in the sequences.
+@@ -1377,7 +1377,7 @@ ZSTDLIB_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size
+ /*! ZSTD_writeSkippableFrame() :
+ * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer.
+ *
+- * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number,
++ * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number,
+ * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15.
+ * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so
+ * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant.
+diff --git a/init/Kconfig b/init/Kconfig
+index a1a5a68aea07..c315dfa7b107 100644
+--- a/init/Kconfig
++++ b/init/Kconfig
+@@ -329,6 +329,25 @@ config KERNEL_UNCOMPRESSED
+
+ endchoice
+
++menu "ZSTD compression options"
++ depends on KERNEL_ZSTD
++
++config KERNEL_ZSTD_LEVEL
++ int "Compression level (1-19)"
++ range 1 19
++ default 19
++ help
++ Choose a compression level for zstd kernel compression.
++ Default is 19.
++
++config KERNEL_ZSTD_LEVEL_ULTRA
++ bool "Ultra compression"
++ help
++ Selecting this overrides the chosen compression level and enables
++ the highest possible compression level for zstd (ultra-22).
++
++endmenu
++
+ config DEFAULT_INIT
+ string "Default init path"
+ default ""
+@@ -2306,6 +2325,31 @@ config MODULE_DECOMPRESS
+
+ If unsure, say N.
+
++menu "ZSTD module compression options"
++ depends on MODULE_COMPRESS_ZSTD
++
++config MODULE_COMPRESS_ZSTD_LEVEL
++ int "Compression level (1-19)"
++ range 1 19
++ default 19
++ help
++ Compression level used by zstd for compressing modules.
++
++config MODULE_COMPRESS_ZSTD_ULTRA
++ bool "Enable ZSTD ultra compression"
++ help
++ Compress modules with ZSTD using the highest possible compression.
++
++config MODULE_COMPRESS_ZSTD_LEVEL_ULTRA
++ int "Compression level (20-22)"
++ depends on MODULE_COMPRESS_ZSTD_ULTRA
++ range 20 22
++ default 22
++ help
++ Ultra compression level used by zstd for compressing modules.
++
++endmenu
++
+ config MODULE_ALLOW_MISSING_NAMESPACE_IMPORTS
+ bool "Allow loading of modules with missing namespace imports"
+ help
+diff --git a/lib/zstd/compress/zstd_double_fast.c b/lib/zstd/compress/zstd_double_fast.c
+index b0424d23ac57..fb941a5b70f5 100644
+--- a/lib/zstd/compress/zstd_double_fast.c
++++ b/lib/zstd/compress/zstd_double_fast.c
+@@ -313,6 +313,26 @@ size_t ZSTD_compressBlock_doubleFast_generic(
+ return (size_t)(iend - anchor);
+ }
+
++#define ZSTD_GEN_FN(dictMode, mls) \
++ static size_t ZSTD_compressBlock_doubleFast_##dictMode##_##mls( \
++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
++ void const* src, size_t srcSize) \
++ { \
++ return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_##dictMode); \
++ }
++
++ZSTD_GEN_FN(noDict, 4)
++ZSTD_GEN_FN(noDict, 5)
++ZSTD_GEN_FN(noDict, 6)
++ZSTD_GEN_FN(noDict, 7)
++
++ZSTD_GEN_FN(dictMatchState, 4)
++ZSTD_GEN_FN(dictMatchState, 5)
++ZSTD_GEN_FN(dictMatchState, 6)
++ZSTD_GEN_FN(dictMatchState, 7)
++
++#undef ZSTD_GEN_FN
++
+
+ size_t ZSTD_compressBlock_doubleFast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+@@ -323,13 +343,13 @@ size_t ZSTD_compressBlock_doubleFast(
+ {
+ default: /* includes case 3 */
+ case 4 :
+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_noDict);
++ return ZSTD_compressBlock_doubleFast_noDict_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_noDict);
++ return ZSTD_compressBlock_doubleFast_noDict_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_noDict);
++ return ZSTD_compressBlock_doubleFast_noDict_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_noDict);
++ return ZSTD_compressBlock_doubleFast_noDict_7(ms, seqStore, rep, src, srcSize);
+ }
+ }
+
+@@ -343,13 +363,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState(
+ {
+ default: /* includes case 3 */
+ case 4 :
+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 4, ZSTD_dictMatchState);
++ return ZSTD_compressBlock_doubleFast_dictMatchState_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 5, ZSTD_dictMatchState);
++ return ZSTD_compressBlock_doubleFast_dictMatchState_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 6, ZSTD_dictMatchState);
++ return ZSTD_compressBlock_doubleFast_dictMatchState_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, 7, ZSTD_dictMatchState);
++ return ZSTD_compressBlock_doubleFast_dictMatchState_7(ms, seqStore, rep, src, srcSize);
+ }
+ }
+
+@@ -385,7 +405,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
+
+ /* if extDict is invalidated due to maxDistance, switch to "regular" variant */
+ if (prefixStartIndex == dictStartIndex)
+- return ZSTD_compressBlock_doubleFast_generic(ms, seqStore, rep, src, srcSize, mls, ZSTD_noDict);
++ return ZSTD_compressBlock_doubleFast(ms, seqStore, rep, src, srcSize);
+
+ /* Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
+@@ -499,6 +519,21 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic(
+ }
+
+
++#define ZSTD_GEN_FN(mls) \
++ static size_t ZSTD_compressBlock_doubleFast_extDict_##mls( \
++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
++ void const* src, size_t srcSize) \
++ { \
++ return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \
++ }
++
++ZSTD_GEN_FN(4)
++ZSTD_GEN_FN(5)
++ZSTD_GEN_FN(6)
++ZSTD_GEN_FN(7)
++
++#undef ZSTD_GEN_FN
++
+ size_t ZSTD_compressBlock_doubleFast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+@@ -508,12 +543,12 @@ size_t ZSTD_compressBlock_doubleFast_extDict(
+ {
+ default: /* includes case 3 */
+ case 4 :
+- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);
++ return ZSTD_compressBlock_doubleFast_extDict_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);
++ return ZSTD_compressBlock_doubleFast_extDict_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);
++ return ZSTD_compressBlock_doubleFast_extDict_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+- return ZSTD_compressBlock_doubleFast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);
++ return ZSTD_compressBlock_doubleFast_extDict_7(ms, seqStore, rep, src, srcSize);
+ }
+ }
+diff --git a/lib/zstd/compress/zstd_fast.c b/lib/zstd/compress/zstd_fast.c
+index 96b7d48e2868..e0652e31d421 100644
+--- a/lib/zstd/compress/zstd_fast.c
++++ b/lib/zstd/compress/zstd_fast.c
+@@ -182,6 +182,20 @@ ZSTD_compressBlock_fast_generic(
+ return (size_t)(iend - anchor);
+ }
+
++#define ZSTD_GEN_FN(mls) \
++ static size_t ZSTD_compressBlock_fast_##mls( \
++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
++ void const* src, size_t srcSize) \
++ { \
++ return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls); \
++ }
++
++ZSTD_GEN_FN(4)
++ZSTD_GEN_FN(5)
++ZSTD_GEN_FN(6)
++ZSTD_GEN_FN(7)
++
++#undef ZSTD_GEN_FN
+
+ size_t ZSTD_compressBlock_fast(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+@@ -193,13 +207,13 @@ size_t ZSTD_compressBlock_fast(
+ {
+ default: /* includes case 3 */
+ case 4 :
+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 4);
++ return ZSTD_compressBlock_fast_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 5);
++ return ZSTD_compressBlock_fast_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 6);
++ return ZSTD_compressBlock_fast_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, 7);
++ return ZSTD_compressBlock_fast_7(ms, seqStore, rep, src, srcSize);
+ }
+ }
+
+@@ -351,6 +365,21 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic(
+ return (size_t)(iend - anchor);
+ }
+
++#define ZSTD_GEN_FN(mls) \
++ static size_t ZSTD_compressBlock_fast_dictMatchState_##mls( \
++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
++ void const* src, size_t srcSize) \
++ { \
++ return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, mls); \
++ }
++
++ZSTD_GEN_FN(4)
++ZSTD_GEN_FN(5)
++ZSTD_GEN_FN(6)
++ZSTD_GEN_FN(7)
++
++#undef ZSTD_GEN_FN
++
+ size_t ZSTD_compressBlock_fast_dictMatchState(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+ void const* src, size_t srcSize)
+@@ -361,13 +390,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState(
+ {
+ default: /* includes case 3 */
+ case 4 :
+- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 4);
++ return ZSTD_compressBlock_fast_dictMatchState_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 5);
++ return ZSTD_compressBlock_fast_dictMatchState_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 6);
++ return ZSTD_compressBlock_fast_dictMatchState_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+- return ZSTD_compressBlock_fast_dictMatchState_generic(ms, seqStore, rep, src, srcSize, 7);
++ return ZSTD_compressBlock_fast_dictMatchState_7(ms, seqStore, rep, src, srcSize);
+ }
+ }
+
+@@ -402,7 +431,7 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
+
+ /* switch to "regular" variant if extDict is invalidated due to maxDistance */
+ if (prefixStartIndex == dictStartIndex)
+- return ZSTD_compressBlock_fast_generic(ms, seqStore, rep, src, srcSize, mls);
++ return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize);
+
+ /* Search Loop */
+ while (ip < ilimit) { /* < instead of <=, because (ip+1) */
+@@ -475,6 +504,20 @@ static size_t ZSTD_compressBlock_fast_extDict_generic(
+ return (size_t)(iend - anchor);
+ }
+
++#define ZSTD_GEN_FN(mls) \
++ static size_t ZSTD_compressBlock_fast_extDict_##mls( \
++ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], \
++ void const* src, size_t srcSize) \
++ { \
++ return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, mls); \
++ }
++
++ZSTD_GEN_FN(4)
++ZSTD_GEN_FN(5)
++ZSTD_GEN_FN(6)
++ZSTD_GEN_FN(7)
++
++#undef ZSTD_GEN_FN
+
+ size_t ZSTD_compressBlock_fast_extDict(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM],
+@@ -485,12 +528,12 @@ size_t ZSTD_compressBlock_fast_extDict(
+ {
+ default: /* includes case 3 */
+ case 4 :
+- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 4);
++ return ZSTD_compressBlock_fast_extDict_4(ms, seqStore, rep, src, srcSize);
+ case 5 :
+- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 5);
++ return ZSTD_compressBlock_fast_extDict_5(ms, seqStore, rep, src, srcSize);
+ case 6 :
+- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 6);
++ return ZSTD_compressBlock_fast_extDict_6(ms, seqStore, rep, src, srcSize);
+ case 7 :
+- return ZSTD_compressBlock_fast_extDict_generic(ms, seqStore, rep, src, srcSize, 7);
++ return ZSTD_compressBlock_fast_extDict_7(ms, seqStore, rep, src, srcSize);
+ }
+ }
+diff --git a/lib/zstd/compress/zstd_lazy.c b/lib/zstd/compress/zstd_lazy.c
+index fb54d4e28a2b..1db22db5b1e9 100644
+--- a/lib/zstd/compress/zstd_lazy.c
++++ b/lib/zstd/compress/zstd_lazy.c
+@@ -392,55 +392,6 @@ ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms,
+ }
+
+
+-static size_t
+-ZSTD_BtFindBestMatch_selectMLS ( ZSTD_matchState_t* ms,
+- const BYTE* ip, const BYTE* const iLimit,
+- size_t* offsetPtr)
+-{
+- switch(ms->cParams.minMatch)
+- {
+- default : /* includes case 3 */
+- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
+- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict);
+- case 7 :
+- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict);
+- }
+-}
+-
+-
+-static size_t ZSTD_BtFindBestMatch_dictMatchState_selectMLS (
+- ZSTD_matchState_t* ms,
+- const BYTE* ip, const BYTE* const iLimit,
+- size_t* offsetPtr)
+-{
+- switch(ms->cParams.minMatch)
+- {
+- default : /* includes case 3 */
+- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
+- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState);
+- case 7 :
+- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState);
+- }
+-}
+-
+-
+-static size_t ZSTD_BtFindBestMatch_extDict_selectMLS (
+- ZSTD_matchState_t* ms,
+- const BYTE* ip, const BYTE* const iLimit,
+- size_t* offsetPtr)
+-{
+- switch(ms->cParams.minMatch)
+- {
+- default : /* includes case 3 */
+- case 4 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
+- case 5 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict);
+- case 7 :
+- case 6 : return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict);
+- }
+-}
+-
+-
+-
+ /* *********************************
+ * Hash Chain
+ ***********************************/
+@@ -595,7 +546,7 @@ void ZSTD_dedicatedDictSearch_lazy_loadDictionary(ZSTD_matchState_t* ms, const B
+
+ /* inlining is important to hardwire a hot branch (template emulation) */
+ FORCE_INLINE_TEMPLATE
+-size_t ZSTD_HcFindBestMatch_generic (
++size_t ZSTD_HcFindBestMatch(
+ ZSTD_matchState_t* ms,
+ const BYTE* const ip, const BYTE* const iLimit,
+ size_t* offsetPtr,
+@@ -783,76 +734,106 @@ size_t ZSTD_HcFindBestMatch_generic (
+ return ml;
+ }
+
++typedef size_t (*searchMax_f)(
++ ZSTD_matchState_t* ms,
++ const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
+
+-FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_selectMLS (
+- ZSTD_matchState_t* ms,
+- const BYTE* ip, const BYTE* const iLimit,
+- size_t* offsetPtr)
+-{
+- switch(ms->cParams.minMatch)
+- {
+- default : /* includes case 3 */
+- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_noDict);
+- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_noDict);
+- case 7 :
+- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_noDict);
+- }
+-}
++/*
++ * This struct contains the functions necessary for lazy to search.
++ * Currently, that is only searchMax. However, it is still valuable to have the
++ * VTable because this makes it easier to add more functions to the VTable later.
++ */
++typedef struct {
++ searchMax_f searchMax;
++} ZSTD_LazyVTable;
++
++#define GEN_ZSTD_BT_VTABLE(dictMode, mls, ...) \
++ static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \
++ ZSTD_matchState_t* ms, \
++ const BYTE* ip, const BYTE* const iLimit, \
++ size_t* offsetPtr) \
++ { \
++ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
++ return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \
++ } \
++ static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \
++ ZSTD_BtFindBestMatch_##dictMode##_##mls \
++ };
+
++#define GEN_ZSTD_HC_VTABLE(dictMode, mls, ...) \
++ static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \
++ ZSTD_matchState_t* ms, \
++ const BYTE* ip, const BYTE* const iLimit, \
++ size_t* offsetPtr) \
++ { \
++ assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \
++ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \
++ } \
++ static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \
++ ZSTD_HcFindBestMatch_##dictMode##_##mls \
++ };
+
+-static size_t ZSTD_HcFindBestMatch_dictMatchState_selectMLS (
+- ZSTD_matchState_t* ms,
+- const BYTE* ip, const BYTE* const iLimit,
+- size_t* offsetPtr)
+-{
+- switch(ms->cParams.minMatch)
+- {
+- default : /* includes case 3 */
+- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dictMatchState);
+- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dictMatchState);
+- case 7 :
+- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dictMatchState);
++#define ZSTD_FOR_EACH_MLS(X, dictMode) \
++ X(dictMode, 4) \
++ X(dictMode, 5) \
++ X(dictMode, 6)
++
++#define ZSTD_FOR_EACH_DICT_MODE(X, ...) \
++ X(__VA_ARGS__, noDict) \
++ X(__VA_ARGS__, extDict) \
++ X(__VA_ARGS__, dictMatchState) \
++ X(__VA_ARGS__, dedicatedDictSearch)
++
++/* Generate Binary Tree VTables for each combination of (dictMode, mls) */
++ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE)
++/* Generate Hash Chain VTables for each combination of (dictMode, mls) */
++ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE)
++
++#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \
++ { \
++ &ZSTD_BtVTable_##dictMode##_4, \
++ &ZSTD_BtVTable_##dictMode##_5, \
++ &ZSTD_BtVTable_##dictMode##_6 \
+ }
+-}
+-
+
+-static size_t ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS (
+- ZSTD_matchState_t* ms,
+- const BYTE* ip, const BYTE* const iLimit,
+- size_t* offsetPtr)
+-{
+- switch(ms->cParams.minMatch)
+- {
+- default : /* includes case 3 */
+- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_dedicatedDictSearch);
+- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_dedicatedDictSearch);
+- case 7 :
+- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_dedicatedDictSearch);
++#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \
++ { \
++ &ZSTD_HcVTable_##dictMode##_4, \
++ &ZSTD_HcVTable_##dictMode##_5, \
++ &ZSTD_HcVTable_##dictMode##_6 \
+ }
+-}
+
+-
+-FORCE_INLINE_TEMPLATE size_t ZSTD_HcFindBestMatch_extDict_selectMLS (
+- ZSTD_matchState_t* ms,
+- const BYTE* ip, const BYTE* const iLimit,
+- size_t* offsetPtr)
+-{
+- switch(ms->cParams.minMatch)
+- {
+- default : /* includes case 3 */
+- case 4 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 4, ZSTD_extDict);
+- case 5 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 5, ZSTD_extDict);
+- case 7 :
+- case 6 : return ZSTD_HcFindBestMatch_generic(ms, ip, iLimit, offsetPtr, 6, ZSTD_extDict);
++#define GEN_ZSTD_VTABLE_ARRAY(X) \
++ { \
++ X(noDict), \
++ X(extDict), \
++ X(dictMatchState), \
++ X(dedicatedDictSearch) \
+ }
+-}
+-
+
+ /* *******************************
+ * Common parser - lazy strategy
+ *********************************/
+ typedef enum { search_hashChain, search_binaryTree } searchMethod_e;
+
++static ZSTD_LazyVTable const* ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode)
++{
++ /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */
++ ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY);
++ ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY);
++ /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */
++
++ U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch));
++ switch (searchMethod) {
++ case search_hashChain:
++ return hcVTables[dictMode][mls - 4];
++ case search_binaryTree:
++ return btVTables[dictMode][mls - 4];
++ default:
++ return NULL;
++ }
++}
++
+ FORCE_INLINE_TEMPLATE size_t
+ ZSTD_compressBlock_lazy_generic(
+ ZSTD_matchState_t* ms, seqStore_t* seqStore,
+@@ -870,36 +851,13 @@ ZSTD_compressBlock_lazy_generic(
+ const U32 prefixLowestIndex = ms->window.dictLimit;
+ const BYTE* const prefixLowest = base + prefixLowestIndex;
+
+- typedef size_t (*searchMax_f)(
+- ZSTD_matchState_t* ms,
+- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
+-
+ /*
+ * This table is indexed first by the four ZSTD_dictMode_e values, and then
+ * by the two searchMethod_e values. NULLs are placed for configurations
+ * that should never occur (extDict modes go to the other implementation
+ * below and there is no DDSS for binary tree search yet).
+ */
+- const searchMax_f searchFuncs[4][2] = {
+- {
+- ZSTD_HcFindBestMatch_selectMLS,
+- ZSTD_BtFindBestMatch_selectMLS
+- },
+- {
+- NULL,
+- NULL
+- },
+- {
+- ZSTD_HcFindBestMatch_dictMatchState_selectMLS,
+- ZSTD_BtFindBestMatch_dictMatchState_selectMLS
+- },
+- {
+- ZSTD_HcFindBestMatch_dedicatedDictSearch_selectMLS,
+- NULL
+- }
+- };
+-
+- searchMax_f const searchMax = searchFuncs[dictMode][searchMethod == search_binaryTree];
++ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax;
+ U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0;
+
+ const int isDMS = dictMode == ZSTD_dictMatchState;
+@@ -1221,10 +1179,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic(
+ const BYTE* const dictStart = dictBase + ms->window.lowLimit;
+ const U32 windowLog = ms->cParams.windowLog;
+
+- typedef size_t (*searchMax_f)(
+- ZSTD_matchState_t* ms,
+- const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr);
+- searchMax_f searchMax = searchMethod==search_binaryTree ? ZSTD_BtFindBestMatch_extDict_selectMLS : ZSTD_HcFindBestMatch_extDict_selectMLS;
++ searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax;
+
+ U32 offset_1 = rep[0], offset_2 = rep[1];
+
+diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib
+index 79be57fdd32a..6c44e49f0f2b 100644
+--- a/scripts/Makefile.lib
++++ b/scripts/Makefile.lib
+@@ -384,13 +384,8 @@ dtc-tmp = $(subst $(comma),_,$(dot-target).dts.tmp)
+
+ # Bzip2 and LZMA do not include size in file... so we have to fake that;
+ # append the size as a 32-bit littleendian number as gzip does.
+-size_append = printf $(shell \
+-dec_size=0; \
+-for F in $(real-prereqs); do \
+- fsize=$$($(CONFIG_SHELL) $(srctree)/scripts/file-size.sh $$F); \
+- dec_size=$$(expr $$dec_size + $$fsize); \
+-done; \
+-printf "%08x\n" $$dec_size | \
++total_size = $(shell $(PERL) $(srctree)/scripts/file-size.pl $(real-prereqs))
++size_append = printf $(shell printf "%08x\n" $(total_size) | \
+ sed 's/\(..\)/\1 /g' | { \
+ read ch0 ch1 ch2 ch3; \
+ for ch in $$ch3 $$ch2 $$ch1 $$ch0; do \
+@@ -494,15 +489,20 @@ quiet_cmd_xzmisc = XZMISC $@
+ # single pass, so zstd doesn't need to allocate a window buffer. When streaming
+ # decompression is used, like initramfs decompression, zstd22 should likely not
+ # be used because it would require zstd to allocate a 128 MB buffer.
++#
++# --stream-size to reduce zstd memory usage (otherwise zstd -22 --ultra
++# allocates, but does not use, 2 GB) and potentially improve compression.
++#
++# --no-content-size to save three bytes which we do not use (we use size_append).
+
+-quiet_cmd_zstd = ZSTD $@
+- cmd_zstd = cat $(real-prereqs) | $(ZSTD) -19 > $@
++# zstd --stream-size is only supported since 1.4.4
++zstd_stream_size = $(shell $(ZSTD) -1c --stream-size=0 --no-content-size </dev/null >/dev/null 2>&1 && printf '%s' '--stream-size=$(total_size) --no-content-size')
+
+-quiet_cmd_zstd22 = ZSTD22 $@
+- cmd_zstd22 = cat $(real-prereqs) | $(ZSTD) -22 --ultra > $@
++quiet_cmd_zstd = ZSTD $@
++ cmd_zstd = { cat $(real-prereqs) | $(ZSTD) $(zstd_stream_size) -$(CONFIG_KERNEL_ZSTD_LEVEL) -T0; $(size_append); } > $@
+
+ quiet_cmd_zstd22_with_size = ZSTD22 $@
+- cmd_zstd22_with_size = { cat $(real-prereqs) | $(ZSTD) -22 --ultra; $(size_append); } > $@
++ cmd_zstd22 = { cat $(real-prereqs) | $(ZSTD) $(zstd_stream_size) -22 --ultra -T0; $(size_append); } > $@
+
+ # ASM offsets
+ # ---------------------------------------------------------------------------
+diff --git a/scripts/Makefile.modinst b/scripts/Makefile.modinst
+index c2c43a0ecfe0..c15e5395fd32 100644
+--- a/scripts/Makefile.modinst
++++ b/scripts/Makefile.modinst
+@@ -96,8 +96,13 @@ quiet_cmd_gzip = GZIP $@
+ cmd_gzip = $(KGZIP) -n -f $<
+ quiet_cmd_xz = XZ $@
+ cmd_xz = $(XZ) --lzma2=dict=2MiB -f $<
++ifdef CONFIG_MODULE_COMPRESS_ZSTD_ULTRA
+ quiet_cmd_zstd = ZSTD $@
+- cmd_zstd = $(ZSTD) -T0 --rm -f -q $<
++ cmd_zstd = $(ZSTD) -$(CONFIG_MODULE_COMPRESS_ZSTD_LEVEL_ULTRA) --ultra --zstd=wlog=21 -T0 --rm -f -q $<
++else
++quiet_cmd_zstd = ZSTD $@
++ cmd_zstd = $(ZSTD) -$(CONFIG_MODULE_COMPRESS_ZSTD_LEVEL) --zstd=wlog=21 -T0 --rm -f -q $<
++endif
+
+ $(dst)/%.ko.gz: $(dst)/%.ko FORCE
+ $(call cmd,gzip)
+diff --git a/scripts/file-size.pl b/scripts/file-size.pl
+new file mode 100755
+index 000000000000..170bb6d048fa
+--- /dev/null
++++ b/scripts/file-size.pl
+@@ -0,0 +1,8 @@
++#!/usr/bin/perl -w
++# SPDX-License-Identifier: GPL-2.0
++my $total = 0;
++foreach (@ARGV) {
++ @stat = stat $_ or die "$_: $!";
++ $total += $stat[7];
++}
++print "$total\n";
+diff --git a/scripts/file-size.sh b/scripts/file-size.sh
+deleted file mode 100755
+index 7eb7423416b5..000000000000
+--- a/scripts/file-size.sh
++++ /dev/null
+@@ -1,4 +0,0 @@
+-#!/bin/sh
+-# SPDX-License-Identifier: GPL-2.0
+-set -- $(ls -dn "$1")
+-printf '%s\n' "$5"
+diff --git a/scripts/link-vmlinux.sh b/scripts/link-vmlinux.sh
+index 666f7bbc13eb..ba819b9376ad 100755
+--- a/scripts/link-vmlinux.sh
++++ b/scripts/link-vmlinux.sh
+@@ -388,8 +388,8 @@ if is_enabled CONFIG_KALLSYMS; then
+ kallsyms_step 2
+
+ # step 3
+- size1=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" ${kallsymso_prev})
+- size2=$(${CONFIG_SHELL} "${srctree}/scripts/file-size.sh" ${kallsymso})
++ size1=$(${PERL} "${srctree}/scripts/file-size.pl" ${kallsymso_prev})
++ size2=$(${PERL} "${srctree}/scripts/file-size.pl" ${kallsymso})
+
+ if [ $size1 -ne $size2 ] || [ -n "${KALLSYMS_EXTRA_PASS}" ]; then
+ kallsyms_step 3
+--
+2.35.1
+